Messages in this thread Patch in this message | | | From | Rusty Russell <> | Subject | [PATCH 3/3] kmalloc_percpu: interface change. | Date | Tue, 20 May 2003 11:30:29 +1000 |
| |
Name: Change kmalloc_percpu interface Author: Rusty Russell Status: Tested on 2.5.69-bk13 Depends: Misc/percpu-up-unify.patch.gz Depends: Misc/percpu-smp-unify.patch.gz
D: Several tweaks to the kmalloc_percpu()/kfree_percpu() interface, to D: allow future implementations to be more flexible, and make easier to D: use now we can see how it's actually being used. D: D: 1) No flags argument: GFP_ATOMIC doesn't make much sense, D: 2) Explicit alignment argument, so we don't have to give SMP_CACHE_BYTES D: alignment always, D: 3) Zeros memory, since most callers want that and it's not entirely D: trivial, D: 4) Convenient type-safe wrapper which takes a typename, and D: 5) Rename to alloc_percpu/__alloc_percpu, since usage no longer D: matches kmalloc.
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/include/linux/percpu.h .22172-2.5.69-bk13-kmalloc_percpu-interface/include/linux/percpu.h --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/include/linux/percpu.h 2003-05-19 16:51:27.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/include/linux/percpu.h 2003-05-19 16:51:27.000000000 +1000 @@ -1,7 +1,8 @@ #ifndef __LINUX_PERCPU_H #define __LINUX_PERCPU_H #include <linux/spinlock.h> /* For preempt_disable() */ -#include <linux/slab.h> /* For kmalloc_percpu() */ +#include <linux/slab.h> /* For kmalloc() */ +#include <linux/string.h> /* For memset() */ #include <asm/percpu.h> /* Must be an lvalue. */ @@ -21,7 +22,7 @@ struct percpu_data { /* * Use this to get to a cpu's version of the per-cpu object allocated using - * kmalloc_percpu. If you want to get "this cpu's version", maybe you want + * alloc_percpu. If you want to get "this cpu's version", maybe you want * to use get_cpu_ptr... */ #define per_cpu_ptr(ptr, cpu) \ @@ -30,19 +31,22 @@ struct percpu_data { (__typeof__(ptr))__p->ptrs[(cpu)]; \ }) -extern void *kmalloc_percpu(size_t size, int flags); -extern void kfree_percpu(const void *); +extern void *__alloc_percpu(size_t size, size_t align); +extern void free_percpu(const void *); extern void kmalloc_percpu_init(void); #else /* CONFIG_SMP */ #define per_cpu_ptr(ptr, cpu) (ptr) -static inline void *kmalloc_percpu(size_t size, int flags) +static inline void *__alloc_percpu(size_t size, size_t align) { - return(kmalloc(size, flags)); + void *ret = kmalloc(size, GFP_KERNEL); + if (ret) + memset(ret, 0, size); + return ret; } -static inline void kfree_percpu(const void *ptr) +static inline void free_percpu(const void *ptr) { kfree(ptr); } @@ -59,9 +63,13 @@ static inline void kmalloc_percpu_init(v #endif /* CONFIG_SMP */ +/* Simple wrapper for the common case: zeros memory. */ +#define alloc_percpu(type) \ + ((type *)(__alloc_percpu(sizeof(type), __alignof__(type)))) + /* - * Use these with kmalloc_percpu. If - * 1. You want to operate on memory allocated by kmalloc_percpu (dereference + * Use these with alloc_percpu. If + * 1. You want to operate on memory allocated by alloc_percpu (dereference * and read/modify/write) AND * 2. You want "this cpu's version" of the object AND * 3. You want to do this safely since: diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/include/linux/genhd.h .22172-2.5.69-bk13-kmalloc_percpu-interface/include/linux/genhd.h --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/include/linux/genhd.h 2003-05-05 12:37:12.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/include/linux/genhd.h 2003-05-19 16:51:27.000000000 +1000 @@ -160,16 +160,15 @@ static inline void disk_stat_set_all(str #ifdef CONFIG_SMP static inline int init_disk_stats(struct gendisk *disk) { - disk->dkstats = kmalloc_percpu(sizeof (struct disk_stats), GFP_KERNEL); + disk->dkstats = alloc_percpu(struct disk_stats); if (!disk->dkstats) return 0; - disk_stat_set_all(disk, 0); return 1; } static inline void free_disk_stats(struct gendisk *disk) { - kfree_percpu(disk->dkstats); + free_percpu(disk->dkstats); } #else /* CONFIG_SMP */ static inline int init_disk_stats(struct gendisk *disk) diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/include/net/ipv6.h .22172-2.5.69-bk13-kmalloc_percpu-interface/include/net/ipv6.h --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/include/net/ipv6.h 2003-05-19 10:53:51.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/include/net/ipv6.h 2003-05-19 16:51:27.000000000 +1000 @@ -145,7 +145,7 @@ extern atomic_t inet6_sock_nr; int snmp6_register_dev(struct inet6_dev *idev); int snmp6_unregister_dev(struct inet6_dev *idev); -int snmp6_mib_init(void *ptr[2], size_t mibsize); +int snmp6_mib_init(void *ptr[2], size_t mibsize, size_t mibalign); void snmp6_mib_free(void *ptr[2]); struct ip6_ra_chain diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/kernel/ksyms.c .22172-2.5.69-bk13-kmalloc_percpu-interface/kernel/ksyms.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/kernel/ksyms.c 2003-05-19 10:53:51.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/kernel/ksyms.c 2003-05-19 16:51:27.000000000 +1000 @@ -98,8 +98,8 @@ EXPORT_SYMBOL(remove_shrinker); EXPORT_SYMBOL(kmalloc); EXPORT_SYMBOL(kfree); #ifdef CONFIG_SMP -EXPORT_SYMBOL(kmalloc_percpu); -EXPORT_SYMBOL(kfree_percpu); +EXPORT_SYMBOL(__alloc_percpu); +EXPORT_SYMBOL(free_percpu); EXPORT_SYMBOL(percpu_counter_mod); #endif EXPORT_SYMBOL(vfree); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/mm/slab.c .22172-2.5.69-bk13-kmalloc_percpu-interface/mm/slab.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/mm/slab.c 2003-05-19 10:53:52.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/mm/slab.c 2003-05-19 16:51:27.000000000 +1000 @@ -1939,26 +1939,18 @@ void * kmalloc (size_t size, int flags) #ifdef CONFIG_SMP /** - * kmalloc_percpu - allocate one copy of the object for every present - * cpu in the system. + * __alloc_percpu - allocate one copy of the object for every present + * cpu in the system, zeroing them. * Objects should be dereferenced using per_cpu_ptr/get_cpu_ptr * macros only. * * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate. - * The @flags argument may be one of: - * - * %GFP_USER - Allocate memory on behalf of user. May sleep. - * - * %GFP_KERNEL - Allocate normal kernel ram. May sleep. - * - * %GFP_ATOMIC - Allocation will not sleep. Use inside interrupt handlers. + * @align: the alignment, which can't be greater than SMP_CACHE_BYTES. */ -void * -kmalloc_percpu(size_t size, int flags) +void *__alloc_percpu(size_t size, size_t align) { int i; - struct percpu_data *pdata = kmalloc(sizeof (*pdata), flags); + struct percpu_data *pdata = kmalloc(sizeof (*pdata), GFP_KERNEL); if (!pdata) return NULL; @@ -1966,9 +1958,10 @@ kmalloc_percpu(size_t size, int flags) for (i = 0; i < NR_CPUS; i++) { if (!cpu_possible(i)) continue; - pdata->ptrs[i] = kmalloc(size, flags); + pdata->ptrs[i] = kmalloc(size, GFP_KERNEL); if (!pdata->ptrs[i]) goto unwind_oom; + memset(pdata->ptrs[i], 0, size); } /* Catch derefs w/o wrappers */ @@ -2025,14 +2018,14 @@ void kfree (const void *objp) #ifdef CONFIG_SMP /** - * kfree_percpu - free previously allocated percpu memory - * @objp: pointer returned by kmalloc_percpu. + * free_percpu - free previously allocated percpu memory + * @objp: pointer returned by alloc_percpu. * - * Don't free memory not originally allocated by kmalloc_percpu() + * Don't free memory not originally allocated by alloc_percpu() * The complemented objp is to check for that. */ void -kfree_percpu(const void *objp) +free_percpu(const void *objp) { int i; struct percpu_data *p = (struct percpu_data *) (~(unsigned long) objp); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv4/af_inet.c .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv4/af_inet.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv4/af_inet.c 2003-05-19 10:53:53.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv4/af_inet.c 2003-05-19 16:51:27.000000000 +1000 @@ -1068,54 +1068,22 @@ static struct inet_protocol icmp_protoco static int __init init_ipv4_mibs(void) { - int i; - - net_statistics[0] = - kmalloc_percpu(sizeof (struct linux_mib), GFP_KERNEL); - net_statistics[1] = - kmalloc_percpu(sizeof (struct linux_mib), GFP_KERNEL); - ip_statistics[0] = kmalloc_percpu(sizeof (struct ip_mib), GFP_KERNEL); - ip_statistics[1] = kmalloc_percpu(sizeof (struct ip_mib), GFP_KERNEL); - icmp_statistics[0] = - kmalloc_percpu(sizeof (struct icmp_mib), GFP_KERNEL); - icmp_statistics[1] = - kmalloc_percpu(sizeof (struct icmp_mib), GFP_KERNEL); - tcp_statistics[0] = kmalloc_percpu(sizeof (struct tcp_mib), GFP_KERNEL); - tcp_statistics[1] = kmalloc_percpu(sizeof (struct tcp_mib), GFP_KERNEL); - udp_statistics[0] = kmalloc_percpu(sizeof (struct udp_mib), GFP_KERNEL); - udp_statistics[1] = kmalloc_percpu(sizeof (struct udp_mib), GFP_KERNEL); + net_statistics[0] = alloc_percpu(struct linux_mib); + net_statistics[1] = alloc_percpu(struct linux_mib); + ip_statistics[0] = alloc_percpu(struct ip_mib); + ip_statistics[1] = alloc_percpu(struct ip_mib); + icmp_statistics[0] = alloc_percpu(struct icmp_mib); + icmp_statistics[1] = alloc_percpu(struct icmp_mib); + tcp_statistics[0] = alloc_percpu(struct tcp_mib); + tcp_statistics[1] = alloc_percpu(struct tcp_mib); + udp_statistics[0] = alloc_percpu(struct udp_mib); + udp_statistics[1] = alloc_percpu(struct udp_mib); if (! (net_statistics[0] && net_statistics[1] && ip_statistics[0] && ip_statistics[1] && tcp_statistics[0] && tcp_statistics[1] && udp_statistics[0] && udp_statistics[1])) return -ENOMEM; - /* Set all the per cpu copies of the mibs to zero */ - for (i = 0; i < NR_CPUS; i++) { - if (cpu_possible(i)) { - memset(per_cpu_ptr(net_statistics[0], i), 0, - sizeof (struct linux_mib)); - memset(per_cpu_ptr(net_statistics[1], i), 0, - sizeof (struct linux_mib)); - memset(per_cpu_ptr(ip_statistics[0], i), 0, - sizeof (struct ip_mib)); - memset(per_cpu_ptr(ip_statistics[1], i), 0, - sizeof (struct ip_mib)); - memset(per_cpu_ptr(icmp_statistics[0], i), 0, - sizeof (struct icmp_mib)); - memset(per_cpu_ptr(icmp_statistics[1], i), 0, - sizeof (struct icmp_mib)); - memset(per_cpu_ptr(tcp_statistics[0], i), 0, - sizeof (struct tcp_mib)); - memset(per_cpu_ptr(tcp_statistics[1], i), 0, - sizeof (struct tcp_mib)); - memset(per_cpu_ptr(udp_statistics[0], i), 0, - sizeof (struct udp_mib)); - memset(per_cpu_ptr(udp_statistics[1], i), 0, - sizeof (struct udp_mib)); - } - } - (void) tcp_mib_init(); return 0; diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv4/route.c .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv4/route.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv4/route.c 2003-05-19 10:53:53.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv4/route.c 2003-05-19 16:51:27.000000000 +1000 @@ -2695,16 +2695,9 @@ int __init ip_rt_init(void) ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1); ip_rt_max_size = (rt_hash_mask + 1) * 16; - rt_cache_stat = kmalloc_percpu(sizeof (struct rt_cache_stat), - GFP_KERNEL); + rt_cache_stat = alloc_percpu(struct rt_cache_stat); if (!rt_cache_stat) goto out_enomem1; - for (i = 0; i < NR_CPUS; i++) { - if (cpu_possible(i)) { - memset(per_cpu_ptr(rt_cache_stat, i), 0, - sizeof (struct rt_cache_stat)); - } - } devinet_init(); ip_fib_init(); @@ -2740,7 +2733,7 @@ int __init ip_rt_init(void) out: return rc; out_enomem: - kfree_percpu(rt_cache_stat); + free_percpu(rt_cache_stat); out_enomem1: rc = -ENOMEM; goto out; diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv6/af_inet6.c .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv6/af_inet6.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv6/af_inet6.c 2003-05-19 10:53:53.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv6/af_inet6.c 2003-05-19 16:51:27.000000000 +1000 @@ -637,32 +637,25 @@ inet6_unregister_protosw(struct inet_pro } int -snmp6_mib_init(void *ptr[2], size_t mibsize) +snmp6_mib_init(void *ptr[2], size_t mibsize, size_t mibalign) { int i; if (ptr == NULL) return -EINVAL; - ptr[0] = kmalloc_percpu(mibsize, GFP_KERNEL); + ptr[0] = __alloc_percpu(mibsize, mibalign); if (!ptr[0]) goto err0; - ptr[1] = kmalloc_percpu(mibsize, GFP_KERNEL); + ptr[1] = __alloc_percpu(mibsize, mibalign); if (!ptr[1]) goto err1; - /* Zero percpu version of the mibs */ - for (i = 0; i < NR_CPUS; i++) { - if (cpu_possible(i)) { - memset(per_cpu_ptr(ptr[0], i), 0, mibsize); - memset(per_cpu_ptr(ptr[1], i), 0, mibsize); - } - } return 0; err1: - kfree_percpu(ptr[0]); + free_percpu(ptr[0]); ptr[0] = NULL; err0: return -ENOMEM; @@ -673,18 +666,21 @@ snmp6_mib_free(void *ptr[2]) { if (ptr == NULL) return; - kfree_percpu(ptr[0]); - kfree_percpu(ptr[1]); + free_percpu(ptr[0]); + free_percpu(ptr[1]); ptr[0] = ptr[1] = NULL; } static int __init init_ipv6_mibs(void) { - if (snmp6_mib_init((void **)ipv6_statistics, sizeof (struct ipv6_mib)) < 0) + if (snmp6_mib_init((void **)ipv6_statistics, sizeof (struct ipv6_mib), + __alignof__(struct ipv6_mib)) < 0) goto err_ip_mib; - if (snmp6_mib_init((void **)icmpv6_statistics, sizeof (struct icmpv6_mib)) < 0) + if (snmp6_mib_init((void **)icmpv6_statistics, sizeof (struct icmpv6_mib), + __alignof__(struct ipv6_mib)) < 0) goto err_icmp_mib; - if (snmp6_mib_init((void **)udp_stats_in6, sizeof (struct udp_mib)) < 0) + if (snmp6_mib_init((void **)udp_stats_in6, sizeof (struct udp_mib), + __alignof__(struct ipv6_mib)) < 0) goto err_udp_mib; return 0; diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv6/proc.c .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv6/proc.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/ipv6/proc.c 2003-05-19 10:53:53.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/net/ipv6/proc.c 2003-05-19 16:51:27.000000000 +1000 @@ -228,7 +228,8 @@ int snmp6_register_dev(struct inet6_dev if (!idev || !idev->dev) return -EINVAL; - if (snmp6_mib_init((void **)idev->stats.icmpv6, sizeof(struct icmpv6_mib)) < 0) + if (snmp6_mib_init((void **)idev->stats.icmpv6, sizeof(struct icmpv6_mib), + __alignof__(struct ipv6_mib)) < 0) goto err_icmp; #ifdef CONFIG_PROC_FS diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/sctp/protocol.c .22172-2.5.69-bk13-kmalloc_percpu-interface/net/sctp/protocol.c --- .22172-2.5.69-bk13-kmalloc_percpu-interface.pre/net/sctp/protocol.c 2003-05-19 10:53:54.000000000 +1000 +++ .22172-2.5.69-bk13-kmalloc_percpu-interface/net/sctp/protocol.c 2003-05-19 16:51:27.000000000 +1000 @@ -880,34 +880,22 @@ static int __init init_sctp_mibs(void) { int i; - sctp_statistics[0] = kmalloc_percpu(sizeof (struct sctp_mib), - GFP_KERNEL); + sctp_statistics[0] = alloc_percpu(struct sctp_mib); if (!sctp_statistics[0]) return -ENOMEM; - sctp_statistics[1] = kmalloc_percpu(sizeof (struct sctp_mib), - GFP_KERNEL); + sctp_statistics[1] = alloc_percpu(struct sctp_mib); if (!sctp_statistics[1]) { - kfree_percpu(sctp_statistics[0]); + free_percpu(sctp_statistics[0]); return -ENOMEM; } - - /* Zero all percpu versions of the mibs */ - for (i = 0; i < NR_CPUS; i++) { - if (cpu_possible(i)) { - memset(per_cpu_ptr(sctp_statistics[0], i), 0, - sizeof (struct sctp_mib)); - memset(per_cpu_ptr(sctp_statistics[1], i), 0, - sizeof (struct sctp_mib)); - } - } return 0; } static void cleanup_sctp_mibs(void) { - kfree_percpu(sctp_statistics[0]); - kfree_percpu(sctp_statistics[1]); + free_percpu(sctp_statistics[0]); + free_percpu(sctp_statistics[1]); } /* Initialize the universe into something sensible. */ -- Anyone who quotes me in their sig is an idiot. -- Rusty Russell. - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
| |