lkml.org 
[lkml]   [2009]   [Oct]   [14]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
Date
From
SubjectRe: [PATCH 7/9] swap_info: swap count continuations
On Thu, 15 Oct 2009 01:56:01 +0100 (BST)
Hugh Dickins <hugh.dickins@tiscali.co.uk> wrote:

> Swap is duplicated (reference count incremented by one) whenever the same
> swap page is inserted into another mm (when forking finds a swap entry in
> place of a pte, or when reclaim unmaps a pte to insert the swap entry).
>
> swap_info_struct's vmalloc'ed swap_map is the array of these reference
> counts: but what happens when the unsigned short (or unsigned char since
> the preceding patch) is full? (and its high bit is kept for a cache flag)
>
> We then lose track of it, never freeing, leaving it in use until swapoff:
> at which point we _hope_ that a single pass will have found all instances,
> assume there are no more, and will lose user data if we're wrong.
>
> Swapping of KSM pages has not yet been enabled; but it is implemented,
> and makes it very easy for a user to overflow the maximum swap count:
> possible with ordinary process pages, but unlikely, even when pid_max
> has been raised from PID_MAX_DEFAULT.
>
> This patch implements swap count continuations: when the count overflows,
> a continuation page is allocated and linked to the original vmalloc'ed
> map page, and this used to hold the continuation counts for that entry
> and its neighbours. These continuation pages are seldom referenced:
> the common paths all work on the original swap_map, only referring to
> a continuation page when the low "digit" of a count is incremented or
> decremented through SWAP_MAP_MAX.
>
> Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>


Hmm...maybe I don't understand the benefit of this style of data structure.

Do we need fine grain chain ?
Is array of "unsigned long" counter is bad ? (too big?)

==
#define EXTENTION_OFFSET_INDEX(offset) (((offset) & PAGE_MASK)
#define EXTENTION_OFFSET_MASK (~(PAGE_SIZE/sizeof(long) - 1))
struct swapcount_extention_array {
unsigned long *map[EXTEND_MAP_SIZE];
};

At adding continuation.

int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
{
struct page *page;
unsigned long *newmap, *map;
struct swapcount_extention_array *array;

newmap = __get_free_page(mask);
si = swap_info_get(entry);
array = kmalloc(sizeof(swapcount_extention_array);

....
(If overflow)
page = vmalloc_to_page(si->swap_map + offset);
if (!PagePrivate(page)) {
page->praivate = array;
} else
kfree(array);

index = EXTENTION_OFFSET_INDEX(offset);
pos = EXTENTION_OFFSET_MASK(offset);

array = page->private;
if (!array->map[index]) {
array->map[index] = newmap;
} else
free_page(newmap);
map = array->map[index];
map[pos] += 1;
mappage = vaddr_to_page(map);
get_page(mappage); # increment page->count of array.
==

Hmm? maybe I just don't like chain...

Regards,
-Kame


> ---
>
> include/linux/swap.h | 22 ++
> mm/memory.c | 19 ++
> mm/rmap.c | 6
> mm/swapfile.c | 304 +++++++++++++++++++++++++++++++++--------
> 4 files changed, 287 insertions(+), 64 deletions(-)
>
> --- si6/include/linux/swap.h 2009-10-14 21:26:49.000000000 +0100
> +++ si7/include/linux/swap.h 2009-10-14 21:26:57.000000000 +0100
> @@ -145,15 +145,18 @@ enum {
> SWP_DISCARDABLE = (1 << 2), /* blkdev supports discard */
> SWP_DISCARDING = (1 << 3), /* now discarding a free cluster */
> SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */
> + SWP_CONTINUED = (1 << 5), /* swap_map has count continuation */
> /* add others here before... */
> SWP_SCANNING = (1 << 8), /* refcount in scan_swap_map */
> };
>
> #define SWAP_CLUSTER_MAX 32
>
> -#define SWAP_MAP_MAX 0x7e
> -#define SWAP_MAP_BAD 0x7f
> -#define SWAP_HAS_CACHE 0x80 /* There is a swap cache of entry. */
> +#define SWAP_MAP_MAX 0x3e /* Max duplication count, in first swap_map */
> +#define SWAP_MAP_BAD 0x3f /* Note pageblock is bad, in first swap_map */
> +#define SWAP_HAS_CACHE 0x40 /* Flag page is cached, in first swap_map */
> +#define SWAP_CONT_MAX 0x7f /* Max count, in each swap_map continuation */
> +#define COUNT_CONTINUED 0x80 /* See swap_map continuation for full count */
>
> /*
> * The in-memory structure used to track swap areas.
> @@ -310,9 +313,10 @@ extern long total_swap_pages;
> extern void si_swapinfo(struct sysinfo *);
> extern swp_entry_t get_swap_page(void);
> extern swp_entry_t get_swap_page_of_type(int);
> -extern void swap_duplicate(swp_entry_t);
> -extern int swapcache_prepare(swp_entry_t);
> extern int valid_swaphandles(swp_entry_t, unsigned long *);
> +extern int add_swap_count_continuation(swp_entry_t, gfp_t);
> +extern int swap_duplicate(swp_entry_t);
> +extern int swapcache_prepare(swp_entry_t);
> extern void swap_free(swp_entry_t);
> extern void swapcache_free(swp_entry_t, struct page *page);
> extern int free_swap_and_cache(swp_entry_t);
> @@ -384,8 +388,14 @@ static inline void show_swap_cache_info(
> #define free_swap_and_cache(swp) is_migration_entry(swp)
> #define swapcache_prepare(swp) is_migration_entry(swp)
>
> -static inline void swap_duplicate(swp_entry_t swp)
> +static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask)
> {
> + return 0;
> +}
> +
> +static inline int swap_duplicate(swp_entry_t swp)
> +{
> + return 0;
> }
>
> static inline void swap_free(swp_entry_t swp)
> --- si6/mm/memory.c 2009-09-28 00:28:41.000000000 +0100
> +++ si7/mm/memory.c 2009-10-14 21:26:57.000000000 +0100
> @@ -572,7 +572,7 @@ out:
> * covered by this vma.
> */
>
> -static inline void
> +static inline unsigned long
> copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
> unsigned long addr, int *rss)
> @@ -586,7 +586,9 @@ copy_one_pte(struct mm_struct *dst_mm, s
> if (!pte_file(pte)) {
> swp_entry_t entry = pte_to_swp_entry(pte);
>
> - swap_duplicate(entry);
> + if (swap_duplicate(entry) < 0)
> + return entry.val;
> +
> /* make sure dst_mm is on swapoff's mmlist. */
> if (unlikely(list_empty(&dst_mm->mmlist))) {
> spin_lock(&mmlist_lock);
> @@ -635,6 +637,7 @@ copy_one_pte(struct mm_struct *dst_mm, s
>
> out_set_pte:
> set_pte_at(dst_mm, addr, dst_pte, pte);
> + return 0;
> }
>
> static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> @@ -645,6 +648,7 @@ static int copy_pte_range(struct mm_stru
> spinlock_t *src_ptl, *dst_ptl;
> int progress = 0;
> int rss[2];
> + swp_entry_t entry = (swp_entry_t){0};
>
> again:
> rss[1] = rss[0] = 0;
> @@ -671,7 +675,10 @@ again:
> progress++;
> continue;
> }
> - copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
> + entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
> + vma, addr, rss);
> + if (entry.val)
> + break;
> progress += 8;
> } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
>
> @@ -681,6 +688,12 @@ again:
> add_mm_rss(dst_mm, rss[0], rss[1]);
> pte_unmap_unlock(dst_pte - 1, dst_ptl);
> cond_resched();
> +
> + if (entry.val) {
> + if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
> + return -ENOMEM;
> + progress = 0;
> + }
> if (addr != end)
> goto again;
> return 0;
> --- si6/mm/rmap.c 2009-10-05 04:33:14.000000000 +0100
> +++ si7/mm/rmap.c 2009-10-14 21:26:57.000000000 +0100
> @@ -822,7 +822,11 @@ static int try_to_unmap_one(struct page
> * Store the swap location in the pte.
> * See handle_pte_fault() ...
> */
> - swap_duplicate(entry);
> + if (swap_duplicate(entry) < 0) {
> + set_pte_at(mm, address, pte, pteval);
> + ret = SWAP_FAIL;
> + goto out_unmap;
> + }
> if (list_empty(&mm->mmlist)) {
> spin_lock(&mmlist_lock);
> if (list_empty(&mm->mmlist))
> --- si6/mm/swapfile.c 2009-10-14 21:26:49.000000000 +0100
> +++ si7/mm/swapfile.c 2009-10-14 21:26:57.000000000 +0100
> @@ -35,11 +35,14 @@
> #include <linux/swapops.h>
> #include <linux/page_cgroup.h>
>
> +static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
> + unsigned char);
> +static void free_swap_count_continuations(struct swap_info_struct *);
> +
> static DEFINE_SPINLOCK(swap_lock);
> static unsigned int nr_swapfiles;
> long nr_swap_pages;
> long total_swap_pages;
> -static int swap_overflow;
> static int least_priority;
>
> static const char Bad_file[] = "Bad swap file entry ";
> @@ -55,7 +58,7 @@ static DEFINE_MUTEX(swapon_mutex);
>
> static inline unsigned char swap_count(unsigned char ent)
> {
> - return ent & ~SWAP_HAS_CACHE;
> + return ent & ~SWAP_HAS_CACHE; /* may include SWAP_HAS_CONT flag */
> }
>
> /* returns 1 if swap entry is freed */
> @@ -545,8 +548,15 @@ static unsigned char swap_entry_free(str
> if (usage == SWAP_HAS_CACHE) {
> VM_BUG_ON(!has_cache);
> has_cache = 0;
> - } else if (count < SWAP_MAP_MAX)
> - count--;
> + } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
> + if (count == COUNT_CONTINUED) {
> + if (swap_count_continued(p, offset, count))
> + count = SWAP_MAP_MAX | COUNT_CONTINUED;
> + else
> + count = SWAP_MAP_MAX;
> + } else
> + count--;
> + }
>
> if (!count)
> mem_cgroup_uncharge_swap(entry);
> @@ -604,6 +614,8 @@ void swapcache_free(swp_entry_t entry, s
>
> /*
> * How many references to page are currently swapped out?
> + * This does not give an exact answer when swap count is continued,
> + * but does include the high COUNT_CONTINUED flag to allow for that.
> */
> static inline int page_swapcount(struct page *page)
> {
> @@ -1019,7 +1031,6 @@ static int try_to_unuse(unsigned int typ
> swp_entry_t entry;
> unsigned int i = 0;
> int retval = 0;
> - int reset_overflow = 0;
> int shmem;
>
> /*
> @@ -1034,8 +1045,7 @@ static int try_to_unuse(unsigned int typ
> * together, child after parent. If we race with dup_mmap(), we
> * prefer to resolve parent before child, lest we miss entries
> * duplicated after we scanned child: using last mm would invert
> - * that. Though it's only a serious concern when an overflowed
> - * swap count is reset from SWAP_MAP_MAX, preventing a rescan.
> + * that.
> */
> start_mm = &init_mm;
> atomic_inc(&init_mm.mm_users);
> @@ -1166,36 +1176,6 @@ static int try_to_unuse(unsigned int typ
> }
>
> /*
> - * How could swap count reach 0x7ffe ?
> - * There's no way to repeat a swap page within an mm
> - * (except in shmem, where it's the shared object which takes
> - * the reference count)?
> - * We believe SWAP_MAP_MAX cannot occur.(if occur, unsigned
> - * short is too small....)
> - * If that's wrong, then we should worry more about
> - * exit_mmap() and do_munmap() cases described above:
> - * we might be resetting SWAP_MAP_MAX too early here.
> - *
> - * Yes, that's wrong: though very unlikely, swap count 0x7ffe
> - * could surely occur if pid_max raised from PID_MAX_DEFAULT;
> - * and we are now lowering SWAP_MAP_MAX to 0x7e, making it
> - * much easier to reach. But the next patch will fix that.
> - *
> - * We know "Undead"s can happen, they're okay, so don't
> - * report them; but do report if we reset SWAP_MAP_MAX.
> - */
> - /* We might release the lock_page() in unuse_mm(). */
> - if (!PageSwapCache(page) || page_private(page) != entry.val)
> - goto retry;
> -
> - if (swap_count(*swap_map) == SWAP_MAP_MAX) {
> - spin_lock(&swap_lock);
> - *swap_map = SWAP_HAS_CACHE;
> - spin_unlock(&swap_lock);
> - reset_overflow = 1;
> - }
> -
> - /*
> * If a reference remains (rare), we would like to leave
> * the page in the swap cache; but try_to_unmap could
> * then re-duplicate the entry once we drop page lock,
> @@ -1236,7 +1216,6 @@ static int try_to_unuse(unsigned int typ
> * mark page dirty so shrink_page_list will preserve it.
> */
> SetPageDirty(page);
> -retry:
> unlock_page(page);
> page_cache_release(page);
>
> @@ -1248,10 +1227,6 @@ retry:
> }
>
> mmput(start_mm);
> - if (reset_overflow) {
> - printk(KERN_WARNING "swapoff: cleared swap entry overflow\n");
> - swap_overflow = 0;
> - }
> return retval;
> }
>
> @@ -1595,6 +1570,9 @@ SYSCALL_DEFINE1(swapoff, const char __us
> up_write(&swap_unplug_sem);
>
> destroy_swap_extents(p);
> + if (p->flags & SWP_CONTINUED)
> + free_swap_count_continuations(p);
> +
> mutex_lock(&swapon_mutex);
> spin_lock(&swap_lock);
> drain_mmlist();
> @@ -2080,14 +2058,13 @@ void si_swapinfo(struct sysinfo *val)
> /*
> * Verify that a swap entry is valid and increment its swap map count.
> *
> - * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
> - * "permanent", but will be reclaimed by the next swapoff.
> * Returns error code in following case.
> * - success -> 0
> * - swp_entry is invalid -> EINVAL
> * - swp_entry is migration entry -> EINVAL
> * - swap-cache reference is requested but there is already one. -> EEXIST
> * - swap-cache reference is requested but the entry is not used. -> ENOENT
> + * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
> */
> static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
> {
> @@ -2127,15 +2104,14 @@ static int __swap_duplicate(swp_entry_t
>
> } else if (count || has_cache) {
>
> - if (count < SWAP_MAP_MAX - 1)
> - count++;
> - else if (count <= SWAP_MAP_MAX) {
> - if (swap_overflow++ < 5)
> - printk(KERN_WARNING
> - "swap_dup: swap entry overflow\n");
> - count = SWAP_MAP_MAX;
> - } else
> + if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
> + count += usage;
> + else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
> err = -EINVAL;
> + else if (swap_count_continued(p, offset, count))
> + count = COUNT_CONTINUED;
> + else
> + err = -ENOMEM;
> } else
> err = -ENOENT; /* unused swap entry */
>
> @@ -2154,9 +2130,13 @@ bad_file:
> /*
> * increase reference count of swap entry by 1.
> */
> -void swap_duplicate(swp_entry_t entry)
> +int swap_duplicate(swp_entry_t entry)
> {
> - __swap_duplicate(entry, 1);
> + int err = 0;
> +
> + while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
> + err = add_swap_count_continuation(entry, GFP_ATOMIC);
> + return err;
> }
>
> /*
> @@ -2223,3 +2203,219 @@ int valid_swaphandles(swp_entry_t entry,
> *offset = ++toff;
> return nr_pages? ++nr_pages: 0;
> }
> +
> +/*
> + * add_swap_count_continuation - called when a swap count is duplicated
> + * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
> + * page of the original vmalloc'ed swap_map, to hold the continuation count
> + * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called
> + * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
> + *
> + * These continuation pages are seldom referenced: the common paths all work
> + * on the original swap_map, only referring to a continuation page when the
> + * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
> + *
> + * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
> + * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
> + * can be called after dropping locks.
> + */
> +int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
> +{
> + struct swap_info_struct *si;
> + struct page *head;
> + struct page *page;
> + struct page *list_page;
> + pgoff_t offset;
> + unsigned char count;
> +
> + /*
> + * When debugging, it's easier to use __GFP_ZERO here; but it's better
> + * for latency not to zero a page while GFP_ATOMIC and holding locks.
> + */
> + page = alloc_page(gfp_mask | __GFP_HIGHMEM);
> +
> + si = swap_info_get(entry);
> + if (!si) {
> + /*
> + * An acceptable race has occurred since the failing
> + * __swap_duplicate(): the swap entry has been freed,
> + * perhaps even the whole swap_map cleared for swapoff.
> + */
> + goto outer;
> + }
> +
> + offset = swp_offset(entry);
> + count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
> +
> + if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
> + /*
> + * The higher the swap count, the more likely it is that tasks
> + * will race to add swap count continuation: we need to avoid
> + * over-provisioning.
> + */
> + goto out;
> + }
> +
> + if (!page) {
> + spin_unlock(&swap_lock);
> + return -ENOMEM;
> + }
> +
> + /*
> + * We are fortunate that although vmalloc_to_page uses pte_offset_map,
> + * no architecture is using highmem pages for kernel pagetables: so it
> + * will not corrupt the GFP_ATOMIC caller's atomic pagetable kmaps.
> + */
> + head = vmalloc_to_page(si->swap_map + offset);
> + offset &= ~PAGE_MASK;
> +
> + /*
> + * Page allocation does not initialize the page's lru field,
> + * but it does always reset its private field.
> + */
> + if (!page_private(head)) {
> + BUG_ON(count & COUNT_CONTINUED);
> + INIT_LIST_HEAD(&head->lru);
> + set_page_private(head, SWP_CONTINUED);
> + si->flags |= SWP_CONTINUED;
> + }
> +
> + list_for_each_entry(list_page, &head->lru, lru) {
> + unsigned char *map;
> +
> + /*
> + * If the previous map said no continuation, but we've found
> + * a continuation page, free our allocation and use this one.
> + */
> + if (!(count & COUNT_CONTINUED))
> + goto out;
> +
> + map = kmap_atomic(list_page, KM_USER0) + offset;
> + count = *map;
> + kunmap_atomic(map, KM_USER0);
> +
> + /*
> + * If this continuation count now has some space in it,
> + * free our allocation and use this one.
> + */
> + if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
> + goto out;
> + }
> +
> + list_add_tail(&page->lru, &head->lru);
> + page = NULL; /* now it's attached, don't free it */
> +out:
> + spin_unlock(&swap_lock);
> +outer:
> + if (page)
> + __free_page(page);
> + return 0;
> +}
> +
> +/*
> + * swap_count_continued - when the original swap_map count is incremented
> + * from SWAP_MAP_MAX, check if there is already a continuation page to carry
> + * into, carry if so, or else fail until a new continuation page is allocated;
> + * when the original swap_map count is decremented from 0 with continuation,
> + * borrow from the continuation and report whether it still holds more.
> + * Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
> + */
> +static bool swap_count_continued(struct swap_info_struct *si,
> + pgoff_t offset, unsigned char count)
> +{
> + struct page *head;
> + struct page *page;
> + unsigned char *map;
> +
> + head = vmalloc_to_page(si->swap_map + offset);
> + if (page_private(head) != SWP_CONTINUED) {
> + BUG_ON(count & COUNT_CONTINUED);
> + return false; /* need to add count continuation */
> + }
> +
> + offset &= ~PAGE_MASK;
> + page = list_entry(head->lru.next, struct page, lru);
> + map = kmap_atomic(page, KM_USER0) + offset;
> +
> + if (count == SWAP_MAP_MAX) /* initial increment from swap_map */
> + goto init_map; /* jump over SWAP_CONT_MAX checks */
> +
> + if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
> + /*
> + * Think of how you add 1 to 999
> + */
> + while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.next, struct page, lru);
> + BUG_ON(page == head);
> + map = kmap_atomic(page, KM_USER0) + offset;
> + }
> + if (*map == SWAP_CONT_MAX) {
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.next, struct page, lru);
> + if (page == head)
> + return false; /* add count continuation */
> + map = kmap_atomic(page, KM_USER0) + offset;
> +init_map: *map = 0; /* we didn't zero the page */
> + }
> + *map += 1;
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.prev, struct page, lru);
> + while (page != head) {
> + map = kmap_atomic(page, KM_USER0) + offset;
> + *map = COUNT_CONTINUED;
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.prev, struct page, lru);
> + }
> + return true; /* incremented */
> +
> + } else { /* decrementing */
> + /*
> + * Think of how you subtract 1 from 1000
> + */
> + BUG_ON(count != COUNT_CONTINUED);
> + while (*map == COUNT_CONTINUED) {
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.next, struct page, lru);
> + BUG_ON(page == head);
> + map = kmap_atomic(page, KM_USER0) + offset;
> + }
> + BUG_ON(*map == 0);
> + *map -= 1;
> + if (*map == 0)
> + count = 0;
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.prev, struct page, lru);
> + while (page != head) {
> + map = kmap_atomic(page, KM_USER0) + offset;
> + *map = SWAP_CONT_MAX | count;
> + count = COUNT_CONTINUED;
> + kunmap_atomic(map, KM_USER0);
> + page = list_entry(page->lru.prev, struct page, lru);
> + }
> + return count == COUNT_CONTINUED;
> + }
> +}
> +
> +/*
> + * free_swap_count_continuations - swapoff free all the continuation pages
> + * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
> + */
> +static void free_swap_count_continuations(struct swap_info_struct *si)
> +{
> + pgoff_t offset;
> +
> + for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
> + struct page *head;
> + head = vmalloc_to_page(si->swap_map + offset);
> + if (page_private(head)) {
> + struct list_head *this, *next;
> + list_for_each_safe(this, next, &head->lru) {
> + struct page *page;
> + page = list_entry(this, struct page, lru);
> + list_del(this);
> + __free_page(page);
> + }
> + }
> + }
> +}
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>


\
 
 \ /
  Last update: 2009-10-15 05:35    [W:0.205 / U:2.716 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site