Messages in this thread Patch in this message | | | Date | Fri, 10 Sep 2004 10:32:10 +0200 | From | Pavel Machek <> | Subject | swsusp: speedup |
| |
Hi!
This removes O(n^2) part from swsusp, making it usable with fragmented memory etc. Okay, I have to do a lot of by-hand-separation. I *hope* I got it right. Pavel
--- clean-mm/include/linux/page-flags.h 2004-09-07 21:12:33.000000000 +0200 +++ linux-mm/include/linux/page-flags.h 2004-09-07 21:15:21.000000000 +0200 @@ -75,7 +75,7 @@ #define PG_swapcache 16 /* Swap page: swp_entry_t in private */ #define PG_mappedtodisk 17 /* Has blocks allocated on-disk */ #define PG_reclaim 18 /* To be reclaimed asap */ - +#define PG_nosave_free 19 /* Page is free and should not be written */ /* * Global page accounting. One instance per CPU. Only unsigned longs are @@ -278,6 +278,10 @@ #define ClearPageNosave(page) clear_bit(PG_nosave, &(page)->flags) #define TestClearPageNosave(page) test_and_clear_bit(PG_nosave, &(page)->flags) +#define PageNosaveFree(page) test_bit(PG_nosave_free, &(page)->flags) +#define SetPageNosaveFree(page) set_bit(PG_nosave_free, &(page)->flags) +#define ClearPageNosaveFree(page) clear_bit(PG_nosave_free, &(page)->flags) + #define PageMappedToDisk(page) test_bit(PG_mappedtodisk, &(page)->flags) #define SetPageMappedToDisk(page) set_bit(PG_mappedtodisk, &(page)->flags) #define ClearPageMappedToDisk(page) clear_bit(PG_mappedtodisk, &(page)->flags) --- clean-mm/include/linux/suspend.h 2004-09-07 21:12:33.000000000 +0200 +++ linux-mm/include/linux/suspend.h 2004-09-07 21:15:22.000000000 +0200 @@ -31,6 +31,7 @@ /* mm/page_alloc.c */ extern void drain_local_pages(void); +extern void mark_free_pages(struct zone *zone); /* kernel/power/swsusp.c */ extern int software_suspend(void); --- clean-mm/kernel/power/disk.c 2004-09-07 21:12:33.000000000 +0200 +++ linux-mm/kernel/power/disk.c 2004-09-08 22:47:41.000000000 +0200 @@ -91,10 +91,20 @@ static void free_some_memory(void) { - printk("Freeing memory: "); - while (shrink_all_memory(10000)) - printk("."); - printk("|\n"); + unsigned int i = 0; + unsigned int tmp; + unsigned long pages = 0; + char *p = "-\\|/"; + + printk("Freeing memory... "); + while ((tmp = shrink_all_memory(10000))) { + pages += tmp; + printk("\b%c", p[i]); + i++; + if (i > 3) + i = 0; + } + printk("\bdone (%li pages freed)\n", pages); } @@ -167,6 +177,7 @@ { int error; + system_state = SYSTEM_SNAPSHOT; if ((error = prepare())) return error; --- clean-mm/kernel/power/swsusp.c 2004-09-07 21:12:33.000000000 +0200 +++ linux-mm/kernel/power/swsusp.c 2004-09-09 08:56:20.000000000 +0200 @@ -74,11 +74,9 @@ /* References to section boundaries */ extern char __nosave_begin, __nosave_end; -extern int is_head_of_free_region(struct page *); - /* Variables to be preserved over suspend */ -int pagedir_order_check; -int nr_copy_pages_check; +static int pagedir_order_check; +static int nr_copy_pages_check; extern char resume_file[]; static dev_t resume_device; @@ -296,15 +292,19 @@ { int error = 0; int i; - - printk( "Writing data to swap (%d pages): ", nr_copy_pages ); + unsigned int mod = nr_copy_pages / 100; + + if (!mod) + mod = 1; + + printk( "Writing data to swap (%d pages)... ", nr_copy_pages ); for (i = 0; i < nr_copy_pages && !error; i++) { - if (!(i%100)) - printk( "." ); + if (!(i%mod)) + printk( "\b\b\b\b%3d%%", i / mod ); error = write_page((pagedir_nosave+i)->address, &((pagedir_nosave+i)->swap_address)); } - printk(" %d Pages done.\n",i); + printk("\b\b\b\bdone\n"); return error; } @@ -423,12 +424,12 @@ static int save_highmem_zone(struct zone *zone) { unsigned long zone_pfn; + mark_free_pages(zone); for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { struct page *page; struct highmem_page *save; void *kaddr; unsigned long pfn = zone_pfn + zone->zone_start_pfn; - int chunk_size; if (!(pfn%1000)) printk("."); @@ -445,11 +446,8 @@ printk("highmem reserved page?!\n"); continue; } - if ((chunk_size = is_head_of_free_region(page))) { - pfn += chunk_size - 1; - zone_pfn += chunk_size - 1; + if (PageNosaveFree(page)) continue; - } save = kmalloc(sizeof(struct highmem_page), GFP_ATOMIC); if (!save) return -ENOMEM; @@ -528,14 +526,11 @@ static int saveable(struct zone * zone, unsigned long * zone_pfn) { unsigned long pfn = *zone_pfn + zone->zone_start_pfn; - unsigned long chunk_size; struct page * page; if (!pfn_valid(pfn)) return 0; - if (!(pfn%1000)) - printk("."); page = pfn_to_page(pfn); BUG_ON(PageReserved(page) && PageNosave(page)); if (PageNosave(page)) @@ -544,10 +539,8 @@ pr_debug("[nosave pfn 0x%lx]", pfn); return 0; } - if ((chunk_size = is_head_of_free_region(page))) { - *zone_pfn += chunk_size - 1; + if (PageNosaveFree(page)) return 0; - } return 1; } @@ -561,6 +554,7 @@ for_each_zone(zone) { if (!is_highmem(zone)) { + mark_free_pages(zone); for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) nr_copy_pages += saveable(zone, &zone_pfn); } @@ -575,50 +569,20 @@ struct pbe * pbe = pagedir_nosave; for_each_zone(zone) { - if (!is_highmem(zone)) - for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { - if (saveable(zone, &zone_pfn)) { - struct page * page; - page = pfn_to_page(zone_pfn + zone->zone_start_pfn); - pbe->orig_address = (long) page_address(page); - /* copy_page is no usable for copying task structs. */ - memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE); - pbe++; - } - } - } -} - - -static void free_suspend_pagedir_zone(struct zone *zone, unsigned long pagedir) -{ - unsigned long zone_pfn, pagedir_end, pagedir_pfn, pagedir_end_pfn; - pagedir_end = pagedir + (PAGE_SIZE << pagedir_order); - pagedir_pfn = __pa(pagedir) >> PAGE_SHIFT; - pagedir_end_pfn = __pa(pagedir_end) >> PAGE_SHIFT; - for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { - struct page *page; - unsigned long pfn = zone_pfn + zone->zone_start_pfn; - if (!pfn_valid(pfn)) - continue; - page = pfn_to_page(pfn); - if (!TestClearPageNosave(page)) - continue; - else if (pfn >= pagedir_pfn && pfn < pagedir_end_pfn) + if (is_highmem(zone)) continue; - __free_page(page); - } -} - -void swsusp_free(void) -{ - unsigned long p = (unsigned long)pagedir_save; - struct zone *zone; - for_each_zone(zone) { - if (!is_highmem(zone)) - free_suspend_pagedir_zone(zone, p); + mark_free_pages(zone); + for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { + if (saveable(zone, &zone_pfn)) { + struct page * page; + page = pfn_to_page(zone_pfn + zone->zone_start_pfn); + pbe->orig_address = (long) page_address(page); + /* copy_page is not usable for copying task structs. */ + memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE); + pbe++; + } + } } - free_pages(p, pagedir_order); } @@ -684,6 +648,24 @@ return 0; } +/** + * free_image_pages - Free pages allocated for snapshot + */ + +static void free_image_pages(void) +{ + struct pbe * p; + int i; + + p = pagedir_save; + for (i = 0, p = pagedir_save; i < nr_copy_pages; i++, p++) { + if (p->address) { + ClearPageNosave(virt_to_page(p->address)); + free_page(p->address); + p->address = 0; + } + } +} /** * alloc_image_pages - Allocate pages for the snapshot. @@ -697,18 +679,19 @@ for (i = 0, p = pagedir_save; i < nr_copy_pages; i++, p++) { p->address = get_zeroed_page(GFP_ATOMIC | __GFP_COLD); - if(!p->address) - goto Error; + if (!p->address) + return -ENOMEM; SetPageNosave(virt_to_page(p->address)); } return 0; - Error: - do { - if (p->address) - free_page(p->address); - p->address = 0; - } while (p-- > pagedir_save); - return -ENOMEM; +} + +void swsusp_free(void) +{ + BUG_ON(PageNosave(virt_to_page(pagedir_save))); + BUG_ON(PageNosaveFree(virt_to_page(pagedir_save))); + free_image_pages(); + free_pages((unsigned long) pagedir_save, pagedir_order); } @@ -950,9 +934,9 @@ printk("Relocating pagedir "); - if(!does_collide_order(old_pagedir, (unsigned long)old_pagedir, pagedir_order)) { + if (!does_collide_order(old_pagedir, (unsigned long)old_pagedir, pagedir_order)) { printk("not necessary\n"); - return 0; + return check_pagedir(); } while ((m = (void *) __get_free_pages(GFP_ATOMIC, pagedir_order)) != NULL) { @@ -994,24 +978,14 @@ static atomic_t io_done = ATOMIC_INIT(0); -static void start_io(void) -{ - atomic_set(&io_done,1); -} - static int end_io(struct bio * bio, unsigned int num, int err) { - atomic_set(&io_done,0); + if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) + panic("I/O error reading memory image"); + atomic_set(&io_done, 0); return 0; } -static void wait_io(void) -{ - while(atomic_read(&io_done)) - io_schedule(); -} - - static struct block_device * resume_bdev; /** @@ -1046,9 +1020,12 @@ if (rw == WRITE) bio_set_pages_dirty(bio); - start_io(); + + atomic_set(&io_done, 1); submit_bio(rw | (1 << BIO_RW_SYNC), bio); - wait_io(); + while (atomic_read(&io_done)) + yield(); + Done: bio_put(bio); return error; @@ -1104,6 +1081,7 @@ return -EPERM; } nr_copy_pages = swsusp_info.image_pages; + pagedir_order = get_bitmask_order(SUSPEND_PD_PAGES(nr_copy_pages)); return error; } @@ -1130,17 +1108,6 @@ return error; } - -int __init verify(void) -{ - int error; - - if (!(error = check_sig())) - error = check_header(); - return error; -} - - /** * swsusp_read_data - Read image pages from swap. * @@ -1153,14 +1120,18 @@ struct pbe * p; int error; int i; - + int mod = nr_copy_pages / 100; + + if (!mod) + mod = 1; + if ((error = swsusp_pagedir_relocate())) return error; - printk( "Reading image data (%d pages): ", nr_copy_pages ); + printk( "Reading image data (%d pages): ", nr_copy_pages ); for(i = 0, p = pagedir_nosave; i < nr_copy_pages && !error; i++, p++) { - if (!(i%100)) - printk( "." ); + if (!(i%mod)) + printk( "\b\b\b\b%3d%%", i / mod ); error = bio_read_page(swp_offset(p->swap_address), (void *)p->address); } @@ -1177,9 +1148,7 @@ int i, n = swsusp_info.pagedir_pages; int error = 0; - pagedir_order = get_bitmask_order(n); - - addr =__get_free_pages(GFP_ATOMIC, pagedir_order); + addr = __get_free_pages(GFP_ATOMIC, pagedir_order); if (!addr) return -ENOMEM; pagedir_nosave = (struct pbe *)addr; @@ -1202,13 +1171,14 @@ { int error = 0; - if ((error = verify())) + if ((error = check_sig())) + return error; + if ((error = check_header())) return error; if ((error = read_pagedir())) return error; - if ((error = data_read())) { - free_pages((unsigned long)pagedir_nosave,pagedir_order); - } + if ((error = data_read())) + free_pages((unsigned long)pagedir_nosave, pagedir_order); return error; } --- clean-mm/mm/page_alloc.c 2004-09-07 21:12:33.000000000 +0200 +++ linux-mm/mm/page_alloc.c 2004-09-07 21:15:23.000000000 +0200 @@ -437,26 +437,30 @@ #endif /* CONFIG_PM || CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_PM -int is_head_of_free_region(struct page *page) + +void mark_free_pages(struct zone *zone) { - struct zone *zone = page_zone(page); - unsigned long flags; + unsigned long zone_pfn, flags; int order; struct list_head *curr; - /* - * Should not matter as we need quiescent system for - * suspend anyway, but... - */ + if (!zone->spanned_pages) + return; + spin_lock_irqsave(&zone->lock, flags); + for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) + ClearPageNosaveFree(pfn_to_page(zone_pfn + zone->zone_start_pfn)); + for (order = MAX_ORDER - 1; order >= 0; --order) - list_for_each(curr, &zone->free_area[order].free_list) - if (page == list_entry(curr, struct page, lru)) { - spin_unlock_irqrestore(&zone->lock, flags); - return 1 << order; - } + list_for_each(curr, &zone->free_area[order].free_list) { + unsigned long start_pfn, i; + + start_pfn = page_to_pfn(list_entry(curr, struct page, lru)); + + for (i=0; i < (1<<order); i++) + SetPageNosaveFree(pfn_to_page(start_pfn+i)); + } spin_unlock_irqrestore(&zone->lock, flags); - return 0; } /* - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
| |