lkml.org 
[lkml]   [2012]   [Feb]   [20]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[PATCH 3/10] mm/memcg: add zone pointer into lruvec
    The lruvec is looking rather useful: if we just add a zone pointer
    into the lruvec, then we can pass the lruvec pointer around and save
    some superfluous arguments and recomputations in various places.

    Just occasionally we do want mem_cgroup_from_lruvec() to get back from
    lruvec to memcg; but then we can remove all uses of vmscan.c's private
    mem_cgroup_zone *mz, passing the lruvec pointer instead.

    And while we're there, get_scan_count() can call vmscan_swappiness()
    once, instead of twice in a row.

    Signed-off-by: Hugh Dickins <hughd@google.com>
    ---
    include/linux/memcontrol.h | 23 ++-
    include/linux/mmzone.h | 1
    mm/memcontrol.c | 47 ++++----
    mm/page_alloc.c | 1
    mm/vmscan.c | 203 +++++++++++++++--------------------
    5 files changed, 128 insertions(+), 147 deletions(-)

    --- mmotm.orig/include/linux/memcontrol.h 2012-02-18 11:57:20.391524062 -0800
    +++ mmotm/include/linux/memcontrol.h 2012-02-18 11:57:28.371524252 -0800
    @@ -63,6 +63,7 @@ extern int mem_cgroup_cache_charge(struc
    gfp_t gfp_mask);

    struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
    +extern struct mem_cgroup *mem_cgroup_from_lruvec(struct lruvec *lruvec);
    struct lruvec *mem_cgroup_lru_add_list(struct zone *, struct page *,
    enum lru_list);
    void mem_cgroup_lru_del_list(struct page *, enum lru_list);
    @@ -113,13 +114,11 @@ void mem_cgroup_iter_break(struct mem_cg
    /*
    * For memory reclaim.
    */
    -int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg,
    - struct zone *zone);
    -int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg,
    - struct zone *zone);
    +int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec);
    +int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec);
    int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
    -unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
    - int nid, int zid, unsigned int lrumask);
    +unsigned long mem_cgroup_zone_nr_lru_pages(struct lruvec *lruvec,
    + unsigned int lrumask);
    struct zone_reclaim_stat*
    mem_cgroup_get_reclaim_stat_from_page(struct page *page);
    extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
    @@ -249,6 +248,11 @@ static inline struct lruvec *mem_cgroup_
    return &zone->lruvec;
    }

    +static inline struct mem_cgroup *mem_cgroup_from_lruvec(struct lruvec *lruvec)
    +{
    + return NULL;
    +}
    +
    static inline struct lruvec *mem_cgroup_lru_add_list(struct zone *zone,
    struct page *page,
    enum lru_list lru)
    @@ -331,20 +335,19 @@ static inline bool mem_cgroup_disabled(v
    }

    static inline int
    -mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
    +mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
    {
    return 1;
    }

    static inline int
    -mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
    +mem_cgroup_inactive_file_is_low(struct lruvec *lruvec)
    {
    return 1;
    }

    static inline unsigned long
    -mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
    - unsigned int lru_mask)
    +mem_cgroup_zone_nr_lru_pages(struct lruvec *lruvec, unsigned int lru_mask)
    {
    return 0;
    }
    --- mmotm.orig/include/linux/mmzone.h 2012-02-18 11:57:20.391524062 -0800
    +++ mmotm/include/linux/mmzone.h 2012-02-18 11:57:28.371524252 -0800
    @@ -173,6 +173,7 @@ struct zone_reclaim_stat {
    };

    struct lruvec {
    + struct zone *zone;
    struct list_head lists[NR_LRU_LISTS];
    struct zone_reclaim_stat reclaim_stat;
    };
    --- mmotm.orig/mm/memcontrol.c 2012-02-18 11:57:20.391524062 -0800
    +++ mmotm/mm/memcontrol.c 2012-02-18 11:57:28.371524252 -0800
    @@ -703,14 +703,13 @@ static void mem_cgroup_charge_statistics
    }

    unsigned long
    -mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
    - unsigned int lru_mask)
    +mem_cgroup_zone_nr_lru_pages(struct lruvec *lruvec, unsigned int lru_mask)
    {
    struct mem_cgroup_per_zone *mz;
    enum lru_list lru;
    unsigned long ret = 0;

    - mz = mem_cgroup_zoneinfo(memcg, nid, zid);
    + mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);

    for_each_lru(lru) {
    if (BIT(lru) & lru_mask)
    @@ -723,12 +722,14 @@ static unsigned long
    mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
    int nid, unsigned int lru_mask)
    {
    + struct mem_cgroup_per_zone *mz;
    u64 total = 0;
    int zid;

    - for (zid = 0; zid < MAX_NR_ZONES; zid++)
    - total += mem_cgroup_zone_nr_lru_pages(memcg,
    - nid, zid, lru_mask);
    + for (zid = 0; zid < MAX_NR_ZONES; zid++) {
    + mz = mem_cgroup_zoneinfo(memcg, nid, zid);
    + total += mem_cgroup_zone_nr_lru_pages(&mz->lruvec, lru_mask);
    + }

    return total;
    }
    @@ -1003,13 +1004,24 @@ struct lruvec *mem_cgroup_zone_lruvec(st
    {
    struct mem_cgroup_per_zone *mz;

    - if (mem_cgroup_disabled())
    + if (!memcg || mem_cgroup_disabled())
    return &zone->lruvec;

    mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
    return &mz->lruvec;
    }

    +struct mem_cgroup *mem_cgroup_from_lruvec(struct lruvec *lruvec)
    +{
    + struct mem_cgroup_per_zone *mz;
    +
    + if (mem_cgroup_disabled())
    + return NULL;
    +
    + mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
    + return mz->memcg;
    +}
    +
    /*
    * Following LRU functions are allowed to be used without PCG_LOCK.
    * Operations are called by routine of global LRU independently from memcg.
    @@ -1161,19 +1173,15 @@ int task_in_mem_cgroup(struct task_struc
    return ret;
    }

    -int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
    +int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
    {
    unsigned long inactive_ratio;
    - int nid = zone_to_nid(zone);
    - int zid = zone_idx(zone);
    unsigned long inactive;
    unsigned long active;
    unsigned long gb;

    - inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
    - BIT(LRU_INACTIVE_ANON));
    - active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
    - BIT(LRU_ACTIVE_ANON));
    + inactive = mem_cgroup_zone_nr_lru_pages(lruvec, BIT(LRU_INACTIVE_ANON));
    + active = mem_cgroup_zone_nr_lru_pages(lruvec, BIT(LRU_ACTIVE_ANON));

    gb = (inactive + active) >> (30 - PAGE_SHIFT);
    if (gb)
    @@ -1184,17 +1192,13 @@ int mem_cgroup_inactive_anon_is_low(stru
    return inactive * inactive_ratio < active;
    }

    -int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
    +int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec)
    {
    unsigned long active;
    unsigned long inactive;
    - int zid = zone_idx(zone);
    - int nid = zone_to_nid(zone);

    - inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
    - BIT(LRU_INACTIVE_FILE));
    - active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
    - BIT(LRU_ACTIVE_FILE));
    + inactive = mem_cgroup_zone_nr_lru_pages(lruvec, BIT(LRU_INACTIVE_FILE));
    + active = mem_cgroup_zone_nr_lru_pages(lruvec, BIT(LRU_ACTIVE_FILE));

    return (active > inactive);
    }
    @@ -4755,6 +4759,7 @@ static int alloc_mem_cgroup_per_zone_inf

    for (zone = 0; zone < MAX_NR_ZONES; zone++) {
    mz = &pn->zoneinfo[zone];
    + mz->lruvec.zone = &NODE_DATA(node)->node_zones[zone];
    for_each_lru(lru)
    INIT_LIST_HEAD(&mz->lruvec.lists[lru]);
    mz->usage_in_excess = 0;
    --- mmotm.orig/mm/page_alloc.c 2012-02-18 11:57:20.395524062 -0800
    +++ mmotm/mm/page_alloc.c 2012-02-18 11:57:28.375524252 -0800
    @@ -4365,6 +4365,7 @@ static void __paginginit free_area_init_
    zone->zone_pgdat = pgdat;

    zone_pcp_init(zone);
    + zone->lruvec.zone = zone;
    for_each_lru(lru)
    INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
    zone->lruvec.reclaim_stat.recent_rotated[0] = 0;
    --- mmotm.orig/mm/vmscan.c 2012-02-18 11:57:20.395524062 -0800
    +++ mmotm/mm/vmscan.c 2012-02-18 11:57:28.375524252 -0800
    @@ -115,11 +115,6 @@ struct scan_control {
    nodemask_t *nodemask;
    };

    -struct mem_cgroup_zone {
    - struct mem_cgroup *mem_cgroup;
    - struct zone *zone;
    -};
    -
    #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))

    #ifdef ARCH_HAS_PREFETCH
    @@ -171,21 +166,12 @@ static bool global_reclaim(struct scan_c
    }
    #endif

    -static struct zone_reclaim_stat *get_reclaim_stat(struct mem_cgroup_zone *mz)
    -{
    - return &mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup)->reclaim_stat;
    -}
    -
    -static unsigned long zone_nr_lru_pages(struct mem_cgroup_zone *mz,
    - enum lru_list lru)
    +static unsigned long zone_nr_lru_pages(struct lruvec *lruvec, enum lru_list lru)
    {
    if (!mem_cgroup_disabled())
    - return mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
    - zone_to_nid(mz->zone),
    - zone_idx(mz->zone),
    - BIT(lru));
    + return mem_cgroup_zone_nr_lru_pages(lruvec, BIT(lru));

    - return zone_page_state(mz->zone, NR_LRU_BASE + lru);
    + return zone_page_state(lruvec->zone, NR_LRU_BASE + lru);
    }


    @@ -688,13 +674,13 @@ enum page_references {
    };

    static enum page_references page_check_references(struct page *page,
    - struct mem_cgroup_zone *mz,
    + struct mem_cgroup *memcg,
    struct scan_control *sc)
    {
    int referenced_ptes, referenced_page;
    unsigned long vm_flags;

    - referenced_ptes = page_referenced(page, 1, mz->mem_cgroup, &vm_flags);
    + referenced_ptes = page_referenced(page, 1, memcg, &vm_flags);
    referenced_page = TestClearPageReferenced(page);

    /* Lumpy reclaim - ignore references */
    @@ -750,12 +736,13 @@ static enum page_references page_check_r
    * shrink_page_list() returns the number of reclaimed pages
    */
    static unsigned long shrink_page_list(struct list_head *page_list,
    - struct mem_cgroup_zone *mz,
    + struct lruvec *lruvec,
    struct scan_control *sc,
    int priority,
    unsigned long *ret_nr_dirty,
    unsigned long *ret_nr_writeback)
    {
    + struct mem_cgroup *memcg = mem_cgroup_from_lruvec(lruvec);
    LIST_HEAD(ret_pages);
    LIST_HEAD(free_pages);
    int pgactivate = 0;
    @@ -781,7 +768,7 @@ static unsigned long shrink_page_list(st
    goto keep;

    VM_BUG_ON(PageActive(page));
    - VM_BUG_ON(page_zone(page) != mz->zone);
    + VM_BUG_ON(page_zone(page) != lruvec->zone);

    sc->nr_scanned++;

    @@ -815,7 +802,7 @@ static unsigned long shrink_page_list(st
    }
    }

    - references = page_check_references(page, mz, sc);
    + references = page_check_references(page, memcg, sc);
    switch (references) {
    case PAGEREF_ACTIVATE:
    goto activate_locked;
    @@ -1007,7 +994,7 @@ keep_lumpy:
    * will encounter the same problem
    */
    if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
    - zone_set_flag(mz->zone, ZONE_CONGESTED);
    + zone_set_flag(lruvec->zone, ZONE_CONGESTED);

    free_hot_cold_page_list(&free_pages, 1);

    @@ -1122,7 +1109,7 @@ int __isolate_lru_page(struct page *page
    * Appropriate locks must be held before calling this function.
    *
    * @nr_to_scan: The number of pages to look through on the list.
    - * @mz: The mem_cgroup_zone to pull pages from.
    + * @lruvec: The mem_cgroup/zone lruvec to pull pages from.
    * @dst: The temp list to put pages on to.
    * @nr_scanned: The number of pages that were scanned.
    * @sc: The scan_control struct for this reclaim session
    @@ -1133,11 +1120,10 @@ int __isolate_lru_page(struct page *page
    * returns how many pages were moved onto *@dst.
    */
    static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
    - struct mem_cgroup_zone *mz, struct list_head *dst,
    + struct lruvec *lruvec, struct list_head *dst,
    unsigned long *nr_scanned, struct scan_control *sc,
    isolate_mode_t mode, int active, int file)
    {
    - struct lruvec *lruvec;
    struct list_head *src;
    unsigned long nr_taken = 0;
    unsigned long nr_lumpy_taken = 0;
    @@ -1146,7 +1132,6 @@ static unsigned long isolate_lru_pages(u
    unsigned long scan;
    int lru = LRU_BASE;

    - lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
    if (active)
    lru += LRU_ACTIVE;
    if (file)
    @@ -1344,11 +1329,10 @@ static int too_many_isolated(struct zone
    }

    static noinline_for_stack void
    -putback_inactive_pages(struct mem_cgroup_zone *mz,
    - struct list_head *page_list)
    +putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
    {
    - struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
    - struct zone *zone = mz->zone;
    + struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
    + struct zone *zone = lruvec->zone;
    LIST_HEAD(pages_to_free);

    /*
    @@ -1395,12 +1379,9 @@ putback_inactive_pages(struct mem_cgroup
    }

    static noinline_for_stack void
    -update_isolated_counts(struct mem_cgroup_zone *mz,
    - struct list_head *page_list,
    - unsigned long *nr_anon,
    - unsigned long *nr_file)
    +update_isolated_counts(struct zone *zone, struct list_head *page_list,
    + unsigned long *nr_anon, unsigned long *nr_file)
    {
    - struct zone *zone = mz->zone;
    unsigned int count[NR_LRU_LISTS] = { 0, };
    unsigned long nr_active = 0;
    struct page *page;
    @@ -1486,9 +1467,11 @@ static inline bool should_reclaim_stall(
    * of reclaimed pages
    */
    static noinline_for_stack unsigned long
    -shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
    +shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
    struct scan_control *sc, int priority, int file)
    {
    + struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
    + struct zone *zone = lruvec->zone;
    LIST_HEAD(page_list);
    unsigned long nr_scanned;
    unsigned long nr_reclaimed = 0;
    @@ -1498,8 +1481,6 @@ shrink_inactive_list(unsigned long nr_to
    unsigned long nr_dirty = 0;
    unsigned long nr_writeback = 0;
    isolate_mode_t isolate_mode = ISOLATE_INACTIVE;
    - struct zone *zone = mz->zone;
    - struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);

    while (unlikely(too_many_isolated(zone, file, sc))) {
    congestion_wait(BLK_RW_ASYNC, HZ/10);
    @@ -1522,31 +1503,29 @@ shrink_inactive_list(unsigned long nr_to

    spin_lock_irq(&zone->lru_lock);

    - nr_taken = isolate_lru_pages(nr_to_scan, mz, &page_list, &nr_scanned,
    - sc, isolate_mode, 0, file);
    + nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
    + &nr_scanned, sc, isolate_mode, 0, file);
    if (global_reclaim(sc)) {
    zone->pages_scanned += nr_scanned;
    if (current_is_kswapd())
    - __count_zone_vm_events(PGSCAN_KSWAPD, zone,
    - nr_scanned);
    + __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scanned);
    else
    - __count_zone_vm_events(PGSCAN_DIRECT, zone,
    - nr_scanned);
    + __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scanned);
    }
    spin_unlock_irq(&zone->lru_lock);

    if (nr_taken == 0)
    return 0;

    - update_isolated_counts(mz, &page_list, &nr_anon, &nr_file);
    + update_isolated_counts(zone, &page_list, &nr_anon, &nr_file);

    - nr_reclaimed = shrink_page_list(&page_list, mz, sc, priority,
    + nr_reclaimed = shrink_page_list(&page_list, lruvec, sc, priority,
    &nr_dirty, &nr_writeback);

    /* Check if we should syncronously wait for writeback */
    if (should_reclaim_stall(nr_taken, nr_reclaimed, priority, sc)) {
    set_reclaim_mode(priority, sc, true);
    - nr_reclaimed += shrink_page_list(&page_list, mz, sc,
    + nr_reclaimed += shrink_page_list(&page_list, lruvec, sc,
    priority, &nr_dirty, &nr_writeback);
    }

    @@ -1559,7 +1538,7 @@ shrink_inactive_list(unsigned long nr_to
    __count_vm_events(KSWAPD_STEAL, nr_reclaimed);
    __count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);

    - putback_inactive_pages(mz, &page_list);
    + putback_inactive_pages(lruvec, &page_list);

    __mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
    __mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
    @@ -1659,10 +1638,13 @@ static void move_active_pages_to_lru(str
    }

    static void shrink_active_list(unsigned long nr_to_scan,
    - struct mem_cgroup_zone *mz,
    + struct lruvec *lruvec,
    struct scan_control *sc,
    int priority, int file)
    {
    + struct mem_cgroup *memcg = mem_cgroup_from_lruvec(lruvec);
    + struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
    + struct zone *zone = lruvec->zone;
    unsigned long nr_taken;
    unsigned long nr_scanned;
    unsigned long vm_flags;
    @@ -1670,10 +1652,8 @@ static void shrink_active_list(unsigned
    LIST_HEAD(l_active);
    LIST_HEAD(l_inactive);
    struct page *page;
    - struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
    unsigned long nr_rotated = 0;
    isolate_mode_t isolate_mode = ISOLATE_ACTIVE;
    - struct zone *zone = mz->zone;

    lru_add_drain();

    @@ -1684,8 +1664,8 @@ static void shrink_active_list(unsigned

    spin_lock_irq(&zone->lru_lock);

    - nr_taken = isolate_lru_pages(nr_to_scan, mz, &l_hold, &nr_scanned, sc,
    - isolate_mode, 1, file);
    + nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
    + &nr_scanned, sc, isolate_mode, 1, file);
    if (global_reclaim(sc))
    zone->pages_scanned += nr_scanned;

    @@ -1717,7 +1697,7 @@ static void shrink_active_list(unsigned
    }
    }

    - if (page_referenced(page, 0, mz->mem_cgroup, &vm_flags)) {
    + if (page_referenced(page, 0, memcg, &vm_flags)) {
    nr_rotated += hpage_nr_pages(page);
    /*
    * Identify referenced, file-backed active pages and
    @@ -1776,13 +1756,12 @@ static int inactive_anon_is_low_global(s

    /**
    * inactive_anon_is_low - check if anonymous pages need to be deactivated
    - * @zone: zone to check
    - * @sc: scan control of this context
    + * @lruvec: The mem_cgroup/zone lruvec to check
    *
    * Returns true if the zone does not have enough inactive anon pages,
    * meaning some active anon pages need to be deactivated.
    */
    -static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
    +static int inactive_anon_is_low(struct lruvec *lruvec)
    {
    /*
    * If we don't have swap space, anonymous page deactivation
    @@ -1792,13 +1771,12 @@ static int inactive_anon_is_low(struct m
    return 0;

    if (!mem_cgroup_disabled())
    - return mem_cgroup_inactive_anon_is_low(mz->mem_cgroup,
    - mz->zone);
    + return mem_cgroup_inactive_anon_is_low(lruvec);

    - return inactive_anon_is_low_global(mz->zone);
    + return inactive_anon_is_low_global(lruvec->zone);
    }
    #else
    -static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)
    +static inline int inactive_anon_is_low(struct lruvec *lruvec)
    {
    return 0;
    }
    @@ -1816,7 +1794,7 @@ static int inactive_file_is_low_global(s

    /**
    * inactive_file_is_low - check if file pages need to be deactivated
    - * @mz: memory cgroup and zone to check
    + * @lruvec: The mem_cgroup/zone lruvec to check
    *
    * When the system is doing streaming IO, memory pressure here
    * ensures that active file pages get deactivated, until more
    @@ -1828,44 +1806,44 @@ static int inactive_file_is_low_global(s
    * This uses a different ratio than the anonymous pages, because
    * the page cache uses a use-once replacement algorithm.
    */
    -static int inactive_file_is_low(struct mem_cgroup_zone *mz)
    +static int inactive_file_is_low(struct lruvec *lruvec)
    {
    if (!mem_cgroup_disabled())
    - return mem_cgroup_inactive_file_is_low(mz->mem_cgroup,
    - mz->zone);
    + return mem_cgroup_inactive_file_is_low(lruvec);

    - return inactive_file_is_low_global(mz->zone);
    + return inactive_file_is_low_global(lruvec->zone);
    }

    -static int inactive_list_is_low(struct mem_cgroup_zone *mz, int file)
    +static int inactive_list_is_low(struct lruvec *lruvec, int file)
    {
    if (file)
    - return inactive_file_is_low(mz);
    + return inactive_file_is_low(lruvec);
    else
    - return inactive_anon_is_low(mz);
    + return inactive_anon_is_low(lruvec);
    }

    static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
    - struct mem_cgroup_zone *mz,
    + struct lruvec *lruvec,
    struct scan_control *sc, int priority)
    {
    int file = is_file_lru(lru);

    if (is_active_lru(lru)) {
    - if (inactive_list_is_low(mz, file))
    - shrink_active_list(nr_to_scan, mz, sc, priority, file);
    + if (inactive_list_is_low(lruvec, file))
    + shrink_active_list(nr_to_scan, lruvec, sc, priority,
    + file);
    return 0;
    }

    - return shrink_inactive_list(nr_to_scan, mz, sc, priority, file);
    + return shrink_inactive_list(nr_to_scan, lruvec, sc, priority, file);
    }

    -static int vmscan_swappiness(struct mem_cgroup_zone *mz,
    +static int vmscan_swappiness(struct lruvec *lruvec,
    struct scan_control *sc)
    {
    if (global_reclaim(sc))
    return vm_swappiness;
    - return mem_cgroup_swappiness(mz->mem_cgroup);
    + return mem_cgroup_swappiness(mem_cgroup_from_lruvec(lruvec));
    }

    /*
    @@ -1876,13 +1854,14 @@ static int vmscan_swappiness(struct mem_
    *
    * nr[0] = anon pages to scan; nr[1] = file pages to scan
    */
    -static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
    +static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
    unsigned long *nr, int priority)
    {
    + struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
    + struct zone *zone = lruvec->zone;
    unsigned long anon, file, free;
    unsigned long anon_prio, file_prio;
    unsigned long ap, fp;
    - struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
    u64 fraction[2], denominator;
    enum lru_list lru;
    int noswap = 0;
    @@ -1898,7 +1877,7 @@ static void get_scan_count(struct mem_cg
    * latencies, so it's better to scan a minimum amount there as
    * well.
    */
    - if (current_is_kswapd() && mz->zone->all_unreclaimable)
    + if (current_is_kswapd() && zone->all_unreclaimable)
    force_scan = true;
    if (!global_reclaim(sc))
    force_scan = true;
    @@ -1912,16 +1891,16 @@ static void get_scan_count(struct mem_cg
    goto out;
    }

    - anon = zone_nr_lru_pages(mz, LRU_ACTIVE_ANON) +
    - zone_nr_lru_pages(mz, LRU_INACTIVE_ANON);
    - file = zone_nr_lru_pages(mz, LRU_ACTIVE_FILE) +
    - zone_nr_lru_pages(mz, LRU_INACTIVE_FILE);
    + anon = zone_nr_lru_pages(lruvec, LRU_ACTIVE_ANON) +
    + zone_nr_lru_pages(lruvec, LRU_INACTIVE_ANON);
    + file = zone_nr_lru_pages(lruvec, LRU_ACTIVE_FILE) +
    + zone_nr_lru_pages(lruvec, LRU_INACTIVE_FILE);

    if (global_reclaim(sc)) {
    - free = zone_page_state(mz->zone, NR_FREE_PAGES);
    + free = zone_page_state(zone, NR_FREE_PAGES);
    /* If we have very few page cache pages,
    force-scan anon pages. */
    - if (unlikely(file + free <= high_wmark_pages(mz->zone))) {
    + if (unlikely(file + free <= high_wmark_pages(zone))) {
    fraction[0] = 1;
    fraction[1] = 0;
    denominator = 1;
    @@ -1933,8 +1912,8 @@ static void get_scan_count(struct mem_cg
    * With swappiness at 100, anonymous and file have the same priority.
    * This scanning priority is essentially the inverse of IO cost.
    */
    - anon_prio = vmscan_swappiness(mz, sc);
    - file_prio = 200 - vmscan_swappiness(mz, sc);
    + anon_prio = vmscan_swappiness(lruvec, sc);
    + file_prio = 200 - anon_prio;

    /*
    * OK, so we have swap space and a fair amount of page cache
    @@ -1947,7 +1926,7 @@ static void get_scan_count(struct mem_cg
    *
    * anon in [0], file in [1]
    */
    - spin_lock_irq(&mz->zone->lru_lock);
    + spin_lock_irq(&zone->lru_lock);
    if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
    reclaim_stat->recent_scanned[0] /= 2;
    reclaim_stat->recent_rotated[0] /= 2;
    @@ -1968,7 +1947,7 @@ static void get_scan_count(struct mem_cg

    fp = (file_prio + 1) * (reclaim_stat->recent_scanned[1] + 1);
    fp /= reclaim_stat->recent_rotated[1] + 1;
    - spin_unlock_irq(&mz->zone->lru_lock);
    + spin_unlock_irq(&zone->lru_lock);

    fraction[0] = ap;
    fraction[1] = fp;
    @@ -1978,7 +1957,7 @@ out:
    int file = is_file_lru(lru);
    unsigned long scan;

    - scan = zone_nr_lru_pages(mz, lru);
    + scan = zone_nr_lru_pages(lruvec, lru);
    if (priority || noswap) {
    scan >>= priority;
    if (!scan && force_scan)
    @@ -1996,7 +1975,7 @@ out:
    * back to the allocator and call try_to_compact_zone(), we ensure that
    * there are enough free pages for it to be likely successful
    */
    -static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
    +static inline bool should_continue_reclaim(struct lruvec *lruvec,
    unsigned long nr_reclaimed,
    unsigned long nr_scanned,
    struct scan_control *sc)
    @@ -2036,15 +2015,16 @@ static inline bool should_continue_recla
    * inactive lists are large enough, continue reclaiming
    */
    pages_for_compaction = (2UL << sc->order);
    - inactive_lru_pages = zone_nr_lru_pages(mz, LRU_INACTIVE_FILE);
    + inactive_lru_pages = zone_nr_lru_pages(lruvec, LRU_INACTIVE_FILE);
    if (nr_swap_pages > 0)
    - inactive_lru_pages += zone_nr_lru_pages(mz, LRU_INACTIVE_ANON);
    + inactive_lru_pages += zone_nr_lru_pages(lruvec,
    + LRU_INACTIVE_ANON);
    if (sc->nr_reclaimed < pages_for_compaction &&
    inactive_lru_pages > pages_for_compaction)
    return true;

    /* If compaction would go ahead or the allocation would succeed, stop */
    - switch (compaction_suitable(mz->zone, sc->order)) {
    + switch (compaction_suitable(lruvec->zone, sc->order)) {
    case COMPACT_PARTIAL:
    case COMPACT_CONTINUE:
    return false;
    @@ -2056,7 +2036,7 @@ static inline bool should_continue_recla
    /*
    * This is a basic per-zone page freer. Used by both kswapd and direct reclaim.
    */
    -static void shrink_mem_cgroup_zone(int priority, struct mem_cgroup_zone *mz,
    +static void shrink_mem_cgroup_zone(int priority, struct lruvec *lruvec,
    struct scan_control *sc)
    {
    unsigned long nr[NR_LRU_LISTS];
    @@ -2069,7 +2049,7 @@ static void shrink_mem_cgroup_zone(int p
    restart:
    nr_reclaimed = 0;
    nr_scanned = sc->nr_scanned;
    - get_scan_count(mz, sc, nr, priority);
    + get_scan_count(lruvec, sc, nr, priority);

    blk_start_plug(&plug);
    while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
    @@ -2081,7 +2061,7 @@ restart:
    nr[lru] -= nr_to_scan;

    nr_reclaimed += shrink_list(lru, nr_to_scan,
    - mz, sc, priority);
    + lruvec, sc, priority);
    }
    }
    /*
    @@ -2107,11 +2087,11 @@ restart:
    * Even if we did not try to evict anon pages at all, we want to
    * rebalance the anon lru active/inactive ratio.
    */
    - if (inactive_anon_is_low(mz))
    - shrink_active_list(SWAP_CLUSTER_MAX, mz, sc, priority, 0);
    + if (inactive_anon_is_low(lruvec))
    + shrink_active_list(SWAP_CLUSTER_MAX, lruvec, sc, priority, 0);

    /* reclaim/compaction might need reclaim to continue */
    - if (should_continue_reclaim(mz, nr_reclaimed,
    + if (should_continue_reclaim(lruvec, nr_reclaimed,
    sc->nr_scanned - nr_scanned, sc))
    goto restart;

    @@ -2130,12 +2110,9 @@ static void shrink_zone(int priority, st

    memcg = mem_cgroup_iter(root, NULL, &reclaim);
    do {
    - struct mem_cgroup_zone mz = {
    - .mem_cgroup = memcg,
    - .zone = zone,
    - };
    + struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);

    - shrink_mem_cgroup_zone(priority, &mz, sc);
    + shrink_mem_cgroup_zone(priority, lruvec, sc);
    /*
    * Limit reclaim has historically picked one memcg and
    * scanned it with decreasing priority levels until
    @@ -2463,10 +2440,7 @@ unsigned long mem_cgroup_shrink_node_zon
    .order = 0,
    .target_mem_cgroup = memcg,
    };
    - struct mem_cgroup_zone mz = {
    - .mem_cgroup = memcg,
    - .zone = zone,
    - };
    + struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);

    sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
    (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
    @@ -2482,7 +2456,7 @@ unsigned long mem_cgroup_shrink_node_zon
    * will pick up pages from other mem cgroup's as well. We hack
    * the priority and make it zero.
    */
    - shrink_mem_cgroup_zone(0, &mz, &sc);
    + shrink_mem_cgroup_zone(0, lruvec, &sc);

    trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);

    @@ -2543,13 +2517,10 @@ static void age_active_anon(struct zone

    memcg = mem_cgroup_iter(NULL, NULL, NULL);
    do {
    - struct mem_cgroup_zone mz = {
    - .mem_cgroup = memcg,
    - .zone = zone,
    - };
    + struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);

    - if (inactive_anon_is_low(&mz))
    - shrink_active_list(SWAP_CLUSTER_MAX, &mz,
    + if (inactive_anon_is_low(lruvec))
    + shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
    sc, priority, 0);

    memcg = mem_cgroup_iter(NULL, memcg, NULL);

    \
     
     \ /
      Last update: 2012-02-21 00:33    [W:0.069 / U:30.636 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site