2.6.24

 mm

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 1a8c595..b2838c2 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -27,6 +27,7 @@
 #include <linux/pagevec.h>
 #include <linux/blkdev.h>
 #include <linux/slab.h>
+#include <linux/oom.h>
 #include <linux/notifier.h>
 #include <linux/topology.h>
 #include <linux/sysctl.h>
@@ -41,24 +42,37 @@
 #include <linux/pfn.h>
 #include <linux/backing-dev.h>
 #include <linux/fault-inject.h>
+#include <linux/page-isolation.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
 #include "internal.h"
 
 /*
- * MCD - HACK: Find somewhere to initialize this EARLY, or make this
- * initializer cleaner
+ * Array of node states.
  */
-nodemask_t node_online_map __read_mostly = { { [0] = 1UL } };
-EXPORT_SYMBOL(node_online_map);
-nodemask_t node_possible_map __read_mostly = NODE_MASK_ALL;
-EXPORT_SYMBOL(node_possible_map);
+nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
+   [N_POSSIBLE] = NODE_MASK_ALL,
+   [N_ONLINE] = { { [0] = 1UL } },
+#ifndef CONFIG_NUMA
+   [N_NORMAL_MEMORY] = { { [0] = 1UL } },
+#ifdef CONFIG_HIGHMEM
+   [N_HIGH_MEMORY] = { { [0] = 1UL } },
+#endif
+   [N_CPU] = { { [0] = 1UL } },
+#endif   /* NUMA */
+};
+EXPORT_SYMBOL(node_states);
+
 unsigned long totalram_pages __read_mostly;
 unsigned long totalreserve_pages __read_mostly;
 long nr_swap_pages;
 int percpu_pagelist_fraction;
 
+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
+int pageblock_order __read_mostly;
+#endif
+
 static void __free_pages_ok(struct page *page, unsigned int order);
 
 /*
@@ -109,7 +123,7 @@ static unsigned long __meminitdata dma_reserve;
 
 #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
   /*
-   * MAX_ACTIVE_REGIONS determines the maxmimum number of distinct
+   * MAX_ACTIVE_REGIONS determines the maximum number of distinct
    * ranges of memory (RAM) that may be registered with add_active_range().
    * Ranges passed to add_active_range() will be merged if possible
    * so the number of times add_active_range() can be called is
@@ -137,7 +151,7 @@ static unsigned long __meminitdata dma_reserve;
   static unsigned long __meminitdata node_boundary_end_pfn[MAX_NUMNODES];
 #endif /* CONFIG_MEMORY_HOTPLUG_RESERVE */
   unsigned long __initdata required_kernelcore;
-  unsigned long __initdata required_movablecore;
+  static unsigned long __initdata required_movablecore;
   unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
 
   /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
@@ -150,6 +164,14 @@ int nr_node_ids __read_mostly = MAX_NUMNODES;
 EXPORT_SYMBOL(nr_node_ids);
 #endif
 
+int page_group_by_mobility_disabled __read_mostly;
+
+static void set_pageblock_migratetype(struct page *page, int migratetype)
+{
+   set_pageblock_flags_group(page, (unsigned long)migratetype,
+               PB_migrate, PB_migrate_end);
+}
+
 #ifdef CONFIG_DEBUG_VM
 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 {
@@ -283,7 +305,6 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
 {
    int i;
 
-   VM_BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM);
    /*
     * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
     * and __GFP_HIGHMEM from hard or soft interrupt context.
@@ -293,16 +314,6 @@ static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
       clear_highpage(page + i);
 }
 
-/*
- * function for dealing with page's order in buddy system.
- * zone->lock is already acquired when we use these.
- * So, we don't need atomic page->flags operations here.
- */
-static inline unsigned long page_order(struct page *page)
-{
-   return page_private(page);
-}
-
 static inline void set_page_order(struct page *page, int order)
 {
    set_page_private(page, order);
@@ -404,6 +415,7 @@ static inline void __free_one_page(struct page *page,
 {
    unsigned long page_idx;
    int order_size = 1 << order;
+   int migratetype = get_pageblock_migratetype(page);
 
    if (unlikely(PageCompound(page)))
       destroy_compound_page(page, order);
@@ -416,7 +428,6 @@ static inline void __free_one_page(struct page *page,
    __mod_zone_page_state(zone, NR_FREE_PAGES, order_size);
    while (order < MAX_ORDER-1) {
       unsigned long combined_idx;
-      struct free_area *area;
       struct page *buddy;
 
       buddy = __page_find_buddy(page, page_idx, order);
@@ -424,8 +435,7 @@ static inline void __free_one_page(struct page *page,
          break;      /* Move the buddy up one level. */
 
       list_del(&buddy->lru);
-      area = zone->free_area + order;
-      area->nr_free--;
+      zone->free_area[order].nr_free--;
       rmv_page_order(buddy);
       combined_idx = __find_combined_index(page_idx, order);
       page = page + (combined_idx - page_idx);
@@ -433,7 +443,8 @@ static inline void __free_one_page(struct page *page,
       order++;
    }
    set_page_order(page, order);
-   list_add(&page->lru, &zone->free_area[order].free_list);
+   list_add(&page->lru,
+      &zone->free_area[order].free_list[migratetype]);
    zone->free_area[order].nr_free++;
 }
 
@@ -478,7 +489,7 @@ static void free_pages_bulk(struct zone *zone, int count,
                struct list_head *list, int order)
 {
    spin_lock(&zone->lock);
-   zone->all_unreclaimable = 0;
+   zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
    zone->pages_scanned = 0;
    while (count--) {
       struct page *page;
@@ -495,7 +506,7 @@ static void free_pages_bulk(struct zone *zone, int count,
 static void free_one_page(struct zone *zone, struct page *page, int order)
 {
    spin_lock(&zone->lock);
-   zone->all_unreclaimable = 0;
+   zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE);
    zone->pages_scanned = 0;
    __free_one_page(page, zone, order);
    spin_unlock(&zone->lock);
@@ -567,7 +578,8 @@ void fastcall __init __free_pages_bootmem(struct page *page, unsigned int order)
  * -- wli
  */
 static inline void expand(struct zone *zone, struct page *page,
-    int low, int high, struct free_area *area)
+   int low, int high, struct free_area *area,
+   int migratetype)
 {
    unsigned long size = 1 << high;
 
@@ -576,7 +588,7 @@ static inline void expand(struct zone *zone, struct page *page,
       high--;
       size >>= 1;
       VM_BUG_ON(bad_range(zone, &page[size]));
-      list_add(&page[size].lru, &area->free_list);
+      list_add(&page[size].lru, &area->free_list[migratetype]);
       area->nr_free++;
       set_page_order(&page[size], high);
    }
@@ -628,49 +640,226 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
    return 0;
 }
 
-/* 
- * Do the hard work of removing an element from the buddy allocator.
- * Call me with the zone->lock already held.
+/*
+ * Go through the free lists for the given migratetype and remove
+ * the smallest available page from the freelists
  */
-static struct page *__rmqueue(struct zone *zone, unsigned int order)
+static struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
+                  int migratetype)
 {
-   struct free_area * area;
    unsigned int current_order;
+   struct free_area * area;
    struct page *page;
 
+   /* Find a page of the appropriate size in the preferred list */
    for (current_order = order; current_order < MAX_ORDER; ++current_order) {
-      area = zone->free_area + current_order;
-      if (list_empty(&area->free_list))
+      area = &(zone->free_area[current_order]);
+      if (list_empty(&area->free_list[migratetype]))
          continue;
 
-      page = list_entry(area->free_list.next, struct page, lru);
+      page = list_entry(area->free_list[migratetype].next,
+                     struct page, lru);
       list_del(&page->lru);
       rmv_page_order(page);
       area->nr_free--;
       __mod_zone_page_state(zone, NR_FREE_PAGES, - (1UL << order));
-      expand(zone, page, order, current_order, area);
+      expand(zone, page, order, current_order, area, migratetype);
       return page;
    }
 
    return NULL;
 }
 
+
+/*
+ * This array describes the order lists are fallen back to when
+ * the free lists for the desirable migrate type are depleted
+ */
+static int fallbacks[MIGRATE_TYPES][MIGRATE_TYPES-1] = {
+   [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_RESERVE },
+   [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_RESERVE },
+   [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
+   [MIGRATE_RESERVE]     = { MIGRATE_RESERVE,     MIGRATE_RESERVE,   MIGRATE_RESERVE }, /* Never used */
+};
+
+/*
+ * Move the free pages in a range to the free lists of the requested type.
+ * Note that start_page and end_pages are not aligned on a pageblock
+ * boundary. If alignment is required, use move_freepages_block()
+ */
+int move_freepages(struct zone *zone,
+         struct page *start_page, struct page *end_page,
+         int migratetype)
+{
+   struct page *page;
+   unsigned long order;
+   int pages_moved = 0;
+
+#ifndef CONFIG_HOLES_IN_ZONE
+   /*
+    * page_zone is not safe to call in this context when
+    * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
+    * anyway as we check zone boundaries in move_freepages_block().
+    * Remove at a later date when no bug reports exist related to
+    * grouping pages by mobility
+    */
+   BUG_ON(page_zone(start_page) != page_zone(end_page));
+#endif
+
+   for (page = start_page; page <= end_page;) {
+      if (!pfn_valid_within(page_to_pfn(page))) {
+         page++;
+         continue;
+      }
+
+      if (!PageBuddy(page)) {
+         page++;
+         continue;
+      }
+
+      order = page_order(page);
+      list_del(&page->lru);
+      list_add(&page->lru,
+         &zone->free_area[order].free_list[migratetype]);
+      page += 1 << order;
+      pages_moved += 1 << order;
+   }
+
+   return pages_moved;
+}
+
+int move_freepages_block(struct zone *zone, struct page *page, int migratetype)
+{
+   unsigned long start_pfn, end_pfn;
+   struct page *start_page, *end_page;
+
+   start_pfn = page_to_pfn(page);
+   start_pfn = start_pfn & ~(pageblock_nr_pages-1);
+   start_page = pfn_to_page(start_pfn);
+   end_page = start_page + pageblock_nr_pages - 1;
+   end_pfn = start_pfn + pageblock_nr_pages - 1;
+
+   /* Do not cross zone boundaries */
+   if (start_pfn < zone->zone_start_pfn)
+      start_page = page;
+   if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
+      return 0;
+
+   return move_freepages(zone, start_page, end_page, migratetype);
+}
+
+/* Remove an element from the buddy allocator from the fallback list */
+static struct page *__rmqueue_fallback(struct zone *zone, int order,
+                  int start_migratetype)
+{
+   struct free_area * area;
+   int current_order;
+   struct page *page;
+   int migratetype, i;
+
+   /* Find the largest possible block of pages in the other list */
+   for (current_order = MAX_ORDER-1; current_order >= order;
+                  --current_order) {
+      for (i = 0; i < MIGRATE_TYPES - 1; i++) {
+         migratetype = fallbacks[start_migratetype][i];
+
+         /* MIGRATE_RESERVE handled later if necessary */
+         if (migratetype == MIGRATE_RESERVE)
+            continue;
+
+         area = &(zone->free_area[current_order]);
+         if (list_empty(&area->free_list[migratetype]))
+            continue;
+
+         page = list_entry(area->free_list[migratetype].next,
+               struct page, lru);
+         area->nr_free--;
+
+         /*
+          * If breaking a large block of pages, move all free
+          * pages to the preferred allocation list. If falling
+          * back for a reclaimable kernel allocation, be more
+          * agressive about taking ownership of free pages
+          */
+         if (unlikely(current_order >= (pageblock_order >> 1)) ||
+               start_migratetype == MIGRATE_RECLAIMABLE) {
+            unsigned long pages;
+            pages = move_freepages_block(zone, page,
+                        start_migratetype);
+
+            /* Claim the whole block if over half of it is free */
+            if (pages >= (1 << (pageblock_order-1)))
+               set_pageblock_migratetype(page,
+                        start_migratetype);
+
+            migratetype = start_migratetype;
+         }
+
+         /* Remove the page from the freelists */
+         list_del(&page->lru);
+         rmv_page_order(page);
+         __mod_zone_page_state(zone, NR_FREE_PAGES,
+                     -(1UL << order));
+
+         if (current_order == pageblock_order)
+            set_pageblock_migratetype(page,
+                     start_migratetype);
+
+         expand(zone, page, order, current_order, area, migratetype);
+         return page;
+      }
+   }
+
+   /* Use MIGRATE_RESERVE rather than fail an allocation */
+   return __rmqueue_smallest(zone, order, MIGRATE_RESERVE);
+}
+
+/*
+ * Do the hard work of removing an element from the buddy allocator.
+ * Call me with the zone->lock already held.
+ */
+static struct page *__rmqueue(struct zone *zone, unsigned int order,
+                  int migratetype)
+{
+   struct page *page;
+
+   page = __rmqueue_smallest(zone, order, migratetype);
+
+   if (unlikely(!page))
+      page = __rmqueue_fallback(zone, order, migratetype);
+
+   return page;
+}
+
 /* 
  * Obtain a specified number of elements from the buddy allocator, all under
  * a single hold of the lock, for efficiency.  Add them to the supplied list.
  * Returns the number of new pages which were placed at *list.
  */
 static int rmqueue_bulk(struct zone *zone, unsigned int order, 
-         unsigned long count, struct list_head *list)
+         unsigned long count, struct list_head *list,
+         int migratetype)
 {
    int i;
    
    spin_lock(&zone->lock);
    for (i = 0; i < count; ++i) {
-      struct page *page = __rmqueue(zone, order);
+      struct page *page = __rmqueue(zone, order, migratetype);
       if (unlikely(page == NULL))
          break;
-      list_add_tail(&page->lru, list);
+
+      /*
+       * Split buddy pages returned by expand() are received here
+       * in physical page order. The page is added to the callers and
+       * list and the list head then moves forward. From the callers
+       * perspective, the linked list is ordered by page number in
+       * some conditions. This is useful for IO devices that can
+       * merge IO requests if the physical pages are ordered
+       * properly.
+       */
+      list_add(&page->lru, list);
+      set_page_private(page, migratetype);
+      list = &page->lru;
    }
    spin_unlock(&zone->lock);
    return i;
@@ -732,7 +921,7 @@ void mark_free_pages(struct zone *zone)
 {
    unsigned long pfn, max_zone_pfn;
    unsigned long flags;
-   int order;
+   int order, t;
    struct list_head *curr;
 
    if (!zone->spanned_pages)
@@ -749,17 +938,18 @@ void mark_free_pages(struct zone *zone)
             swsusp_unset_page_free(page);
       }
 
-   for (order = MAX_ORDER - 1; order >= 0; --order)
-      list_for_each(curr, &zone->free_area[order].free_list) {
+   for_each_migratetype_order(order, t) {
+      list_for_each(curr, &zone->free_area[order].free_list[t]) {
          unsigned long i;
 
          pfn = page_to_pfn(list_entry(curr, struct page, lru));
          for (i = 0; i < (1UL << order); i++)
             swsusp_set_page_free(pfn_to_page(pfn + i));
       }
-
+   }
    spin_unlock_irqrestore(&zone->lock, flags);
 }
+#endif /* CONFIG_PM */
 
 /*
  * Spill all of this CPU's per-cpu pages back into the buddy allocator.
@@ -772,7 +962,25 @@ void drain_local_pages(void)
    __drain_pages(smp_processor_id());
    local_irq_restore(flags);   
 }
-#endif /* CONFIG_HIBERNATION */
+
+void smp_drain_local_pages(void *arg)
+{
+   drain_local_pages();
+}
+
+/*
+ * Spill all the per-cpu pages from all CPUs back into the buddy allocator
+ */
+void drain_all_local_pages(void)
+{
+   unsigned long flags;
+
+   local_irq_save(flags);
+   __drain_pages(smp_processor_id());
+   local_irq_restore(flags);
+
+   smp_call_function(smp_drain_local_pages, NULL, 0, 1);
+}
 
 /*
  * Free a 0-order page
@@ -797,6 +1005,7 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
    local_irq_save(flags);
    __count_vm_event(PGFREE);
    list_add(&page->lru, &pcp->list);
+   set_page_private(page, get_pageblock_migratetype(page));
    pcp->count++;
    if (pcp->count >= pcp->high) {
       free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
@@ -846,6 +1055,7 @@ static struct page *buffered_rmqueue(struct zonelist *zonelist,
    struct page *page;
    int cold = !!(gfp_flags & __GFP_COLD);
    int cpu;
+   int migratetype = allocflags_to_migratetype(gfp_flags);
 
 again:
    cpu  = get_cpu();
@@ -856,16 +1066,28 @@ again:
       local_irq_save(flags);
       if (!pcp->count) {
          pcp->count = rmqueue_bulk(zone, 0,
-                  pcp->batch, &pcp->list);
+               pcp->batch, &pcp->list, migratetype);
          if (unlikely(!pcp->count))
             goto failed;
       }
-      page = list_entry(pcp->list.next, struct page, lru);
+
+      /* Find a page of the appropriate migrate type */
+      list_for_each_entry(page, &pcp->list, lru)
+         if (page_private(page) == migratetype)
+            break;
+
+      /* Allocate more to the pcp list if necessary */
+      if (unlikely(&page->lru == &pcp->list)) {
+         pcp->count += rmqueue_bulk(zone, 0,
+               pcp->batch, &pcp->list, migratetype);
+         page = list_entry(pcp->list.next, struct page, lru);
+      }
+
       list_del(&page->lru);
       pcp->count--;
    } else {
       spin_lock_irqsave(&zone->lock, flags);
-      page = __rmqueue(zone, order);
+      page = __rmqueue(zone, order, migratetype);
       spin_unlock(&zone->lock);
       if (!page)
          goto failed;
@@ -1028,11 +1250,11 @@ int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
  * skip over zones that are not allowed by the cpuset, or that have
  * been recently (in last second) found to be nearly full.  See further
  * comments in mmzone.h.  Reduces cache footprint of zonelist scans
- * that have to skip over alot of full or unallowed zones.
+ * that have to skip over a lot of full or unallowed zones.
  *
  * If the zonelist cache is present in the passed in zonelist, then
  * returns a pointer to the allowed node mask (either the current
- * tasks mems_allowed, or node_online_map.)
+ * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
  *
  * If the zonelist cache is not available for this zonelist, does
  * nothing and returns NULL.
@@ -1061,7 +1283,7 @@ static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
 
    allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
                &cpuset_current_mems_allowed :
-               &node_online_map;
+               &node_states[N_HIGH_MEMORY];
    return allowednodes;
 }
 
@@ -1183,9 +1405,6 @@ zonelist_scan:
          !zlc_zone_worth_trying(zonelist, z, allowednodes))
             continue;
       zone = *z;
-      if (unlikely(NUMA_BUILD && (gfp_mask & __GFP_THISNODE) &&
-         zone->zone_pgdat != zonelist->zones[0]->zone_pgdat))
-            break;
       if ((alloc_flags & ALLOC_CPUSET) &&
          !cpuset_zone_allowed_softwall(zone, gfp_mask))
             goto try_next_zone;
@@ -1254,7 +1473,10 @@ restart:
    z = zonelist->zones;  /* the list of zones suitable for gfp_mask */
 
    if (unlikely(*z == NULL)) {
-      /* Should this ever happen?? */
+      /*
+       * Happens if we have an empty zonelist as a result of
+       * GFP_THISNODE being used on a memoryless node
+       */
       return NULL;
    }
 
@@ -1346,12 +1568,20 @@ nofail_alloc:
 
    cond_resched();
 
+   if (order != 0)
+      drain_all_local_pages();
+
    if (likely(did_some_progress)) {
       page = get_page_from_freelist(gfp_mask, order,
                   zonelist, alloc_flags);
       if (page)
          goto got_pg;
    } else if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
+      if (!try_set_zone_oom(zonelist)) {
+         schedule_timeout_uninterruptible(1);
+         goto restart;
+      }
+
       /*
        * Go through the zonelist yet one more time, keep
        * very high watermark here, this is only to catch
@@ -1360,14 +1590,19 @@ nofail_alloc:
        */
       page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, order,
             zonelist, ALLOC_WMARK_HIGH|ALLOC_CPUSET);
-      if (page)
+      if (page) {
+         clear_zonelist_oom(zonelist);
          goto got_pg;
+      }
 
       /* The OOM killer will not help higher order allocs so fail */
-      if (order > PAGE_ALLOC_COSTLY_ORDER)
+      if (order > PAGE_ALLOC_COSTLY_ORDER) {
+         clear_zonelist_oom(zonelist);
          goto nopage;
+      }
 
       out_of_memory(zonelist, gfp_mask, order);
+      clear_zonelist_oom(zonelist);
       goto restart;
    }
 
@@ -1616,7 +1851,7 @@ void show_free_areas(void)
          K(zone_page_state(zone, NR_INACTIVE)),
          K(zone->present_pages),
          zone->pages_scanned,
-         (zone->all_unreclaimable ? "yes" : "no")
+         (zone_is_all_unreclaimable(zone) ? "yes" : "no")
          );
       printk("lowmem_reserve[]:");
       for (i = 0; i < MAX_NR_ZONES; i++)
@@ -1794,7 +2029,7 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
       return node;
    }
 
-   for_each_online_node(n) {
+   for_each_node_state(n, N_HIGH_MEMORY) {
       cpumask_t tmp;
 
       /* Don't want a node to appear more than once */
@@ -1850,6 +2085,22 @@ static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
 }
 
 /*
+ * Build gfp_thisnode zonelists
+ */
+static void build_thisnode_zonelists(pg_data_t *pgdat)
+{
+   enum zone_type i;
+   int j;
+   struct zonelist *zonelist;
+
+   for (i = 0; i < MAX_NR_ZONES; i++) {
+      zonelist = pgdat->node_zonelists + MAX_NR_ZONES + i;
+      j = build_zonelists_node(pgdat, zonelist, 0, i);
+      zonelist->zones[j] = NULL;
+   }
+}
+
+/*
  * Build zonelists ordered by zone and nodes within zones.
  * This results in conserving DMA zone[s] until all Normal memory is
  * exhausted, but results in overflowing to remote node while memory
@@ -1915,7 +2166,8 @@ static int default_zonelist_order(void)
       * If there is a node whose DMA/DMA32 memory is very big area on
      * local memory, NODE_ORDER may be suitable.
          */
-   average_size = total_size / (num_online_nodes() + 1);
+   average_size = total_size /
+            (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
    for_each_online_node(nid) {
       low_kmem_size = 0;
       total_size = 0;
@@ -1953,7 +2205,7 @@ static void build_zonelists(pg_data_t *pgdat)
    int order = current_zonelist_order;
 
    /* initialize zonelists */
-   for (i = 0; i < MAX_NR_ZONES; i++) {
+   for (i = 0; i < MAX_ZONELISTS; i++) {
       zonelist = pgdat->node_zonelists + i;
       zonelist->zones[0] = NULL;
    }
@@ -1998,6 +2250,8 @@ static void build_zonelists(pg_data_t *pgdat)
       /* calculate node order -- i.e., DMA last! */
       build_zonelists_in_zone_order(pgdat, j);
    }
+
+   build_thisnode_zonelists(pgdat);
 }
 
 /* Construct the zonelist performance cache - see further mmzone.h */
@@ -2078,8 +2332,10 @@ static int __build_all_zonelists(void *dummy)
    int nid;
 
    for_each_online_node(nid) {
-      build_zonelists(NODE_DATA(nid));
-      build_zonelist_cache(NODE_DATA(nid));
+      pg_data_t *pgdat = NODE_DATA(nid);
+
+      build_zonelists(pgdat);
+      build_zonelist_cache(pgdat);
    }
    return 0;
 }
@@ -2092,15 +2348,29 @@ void build_all_zonelists(void)
       __build_all_zonelists(NULL);
       cpuset_init_current_mems_allowed();
    } else {
-      /* we have to stop all cpus to guaranntee there is no user
+      /* we have to stop all cpus to guarantee there is no user
          of zonelist */
       stop_machine_run(__build_all_zonelists, NULL, NR_CPUS);
       /* cpuset refresh routine should be here */
    }
    vm_total_pages = nr_free_pagecache_pages();
-   printk("Built %i zonelists in %s order.  Total pages: %ld\n",
+   /*
+    * Disable grouping by mobility if the number of pages in the
+    * system is too low to allow the mechanism to work. It would be
+    * more accurate, but expensive to check per-zone. This check is
+    * made on memory-hotadd so a system can start with mobility
+    * disabled and enable it later
+    */
+   if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
+      page_group_by_mobility_disabled = 1;
+   else
+      page_group_by_mobility_disabled = 0;
+
+   printk("Built %i zonelists in %s order, mobility grouping %s.  "
+      "Total pages: %ld\n",
          num_online_nodes(),
          zonelist_order_name[current_zonelist_order],
+         page_group_by_mobility_disabled ? "off" : "on",
          vm_total_pages);
 #ifdef CONFIG_NUMA
    printk("Policy zone: %s\n", zone_names[policy_zone]);
@@ -2176,6 +2446,61 @@ static inline unsigned long wait_table_bits(unsigned long size)
 #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
 
 /*
+ * Mark a number of pageblocks as MIGRATE_RESERVE. The number
+ * of blocks reserved is based on zone->pages_min. The memory within the
+ * reserve will tend to store contiguous free pages. Setting min_free_kbytes
+ * higher will lead to a bigger reserve which will get freed as contiguous
+ * blocks as reclaim kicks in
+ */
+static void setup_zone_migrate_reserve(struct zone *zone)
+{
+   unsigned long start_pfn, pfn, end_pfn;
+   struct page *page;
+   unsigned long reserve, block_migratetype;
+
+   /* Get the start pfn, end pfn and the number of blocks to reserve */
+   start_pfn = zone->zone_start_pfn;
+   end_pfn = start_pfn + zone->spanned_pages;
+   reserve = roundup(zone->pages_min, pageblock_nr_pages) >>
+                     pageblock_order;
+
+   for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
+      if (!pfn_valid(pfn))
+         continue;
+      page = pfn_to_page(pfn);
+
+      /* Blocks with reserved pages will never free, skip them. */
+      if (PageReserved(page))
+         continue;
+
+      block_migratetype = get_pageblock_migratetype(page);
+
+      /* If this block is reserved, account for it */
+      if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) {
+         reserve--;
+         continue;
+      }
+
+      /* Suitable for reserving if this block is movable */
+      if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) {
+         set_pageblock_migratetype(page, MIGRATE_RESERVE);
+         move_freepages_block(zone, page, MIGRATE_RESERVE);
+         reserve--;
+         continue;
+      }
+
+      /*
+       * If the reserve is met and this is a previous reserved block,
+       * take it back
+       */
+      if (block_migratetype == MIGRATE_RESERVE) {
+         set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+         move_freepages_block(zone, page, MIGRATE_MOVABLE);
+      }
+   }
+}
+
+/*
  * Initially all pages are reserved - free ones are freed
  * up by free_all_bootmem() once the early boot process is
  * done. Non-atomic initialization, single-pass.
@@ -2204,6 +2529,19 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
       init_page_count(page);
       reset_page_mapcount(page);
       SetPageReserved(page);
+
+      /*
+       * Mark the block movable so that blocks are reserved for
+       * movable at startup. This will force kernel allocations
+       * to reserve their blocks rather than leaking throughout
+       * the address space during boot when many long-lived
+       * kernel allocations are made. Later some blocks near
+       * the start are marked MIGRATE_RESERVE by
+       * setup_zone_migrate_reserve()
+       */
+      if ((pfn & (pageblock_nr_pages-1)))
+         set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+
       INIT_LIST_HEAD(&page->lru);
 #ifdef WANT_PAGE_VIRTUAL
       /* The shift won't overflow because ZONE_NORMAL is below 4G. */
@@ -2216,9 +2554,9 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
 static void __meminit zone_init_free_lists(struct pglist_data *pgdat,
             struct zone *zone, unsigned long size)
 {
-   int order;
-   for (order = 0; order < MAX_ORDER ; order++) {
-      INIT_LIST_HEAD(&zone->free_area[order].free_list);
+   int order, t;
+   for_each_migratetype_order(order, t) {
+      INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
       zone->free_area[order].nr_free = 0;
    }
 }
@@ -2228,7 +2566,7 @@ static void __meminit zone_init_free_lists(struct pglist_data *pgdat,
    memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
 #endif
 
-static int __devinit zone_batchsize(struct zone *zone)
+static int zone_batchsize(struct zone *zone)
 {
    int batch;
 
@@ -2324,6 +2662,9 @@ static struct per_cpu_pageset boot_pageset[NR_CPUS];
 static int __cpuinit process_zones(int cpu)
 {
    struct zone *zone, *dzone;
+   int node = cpu_to_node(cpu);
+
+   node_set_state(node, N_CPU);   /* this node has a cpu */
 
    for_each_zone(zone) {
 
@@ -2331,7 +2672,7 @@ static int __cpuinit process_zones(int cpu)
          continue;
 
       zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset),
-                GFP_KERNEL, cpu_to_node(cpu));
+                GFP_KERNEL, node);
       if (!zone_pcp(zone, cpu))
          goto bad;
 
@@ -2444,7 +2785,7 @@ int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
        * To use this new node's memory, further consideration will be
        * necessary.
        */
-      zone->wait_table = (wait_queue_head_t *)vmalloc(alloc_size);
+      zone->wait_table = vmalloc(alloc_size);
    }
    if (!zone->wait_table)
       return -ENOMEM;
@@ -2513,7 +2854,7 @@ static int __meminit first_active_region_index_in_nid(int nid)
 
 /*
  * Basic iterator support. Return the next active range of PFNs for a node
- * Note: nid == MAX_NUMNODES returns next region regardles of node
+ * Note: nid == MAX_NUMNODES returns next region regardless of node
  */
 static int __meminit next_active_region_index_in_nid(int index, int nid)
 {
@@ -2680,10 +3021,8 @@ void __meminit get_pfn_range_for_nid(unsigned int nid,
       *end_pfn = max(*end_pfn, early_node_map[i].end_pfn);
    }
 
-   if (*start_pfn == -1UL) {
-      printk(KERN_WARNING "Node %u active with no memory\n", nid);
+   if (*start_pfn == -1UL)
       *start_pfn = 0;
-   }
 
    /* Push the node boundaries out if requested */
    account_node_boundary(nid, start_pfn, end_pfn);
@@ -2901,6 +3240,81 @@ static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
                      realtotalpages);
 }
 
+#ifndef CONFIG_SPARSEMEM
+/*
+ * Calculate the size of the zone->blockflags rounded to an unsigned long
+ * Start by making sure zonesize is a multiple of pageblock_order by rounding
+ * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
+ * round what is now in bits to nearest long in bits, then return it in
+ * bytes.
+ */
+static unsigned long __init usemap_size(unsigned long zonesize)
+{
+   unsigned long usemapsize;
+
+   usemapsize = roundup(zonesize, pageblock_nr_pages);
+   usemapsize = usemapsize >> pageblock_order;
+   usemapsize *= NR_PAGEBLOCK_BITS;
+   usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
+
+   return usemapsize / 8;
+}
+
+static void __init setup_usemap(struct pglist_data *pgdat,
+            struct zone *zone, unsigned long zonesize)
+{
+   unsigned long usemapsize = usemap_size(zonesize);
+   zone->pageblock_flags = NULL;
+   if (usemapsize) {
+      zone->pageblock_flags = alloc_bootmem_node(pgdat, usemapsize);
+      memset(zone->pageblock_flags, 0, usemapsize);
+   }
+}
+#else
+static void inline setup_usemap(struct pglist_data *pgdat,
+            struct zone *zone, unsigned long zonesize) {}
+#endif /* CONFIG_SPARSEMEM */
+
+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
+
+/* Return a sensible default order for the pageblock size. */
+static inline int pageblock_default_order(void)
+{
+   if (HPAGE_SHIFT > PAGE_SHIFT)
+      return HUGETLB_PAGE_ORDER;
+
+   return MAX_ORDER-1;
+}
+
+/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
+static inline void __init set_pageblock_order(unsigned int order)
+{
+   /* Check that pageblock_nr_pages has not already been setup */
+   if (pageblock_order)
+      return;
+
+   /*
+    * Assume the largest contiguous order of interest is a huge page.
+    * This value may be variable depending on boot parameters on IA64
+    */
+   pageblock_order = order;
+}
+#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
+
+/*
+ * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
+ * and pageblock_default_order() are unused as pageblock_order is set
+ * at compile-time. See include/linux/pageblock-flags.h for the values of
+ * pageblock_order based on the kernel config
+ */
+static inline int pageblock_default_order(unsigned int order)
+{
+   return MAX_ORDER-1;
+}
+#define set_pageblock_order(x)   do {} while (0)
+
+#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
+
 /*
  * Set up the zone data structures:
  *   - mark all pages reserved
@@ -2977,10 +3391,12 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat,
       zone->nr_scan_active = 0;
       zone->nr_scan_inactive = 0;
       zap_zone_vm_stats(zone);
-      atomic_set(&zone->reclaim_in_progress, 0);
+      zone->flags = 0;
       if (!size)
          continue;
 
+      set_pageblock_order(pageblock_default_order());
+      setup_usemap(pgdat, zone, size);
       ret = init_currently_empty_zone(zone, zone_start_pfn,
                   size, MEMMAP_EARLY);
       BUG_ON(ret);
@@ -3022,7 +3438,7 @@ static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
       mem_map = NODE_DATA(0)->node_mem_map;
 #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
       if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
-         mem_map -= pgdat->node_start_pfn;
+         mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
 #endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
    }
 #endif
@@ -3234,16 +3650,24 @@ unsigned long __init find_max_pfn_with_active_regions(void)
    return max_pfn;
 }
 
-unsigned long __init early_calculate_totalpages(void)
+/*
+ * early_calculate_totalpages()
+ * Sum pages in active regions for movable zone.
+ * Populate N_HIGH_MEMORY for calculating usable_nodes.
+ */
+static unsigned long __init early_calculate_totalpages(void)
 {
    int i;
    unsigned long totalpages = 0;
 
-   for (i = 0; i < nr_nodemap_entries; i++)
-      totalpages += early_node_map[i].end_pfn -
+   for (i = 0; i < nr_nodemap_entries; i++) {
+      unsigned long pages = early_node_map[i].end_pfn -
                   early_node_map[i].start_pfn;
-
-   return totalpages;
+      totalpages += pages;
+      if (pages)
+         node_set_state(early_node_map[i].nid, N_HIGH_MEMORY);
+   }
+     return totalpages;
 }
 
 /*
@@ -3257,7 +3681,8 @@ void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
    int i, nid;
    unsigned long usable_startpfn;
    unsigned long kernelcore_node, kernelcore_remaining;
-   int usable_nodes = num_online_nodes();
+   unsigned long totalpages = early_calculate_totalpages();
+   int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
 
    /*
     * If movablecore was specified, calculate what size of
@@ -3268,7 +3693,6 @@ void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
     * what movablecore would have allowed.
     */
    if (required_movablecore) {
-      unsigned long totalpages = early_calculate_totalpages();
       unsigned long corepages;
 
       /*
@@ -3293,7 +3717,7 @@ void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn)
 restart:
    /* Spread kernelcore memory as evenly as possible throughout nodes */
    kernelcore_node = required_kernelcore / usable_nodes;
-   for_each_online_node(nid) {
+   for_each_node_state(nid, N_HIGH_MEMORY) {
       /*
        * Recalculate kernelcore_node if the division per node
        * now exceeds what is necessary to satisfy the requested
@@ -3385,6 +3809,20 @@ restart:
          roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
 }
 
+/* Any regular memory on that node ? */
+static void check_for_regular_memory(pg_data_t *pgdat)
+{
+#ifdef CONFIG_HIGHMEM
+   enum zone_type zone_type;
+
+   for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
+      struct zone *zone = &pgdat->node_zones[zone_type];
+      if (zone->present_pages)
+         node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
+   }
+#endif
+}
+
 /**
  * free_area_init_nodes - Initialise all pg_data_t and zone data
  * @max_zone_pfn: an array of max PFNs for each zone
@@ -3459,6 +3897,11 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
       pg_data_t *pgdat = NODE_DATA(nid);
       free_area_init_node(nid, pgdat, NULL,
             find_min_pfn_for_node(nid), NULL);
+
+      /* Any memory on that node */
+      if (pgdat->node_present_pages)
+         node_set_state(nid, N_HIGH_MEMORY);
+      check_for_regular_memory(pgdat);
    }
 }
 
@@ -3673,6 +4116,7 @@ void setup_per_zone_pages_min(void)
 
       zone->pages_low   = zone->pages_min + (tmp >> 2);
       zone->pages_high  = zone->pages_min + (tmp >> 1);
+      setup_zone_migrate_reserve(zone);
       spin_unlock_irqrestore(&zone->lru_lock, flags);
    }
 
@@ -3934,4 +4378,169 @@ EXPORT_SYMBOL(pfn_to_page);
 EXPORT_SYMBOL(page_to_pfn);
 #endif /* CONFIG_OUT_OF_LINE_PFN_TO_PAGE */
 
+/* Return a pointer to the bitmap storing bits affecting a block of pages */
+static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
+                     unsigned long pfn)
+{
+#ifdef CONFIG_SPARSEMEM
+   return __pfn_to_section(pfn)->pageblock_flags;
+#else
+   return zone->pageblock_flags;
+#endif /* CONFIG_SPARSEMEM */
+}
+
+static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
+{
+#ifdef CONFIG_SPARSEMEM
+   pfn &= (PAGES_PER_SECTION-1);
+   return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
+#else
+   pfn = pfn - zone->zone_start_pfn;
+   return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
+#endif /* CONFIG_SPARSEMEM */
+}
+
+/**
+ * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
+ * @page: The page within the block of interest
+ * @start_bitidx: The first bit of interest to retrieve
+ * @end_bitidx: The last bit of interest
+ * returns pageblock_bits flags
+ */
+unsigned long get_pageblock_flags_group(struct page *page,
+               int start_bitidx, int end_bitidx)
+{
+   struct zone *zone;
+   unsigned long *bitmap;
+   unsigned long pfn, bitidx;
+   unsigned long flags = 0;
+   unsigned long value = 1;
+
+   zone = page_zone(page);
+   pfn = page_to_pfn(page);
+   bitmap = get_pageblock_bitmap(zone, pfn);
+   bitidx = pfn_to_bitidx(zone, pfn);
+
+   for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
+      if (test_bit(bitidx + start_bitidx, bitmap))
+         flags |= value;
+
+   return flags;
+}
+
+/**
+ * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
+ * @page: The page within the block of interest
+ * @start_bitidx: The first bit of interest
+ * @end_bitidx: The last bit of interest
+ * @flags: The flags to set
+ */
+void set_pageblock_flags_group(struct page *page, unsigned long flags,
+               int start_bitidx, int end_bitidx)
+{
+   struct zone *zone;
+   unsigned long *bitmap;
+   unsigned long pfn, bitidx;
+   unsigned long value = 1;
+
+   zone = page_zone(page);
+   pfn = page_to_pfn(page);
+   bitmap = get_pageblock_bitmap(zone, pfn);
+   bitidx = pfn_to_bitidx(zone, pfn);
+
+   for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
+      if (flags & value)
+         __set_bit(bitidx + start_bitidx, bitmap);
+      else
+         __clear_bit(bitidx + start_bitidx, bitmap);
+}
+
+/*
+ * This is designed as sub function...plz see page_isolation.c also.
+ * set/clear page block's type to be ISOLATE.
+ * page allocater never alloc memory from ISOLATE block.
+ */
+
+int set_migratetype_isolate(struct page *page)
+{
+   struct zone *zone;
+   unsigned long flags;
+   int ret = -EBUSY;
+
+   zone = page_zone(page);
+   spin_lock_irqsave(&zone->lock, flags);
+   /*
+    * In future, more migrate types will be able to be isolation target.
+    */
+   if (get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
+      goto out;
+   set_pageblock_migratetype(page, MIGRATE_ISOLATE);
+   move_freepages_block(zone, page, MIGRATE_ISOLATE);
+   ret = 0;
+out:
+   spin_unlock_irqrestore(&zone->lock, flags);
+   if (!ret)
+      drain_all_local_pages();
+   return ret;
+}
+
+void unset_migratetype_isolate(struct page *page)
+{
+   struct zone *zone;
+   unsigned long flags;
+   zone = page_zone(page);
+   spin_lock_irqsave(&zone->lock, flags);
+   if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
+      goto out;
+   set_pageblock_migratetype(page, MIGRATE_MOVABLE);
+   move_freepages_block(zone, page, MIGRATE_MOVABLE);
+out:
+   spin_unlock_irqrestore(&zone->lock, flags);
+}
 
+#ifdef CONFIG_MEMORY_HOTREMOVE
+/*
+ * All pages in the range must be isolated before calling this.
+ */
+void
+__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
+{
+   struct page *page;
+   struct zone *zone;
+   int order, i;
+   unsigned long pfn;
+   unsigned long flags;
+   /* find the first valid pfn */
+   for (pfn = start_pfn; pfn < end_pfn; pfn++)
+      if (pfn_valid(pfn))
+         break;
+   if (pfn == end_pfn)
+      return;
+   zone = page_zone(pfn_to_page(pfn));
+   spin_lock_irqsave(&zone->lock, flags);
+   pfn = start_pfn;
+   while (pfn < end_pfn) {
+      if (!pfn_valid(pfn)) {
+         pfn++;
+         continue;
+      }
+      page = pfn_to_page(pfn);
+      BUG_ON(page_count(page));
+      BUG_ON(!PageBuddy(page));
+      order = page_order(page);
+#ifdef CONFIG_DEBUG_VM
+      printk(KERN_INFO "remove from free list %lx %d %lx\n",
+             pfn, 1 << order, end_pfn);
+#endif
+      list_del(&page->lru);
+      rmv_page_order(page);
+      zone->free_area[order].nr_free--;
+      __mod_zone_page_state(zone, NR_FREE_PAGES,
+                  - (1UL << order));
+      for (i = 0; i < (1 << order); i++)
+         SetPageReserved((page+i));
+      pfn += (1 << order);
+   }
+   spin_unlock_irqrestore(&zone->lock, flags);
+}
+#endif