2.6.25-rc5

 mm

diff --git a/mm/slab.c b/mm/slab.c
index b03b2e4..e6c698f 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -304,11 +304,11 @@ struct kmem_list3 {
 /*
  * Need this for bootstrapping a per node allocator.
  */
-#define NUM_INIT_LISTS (2 * MAX_NUMNODES + 1)
+#define NUM_INIT_LISTS (3 * MAX_NUMNODES)
 struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
 #define   CACHE_CACHE 0
-#define   SIZE_AC 1
-#define   SIZE_L3 (1 + MAX_NUMNODES)
+#define   SIZE_AC MAX_NUMNODES
+#define   SIZE_L3 (2 * MAX_NUMNODES)
 
 static int drain_freelist(struct kmem_cache *cache,
          struct kmem_list3 *l3, int tofree);
@@ -333,7 +333,7 @@ static __always_inline int index_of(const size_t size)
       return i; \
    else \
       i++;
-#include "linux/kmalloc_sizes.h"
+#include <linux/kmalloc_sizes.h>
 #undef CACHE
       __bad_size();
    } else
@@ -730,8 +730,7 @@ static inline void init_lock_keys(void)
 #endif
 
 /*
- * 1. Guard access to the cache-chain.
- * 2. Protect sanity of cpu_online_map against cpu hotplug events
+ * Guard access to the cache-chain.
  */
 static DEFINE_MUTEX(cache_chain_mutex);
 static struct list_head cache_chain;
@@ -1331,12 +1330,11 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb,
    int err = 0;
 
    switch (action) {
-   case CPU_LOCK_ACQUIRE:
-      mutex_lock(&cache_chain_mutex);
-      break;
    case CPU_UP_PREPARE:
    case CPU_UP_PREPARE_FROZEN:
+      mutex_lock(&cache_chain_mutex);
       err = cpuup_prepare(cpu);
+      mutex_unlock(&cache_chain_mutex);
       break;
    case CPU_ONLINE:
    case CPU_ONLINE_FROZEN:
@@ -1373,9 +1371,8 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb,
 #endif
    case CPU_UP_CANCELED:
    case CPU_UP_CANCELED_FROZEN:
+      mutex_lock(&cache_chain_mutex);
       cpuup_canceled(cpu);
-      break;
-   case CPU_LOCK_RELEASE:
       mutex_unlock(&cache_chain_mutex);
       break;
    }
@@ -1410,6 +1407,22 @@ static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
 }
 
 /*
+ * For setting up all the kmem_list3s for cache whose buffer_size is same as
+ * size of kmem_list3.
+ */
+static void __init set_up_list3s(struct kmem_cache *cachep, int index)
+{
+   int node;
+
+   for_each_online_node(node) {
+      cachep->nodelists[node] = &initkmem_list3[index + node];
+      cachep->nodelists[node]->next_reap = jiffies +
+          REAPTIMEOUT_LIST3 +
+          ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
+   }
+}
+
+/*
  * Initialisation.  Called after the page allocator have been initialised and
  * before smp_init().
  */
@@ -1432,6 +1445,7 @@ void __init kmem_cache_init(void)
       if (i < MAX_NUMNODES)
          cache_cache.nodelists[i] = NULL;
    }
+   set_up_list3s(&cache_cache, CACHE_CACHE);
 
    /*
     * Fragmentation resistance on low memory - only use bigger
@@ -1587,10 +1601,9 @@ void __init kmem_cache_init(void)
    {
       int nid;
 
-      /* Replace the static kmem_list3 structures for the boot cpu */
-      init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], node);
-
       for_each_online_node(nid) {
+         init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], nid);
+
          init_list(malloc_sizes[INDEX_AC].cs_cachep,
               &initkmem_list3[SIZE_AC + nid], nid);
 
@@ -1960,22 +1973,6 @@ static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp)
    }
 }
 
-/*
- * For setting up all the kmem_list3s for cache whose buffer_size is same as
- * size of kmem_list3.
- */
-static void __init set_up_list3s(struct kmem_cache *cachep, int index)
-{
-   int node;
-
-   for_each_online_node(node) {
-      cachep->nodelists[node] = &initkmem_list3[index + node];
-      cachep->nodelists[node]->next_reap = jiffies +
-          REAPTIMEOUT_LIST3 +
-          ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
-   }
-}
-
 static void __kmem_cache_destroy(struct kmem_cache *cachep)
 {
    int i;
@@ -2099,7 +2096,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
          g_cpucache_up = PARTIAL_L3;
       } else {
          int node;
-         for_each_node_state(node, N_NORMAL_MEMORY) {
+         for_each_online_node(node) {
             cachep->nodelists[node] =
                 kmalloc_node(sizeof(struct kmem_list3),
                   GFP_KERNEL, node);
@@ -2170,6 +2167,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
     * We use cache_chain_mutex to ensure a consistent view of
     * cpu_online_map as well.  Please see cpuup_callback
     */
+   get_online_cpus();
    mutex_lock(&cache_chain_mutex);
 
    list_for_each_entry(pc, &cache_chain, next) {
@@ -2396,6 +2394,7 @@ oops:
       panic("kmem_cache_create(): failed to create slab `%s'\n",
             name);
    mutex_unlock(&cache_chain_mutex);
+   put_online_cpus();
    return cachep;
 }
 EXPORT_SYMBOL(kmem_cache_create);
@@ -2547,9 +2546,11 @@ int kmem_cache_shrink(struct kmem_cache *cachep)
    int ret;
    BUG_ON(!cachep || in_interrupt());
 
+   get_online_cpus();
    mutex_lock(&cache_chain_mutex);
    ret = __cache_shrink(cachep);
    mutex_unlock(&cache_chain_mutex);
+   put_online_cpus();
    return ret;
 }
 EXPORT_SYMBOL(kmem_cache_shrink);
@@ -2575,6 +2576,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
    BUG_ON(!cachep || in_interrupt());
 
    /* Find the cache in the chain of caches. */
+   get_online_cpus();
    mutex_lock(&cache_chain_mutex);
    /*
     * the chain is never empty, cache_cache is never destroyed
@@ -2584,6 +2586,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
       slab_error(cachep, "Can't free all objects");
       list_add(&cachep->next, &cache_chain);
       mutex_unlock(&cache_chain_mutex);
+      put_online_cpus();
       return;
    }
 
@@ -2592,6 +2595,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep)
 
    __kmem_cache_destroy(cachep);
    mutex_unlock(&cache_chain_mutex);
+   put_online_cpus();
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
 
@@ -2626,6 +2630,7 @@ static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
    slabp->colouroff = colour_off;
    slabp->s_mem = objp + colour_off;
    slabp->nodeid = nodeid;
+   slabp->free = 0;
    return slabp;
 }
 
@@ -2679,7 +2684,6 @@ static void cache_init_objs(struct kmem_cache *cachep,
       slab_bufctl(slabp)[i] = i + 1;
    }
    slab_bufctl(slabp)[i - 1] = BUFCTL_END;
-   slabp->free = 0;
 }
 
 static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
@@ -2812,7 +2816,6 @@ static int cache_grow(struct kmem_cache *cachep,
    if (!slabp)
       goto opps1;
 
-   slabp->nodeid = nodeid;
    slab_map_pages(cachep, slabp, objp);
 
    cache_init_objs(cachep, slabp);
@@ -2961,11 +2964,10 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
    struct array_cache *ac;
    int node;
 
-   node = numa_node_id();
-
+retry:
    check_irq_off();
+   node = numa_node_id();
    ac = cpu_cache_get(cachep);
-retry:
    batchcount = ac->batchcount;
    if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
       /*
@@ -3277,7 +3279,7 @@ retry:
       if (local_flags & __GFP_WAIT)
          local_irq_enable();
       kmem_flagcheck(cache, flags);
-      obj = kmem_getpages(cache, flags, -1);
+      obj = kmem_getpages(cache, local_flags, -1);
       if (local_flags & __GFP_WAIT)
          local_irq_disable();
       if (obj) {