xfs: use separate btree cursor slab for each btree type
Now that we have the infrastructure to track the max possible height of
each btree type, we can create a separate slab zone for cursors of each
type of btree. For smaller indices like the free space btrees, this
means that we can pack more cursors into a slab page, improving slab
utilization.
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
diff --git a/include/kmem.h b/include/kmem.h
index 6d31725..c710635 100644
--- a/include/kmem.h
+++ b/include/kmem.h
@@ -12,9 +12,11 @@
#define KM_NOLOCKDEP 0x0020u
typedef struct kmem_zone {
- int zone_unitsize; /* Size in bytes of zone unit */
- char *zone_name; /* tag name */
- int allocated; /* debug: How many currently allocated */
+ int zone_unitsize; /* Size in bytes of zone unit */
+ int allocated; /* debug: How many allocated? */
+ unsigned int align;
+ const char *zone_name; /* tag name */
+ void (*ctor)(void *);
} kmem_zone_t;
typedef unsigned int __bitwise gfp_t;
@@ -26,11 +28,28 @@
#define __GFP_ZERO (__force gfp_t)1
-extern kmem_zone_t *kmem_zone_init(int, char *);
+kmem_zone_t * kmem_cache_create(const char *name, unsigned int size,
+ unsigned int align, unsigned int slab_flags,
+ void (*ctor)(void *));
+
+static inline kmem_zone_t *
+kmem_zone_init(unsigned int size, const char *name)
+{
+ return kmem_cache_create(name, size, 0, 0, NULL);
+}
+
extern void *kmem_cache_alloc(kmem_zone_t *, gfp_t);
extern void *kmem_cache_zalloc(kmem_zone_t *, gfp_t);
extern int kmem_zone_destroy(kmem_zone_t *);
+
+static inline void
+kmem_cache_destroy(kmem_zone_t *zone)
+{
+ kmem_zone_destroy(zone);
+}
+
+
static inline void
kmem_cache_free(kmem_zone_t *zone, void *ptr)
{
diff --git a/libxfs/init.c b/libxfs/init.c
index 39f2b9d..35dcffd 100644
--- a/libxfs/init.c
+++ b/libxfs/init.c
@@ -268,9 +268,6 @@
if (error)
abort();
- xfs_btree_cur_zone = kmem_zone_init(
- xfs_btree_cur_sizeof(XFS_BTREE_CUR_ZONE_MAXLEVELS),
- "xfs_btree_cur");
xfs_bmap_free_item_zone = kmem_zone_init(
sizeof(struct xfs_extent_free_item),
"xfs_bmap_free_item");
@@ -289,7 +286,7 @@
leaked += kmem_zone_destroy(xfs_ifork_zone);
leaked += kmem_zone_destroy(xfs_buf_item_zone);
leaked += kmem_zone_destroy(xfs_da_state_zone);
- leaked += kmem_zone_destroy(xfs_btree_cur_zone);
+ xfs_btree_destroy_cursor_caches();
leaked += kmem_zone_destroy(xfs_bmap_free_item_zone);
leaked += kmem_zone_destroy(xfs_trans_zone);
diff --git a/libxfs/kmem.c b/libxfs/kmem.c
index 3d72ac9..221b348 100644
--- a/libxfs/kmem.c
+++ b/libxfs/kmem.c
@@ -6,9 +6,9 @@
/*
* Simple memory interface
*/
-
kmem_zone_t *
-kmem_zone_init(int size, char *name)
+kmem_cache_create(const char *name, unsigned int size, unsigned int align,
+ unsigned int slab_flags, void (*ctor)(void *))
{
kmem_zone_t *ptr = malloc(sizeof(kmem_zone_t));
@@ -21,6 +21,9 @@
ptr->zone_unitsize = size;
ptr->zone_name = name;
ptr->allocated = 0;
+ ptr->align = align;
+ ptr->ctor = ctor;
+
return ptr;
}
@@ -41,7 +44,17 @@
void *
kmem_cache_alloc(kmem_zone_t *zone, gfp_t flags)
{
- void *ptr = malloc(zone->zone_unitsize);
+ void *ptr = NULL;
+
+ if (zone->align) {
+ int ret;
+
+ ret = posix_memalign(&ptr, zone->align, zone->zone_unitsize);
+ if (ret)
+ errno = ret;
+ } else {
+ ptr = malloc(zone->zone_unitsize);
+ }
if (ptr == NULL) {
fprintf(stderr, _("%s: zone alloc failed (%s, %d bytes): %s\n"),
@@ -49,6 +62,9 @@
strerror(errno));
exit(1);
}
+
+ if (zone->ctor)
+ zone->ctor(ptr);
zone->allocated++;
return ptr;
}
diff --git a/libxfs/xfs_btree.c b/libxfs/xfs_btree.c
index a51de97..0fed52f 100644
--- a/libxfs/xfs_btree.c
+++ b/libxfs/xfs_btree.c
@@ -24,10 +24,9 @@
/*
* Cursor allocation zone.
*/
-kmem_zone_t *xfs_btree_cur_zone;
-
struct xfs_btree_cur_cache {
const char *name;
+ kmem_zone_t *cache;
unsigned short maxlevels;
bool alias;
};
@@ -371,6 +370,7 @@
struct xfs_btree_cur *cur, /* btree cursor */
int error) /* del because of error */
{
+ struct xfs_btree_cur_cache *cc = &xfs_btree_cur_caches[cur->bc_btnum];
int i; /* btree level */
/*
@@ -393,7 +393,7 @@
kmem_free(cur->bc_ops);
if (!(cur->bc_flags & XFS_BTREE_LONG_PTRS) && cur->bc_ag.pag)
xfs_perag_put(cur->bc_ag.pag);
- kmem_cache_free(xfs_btree_cur_zone, cur);
+ kmem_cache_free(cc->cache, cur);
}
/*
@@ -5330,10 +5330,11 @@
{
struct xfs_btree_cur *cur;
unsigned int maxlevels = xfs_btree_maxlevels(mp, btnum);
+ struct xfs_btree_cur_cache *cc = &xfs_btree_cur_caches[btnum];
- ASSERT(maxlevels <= XFS_BTREE_CUR_ZONE_MAXLEVELS);
+ ASSERT(maxlevels <= cc->maxlevels);
- cur = kmem_cache_zalloc(xfs_btree_cur_zone, GFP_NOFS | __GFP_NOFAIL);
+ cur = kmem_cache_zalloc(cc->cache, GFP_NOFS | __GFP_NOFAIL);
cur->bc_tp = tp;
cur->bc_mp = mp;
cur->bc_btnum = btnum;
@@ -5391,6 +5392,12 @@
unsigned int maxlevels)
{
struct xfs_btree_cur_cache *cc;
+ kmem_zone_t *cache;
+
+ cache = kmem_cache_create(name, xfs_btree_cur_sizeof(maxlevels), 0, 0,
+ NULL);
+ if (!cache)
+ return -ENOMEM;
cc = &xfs_btree_cur_caches[btnum];
@@ -5399,6 +5406,7 @@
cc->name = name;
cc->maxlevels = maxlevels;
cc->alias = false;
+ cc->cache = cache;
return 0;
}
@@ -5422,10 +5430,28 @@
cd->name = cs->name;
cd->maxlevels = cs->maxlevels;
cd->alias = true;
+ cd->cache = cs->cache;
return 0;
}
+/* Destroy all btree cursor caches. */
+void
+xfs_btree_destroy_cursor_caches(void)
+{
+ struct xfs_btree_cur_cache *cc = xfs_btree_cur_caches;
+ unsigned int i = 0;
+
+ for (; i < ARRAY_SIZE(xfs_btree_cur_caches); i--, cc++) {
+ cc->name = NULL;
+ cc->maxlevels = 0;
+ if (!cc->alias && cc->cache)
+ kmem_cache_destroy(cc->cache);
+ cc->alias = NULL;
+ cc->cache = NULL;
+ }
+}
+
/* Return the maximum possible height for a given type of btree. */
unsigned int
xfs_btree_absolute_maxlevels(
diff --git a/libxfs/xfs_btree.h b/libxfs/xfs_btree.h
index f84d36b..276da72 100644
--- a/libxfs/xfs_btree.h
+++ b/libxfs/xfs_btree.h
@@ -13,8 +13,6 @@
struct xfs_ifork;
struct xfs_perag;
-extern kmem_zone_t *xfs_btree_cur_zone;
-
/*
* Generic key, ptr and record wrapper structures.
*
@@ -92,12 +90,6 @@
#define XFS_BTREE_STATS_ADD(cur, stat, val) \
XFS_STATS_ADD_OFF((cur)->bc_mp, (cur)->bc_statoff + __XBTS_ ## stat, val)
-/*
- * The btree cursor zone hands out cursors that can handle up to this many
- * levels. This is the known maximum for all btree types.
- */
-#define XFS_BTREE_CUR_ZONE_MAXLEVELS (9)
-
struct xfs_btree_ops {
/* size of the key and record structures */
size_t key_len;
@@ -599,5 +591,6 @@
unsigned int maxlevels);
int __init xfs_btree_alias_cursor_cache(xfs_btnum_t btnum, xfs_btnum_t src);
unsigned int xfs_btree_absolute_maxlevels(xfs_btnum_t btnum);
+void xfs_btree_destroy_cursor_caches(void);
#endif /* __XFS_BTREE_H__ */
