Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1158498 > unrolled thread
| Started by | Dan Streetman <ddstreet@ieee.org> |
|---|---|
| First post | 2015-06-04 14:10 +0200 |
| Last post | 2015-06-04 14:20 +0200 |
| Articles | 2 — 1 participant |
Back to article view | Back to linux.kernel
[PATCH] zswap: dynamic pool creation Dan Streetman <ddstreet@ieee.org> - 2015-06-04 14:10 +0200
Re: [PATCH] zswap: dynamic pool creation Dan Streetman <ddstreet@ieee.org> - 2015-06-04 14:20 +0200
| From | Dan Streetman <ddstreet@ieee.org> |
|---|---|
| Date | 2015-06-04 14:10 +0200 |
| Subject | [PATCH] zswap: dynamic pool creation |
| Message-ID | <pxC8b-lr-15@gated-at.bofh.it> |
Add dynamic creation of pools. Move the static crypto compression
per-cpu transforms into each pool. Add a pointer to zswap_entry to
the pool it's in.
This is required by a separate patch which enables changing the
zswap zpool and compressor params at runtime.
Signed-off-by: Dan Streetman <ddstreet@ieee.org>
---
mm/zswap.c | 550 +++++++++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 408 insertions(+), 142 deletions(-)
diff --git a/mm/zswap.c b/mm/zswap.c
index 2d5727b..fc93770 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -99,66 +99,19 @@ module_param_named(zpool, zswap_zpool_type, charp, 0444);
static struct zpool *zswap_pool;
/*********************************
-* compression functions
+* data structures
**********************************/
-/* per-cpu compression transforms */
-static struct crypto_comp * __percpu *zswap_comp_pcpu_tfms;
-enum comp_op {
- ZSWAP_COMPOP_COMPRESS,
- ZSWAP_COMPOP_DECOMPRESS
+struct zswap_pool {
+ struct zpool *zpool;
+ struct kref kref;
+ struct list_head list;
+ struct rcu_head rcu_head;
+ struct notifier_block notifier;
+ char tfm_name[CRYPTO_MAX_ALG_NAME];
+ struct crypto_comp * __percpu *tfm;
};
-static int zswap_comp_op(enum comp_op op, const u8 *src, unsigned int slen,
- u8 *dst, unsigned int *dlen)
-{
- struct crypto_comp *tfm;
- int ret;
-
- tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, get_cpu());
- switch (op) {
- case ZSWAP_COMPOP_COMPRESS:
- ret = crypto_comp_compress(tfm, src, slen, dst, dlen);
- break;
- case ZSWAP_COMPOP_DECOMPRESS:
- ret = crypto_comp_decompress(tfm, src, slen, dst, dlen);
- break;
- default:
- ret = -EINVAL;
- }
-
- put_cpu();
- return ret;
-}
-
-static int __init zswap_comp_init(void)
-{
- if (!crypto_has_comp(zswap_compressor, 0, 0)) {
- pr_info("%s compressor not available\n", zswap_compressor);
- /* fall back to default compressor */
- zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
- if (!crypto_has_comp(zswap_compressor, 0, 0))
- /* can't even load the default compressor */
- return -ENODEV;
- }
- pr_info("using %s compressor\n", zswap_compressor);
-
- /* alloc percpu transforms */
- zswap_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *);
- if (!zswap_comp_pcpu_tfms)
- return -ENOMEM;
- return 0;
-}
-
-static void __init zswap_comp_exit(void)
-{
- /* free percpu transforms */
- free_percpu(zswap_comp_pcpu_tfms);
-}
-
-/*********************************
-* data structures
-**********************************/
/*
* struct zswap_entry
*
@@ -166,22 +119,24 @@ static void __init zswap_comp_exit(void)
* page within zswap.
*
* rbnode - links the entry into red-black tree for the appropriate swap type
+ * offset - the swap offset for the entry. Index into the red-black tree.
* refcount - the number of outstanding reference to the entry. This is needed
* to protect against premature freeing of the entry by code
* concurrent calls to load, invalidate, and writeback. The lock
* for the zswap_tree structure that contains the entry must
* be held while changing the refcount. Since the lock must
* be held, there is no reason to also make refcount atomic.
- * offset - the swap offset for the entry. Index into the red-black tree.
- * handle - zpool allocation handle that stores the compressed page data
* length - the length in bytes of the compressed page data. Needed during
* decompression
+ * pool - the zswap_pool the entry's data is in
+ * handle - zpool allocation handle that stores the compressed page data
*/
struct zswap_entry {
struct rb_node rbnode;
pgoff_t offset;
int refcount;
unsigned int length;
+ struct zswap_pool *pool;
unsigned long handle;
};
@@ -201,6 +156,44 @@ struct zswap_tree {
static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
+/* RCU-protected iteration */
+static LIST_HEAD(zswap_pools);
+/* protects zswap_pools list modification */
+static DEFINE_SPINLOCK(zswap_pools_lock);
+
+/*********************************
+* helpers and fwd declarations
+**********************************/
+
+#define zswap_pool_debug(msg, p) \
+ pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
+ zpool_get_type((p)->zpool))
+
+static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
+static int zswap_pool_get(struct zswap_pool *pool);
+static void zswap_pool_put(struct zswap_pool *pool);
+
+static bool zswap_is_full(void)
+{
+ return totalram_pages * zswap_max_pool_percent / 100 <
+ DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
+}
+
+static void zswap_update_total_size(void)
+{
+ struct zswap_pool *pool;
+ u64 total = 0;
+
+ rcu_read_lock();
+
+ list_for_each_entry_rcu(pool, &zswap_pools, list)
+ total += zpool_get_total_size(pool->zpool);
+
+ rcu_read_unlock();
+
+ zswap_pool_total_size = total;
+}
+
/*********************************
* zswap entry functions
**********************************/
@@ -294,10 +287,11 @@ static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
*/
static void zswap_free_entry(struct zswap_entry *entry)
{
- zpool_free(zswap_pool, entry->handle);
+ zpool_free(entry->pool->zpool, entry->handle);
+ zswap_pool_put(entry->pool);
zswap_entry_cache_free(entry);
atomic_dec(&zswap_stored_pages);
- zswap_pool_total_size = zpool_get_total_size(zswap_pool);
+ zswap_update_total_size();
}
/* caller must hold the tree lock */
@@ -339,35 +333,21 @@ static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
**********************************/
static DEFINE_PER_CPU(u8 *, zswap_dstmem);
-static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu)
+static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu)
{
- struct crypto_comp *tfm;
u8 *dst;
switch (action) {
case CPU_UP_PREPARE:
- tfm = crypto_alloc_comp(zswap_compressor, 0, 0);
- if (IS_ERR(tfm)) {
- pr_err("can't allocate compressor transform\n");
- return NOTIFY_BAD;
- }
- *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = tfm;
dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
if (!dst) {
pr_err("can't allocate compressor buffer\n");
- crypto_free_comp(tfm);
- *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
return NOTIFY_BAD;
}
per_cpu(zswap_dstmem, cpu) = dst;
break;
case CPU_DEAD:
case CPU_UP_CANCELED:
- tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu);
- if (tfm) {
- crypto_free_comp(tfm);
- *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
- }
dst = per_cpu(zswap_dstmem, cpu);
kfree(dst);
per_cpu(zswap_dstmem, cpu) = NULL;
@@ -378,43 +358,309 @@ static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu)
return NOTIFY_OK;
}
-static int zswap_cpu_notifier(struct notifier_block *nb,
- unsigned long action, void *pcpu)
+static int zswap_cpu_dstmem_notifier(struct notifier_block *nb,
+ unsigned long action, void *pcpu)
{
- unsigned long cpu = (unsigned long)pcpu;
- return __zswap_cpu_notifier(action, cpu);
+ return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu);
}
-static struct notifier_block zswap_cpu_notifier_block = {
- .notifier_call = zswap_cpu_notifier
+static struct notifier_block zswap_dstmem_notifier = {
+ .notifier_call = zswap_cpu_dstmem_notifier,
};
-static int __init zswap_cpu_init(void)
+static int __init zswap_cpu_dstmem_init(void)
{
unsigned long cpu;
cpu_notifier_register_begin();
for_each_online_cpu(cpu)
- if (__zswap_cpu_notifier(CPU_UP_PREPARE, cpu) != NOTIFY_OK)
+ if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) ==
+ NOTIFY_BAD)
goto cleanup;
- __register_cpu_notifier(&zswap_cpu_notifier_block);
+ __register_cpu_notifier(&zswap_dstmem_notifier);
cpu_notifier_register_done();
return 0;
cleanup:
for_each_online_cpu(cpu)
- __zswap_cpu_notifier(CPU_UP_CANCELED, cpu);
+ __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
cpu_notifier_register_done();
return -ENOMEM;
}
+static void zswap_cpu_dstmem_destroy(void)
+{
+ unsigned long cpu;
+
+ cpu_notifier_register_begin();
+ for_each_online_cpu(cpu)
+ __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
+ __unregister_cpu_notifier(&zswap_dstmem_notifier);
+ cpu_notifier_register_done();
+}
+
+static int __zswap_cpu_comp_notifier(struct zswap_pool *pool,
+ unsigned long action, unsigned long cpu)
+{
+ struct crypto_comp *tfm;
+
+ switch (action) {
+ case CPU_UP_PREPARE:
+ if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
+ break;
+ tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
+ if (IS_ERR_OR_NULL(tfm)) {
+ pr_err("could not alloc crypto comp %s : %ld\n",
+ pool->tfm_name, PTR_ERR(tfm));
+ return NOTIFY_BAD;
+ }
+ *per_cpu_ptr(pool->tfm, cpu) = tfm;
+ break;
+ case CPU_DEAD:
+ case CPU_UP_CANCELED:
+ tfm = *per_cpu_ptr(pool->tfm, cpu);
+ if (!IS_ERR_OR_NULL(tfm))
+ crypto_free_comp(tfm);
+ *per_cpu_ptr(pool->tfm, cpu) = NULL;
+ break;
+ default:
+ break;
+ }
+ return NOTIFY_OK;
+}
+
+static int zswap_cpu_comp_notifier(struct notifier_block *nb,
+ unsigned long action, void *pcpu)
+{
+ unsigned long cpu = (unsigned long)pcpu;
+ struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier);
+
+ return __zswap_cpu_comp_notifier(pool, action, cpu);
+}
+
+static int zswap_cpu_comp_init(struct zswap_pool *pool)
+{
+ unsigned long cpu;
+
+ memset(&pool->notifier, 0, sizeof(pool->notifier));
+ pool->notifier.notifier_call = zswap_cpu_comp_notifier;
+
+ cpu_notifier_register_begin();
+ for_each_online_cpu(cpu)
+ if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) ==
+ NOTIFY_BAD)
+ goto cleanup;
+ __register_cpu_notifier(&pool->notifier);
+ cpu_notifier_register_done();
+ return 0;
+
+cleanup:
+ for_each_online_cpu(cpu)
+ __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
+ cpu_notifier_register_done();
+ return -ENOMEM;
+}
+
+static void zswap_cpu_comp_destroy(struct zswap_pool *pool)
+{
+ unsigned long cpu;
+
+ cpu_notifier_register_begin();
+ for_each_online_cpu(cpu)
+ __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
+ __unregister_cpu_notifier(&pool->notifier);
+ cpu_notifier_register_done();
+}
+
/*********************************
-* helpers
+* pool functions
**********************************/
-static bool zswap_is_full(void)
+
+static struct zswap_pool *__zswap_pool_current(void)
{
- return totalram_pages * zswap_max_pool_percent / 100 <
- DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
+ struct zswap_pool *pool;
+
+ pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
+ WARN_ON(!pool);
+
+ return pool;
+}
+
+static struct zswap_pool *zswap_pool_current(void)
+{
+ assert_spin_locked(&zswap_pools_lock);
+
+ return __zswap_pool_current();
+}
+
+static struct zswap_pool *zswap_pool_current_get(void)
+{
+ struct zswap_pool *pool;
+
+ rcu_read_lock();
+
+ pool = __zswap_pool_current();
+ if (!pool || !zswap_pool_get(pool))
+ pool = NULL;
+
+ rcu_read_unlock();
+
+ return pool;
+}
+
+static struct zswap_pool *zswap_pool_last_get(void)
+{
+ struct zswap_pool *pool, *last = NULL;
+
+ rcu_read_lock();
+
+ list_for_each_entry_rcu(pool, &zswap_pools, list)
+ last = pool;
+ if (!WARN_ON(!last) && !zswap_pool_get(last))
+ last = NULL;
+
+ rcu_read_unlock();
+
+ return last;
+}
+
+static struct zpool_ops zswap_zpool_ops = {
+ .evict = zswap_writeback_entry
+};
+
+static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
+{
+ struct zswap_pool *pool;
+
+ assert_spin_locked(&zswap_pools_lock);
+
+ list_for_each_entry_rcu(pool, &zswap_pools, list) {
+ if (strncmp(pool->tfm_name, compressor, sizeof(pool->tfm_name)))
+ continue;
+ if (strncmp(zpool_get_type(pool->zpool), type,
+ sizeof(zswap_zpool_type)))
+ continue;
+ /* if we can't get it, it's about to be destroyed */
+ if (!zswap_pool_get(pool))
+ continue;
+ return pool;
+ }
+
+ return NULL;
+}
+
+static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
+{
+ struct zswap_pool *pool;
+ gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN;
+
+ pool = kzalloc(sizeof(*pool), GFP_KERNEL);
+ if (!pool) {
+ pr_err("pool alloc failed\n");
+ return NULL;
+ }
+
+ pool->zpool = zpool_create_pool(type, "zswap", gfp, &zswap_zpool_ops);
+ if (!pool->zpool) {
+ pr_err("%s zpool not available\n", type);
+ goto error;
+ }
+ pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
+
+ strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
+ pool->tfm = alloc_percpu(struct crypto_comp *);
+ if (!pool->tfm) {
+ pr_err("percpu alloc failed\n");
+ goto error;
+ }
+
+ if (zswap_cpu_comp_init(pool))
+ goto error;
+ pr_debug("using %s compressor\n", pool->tfm_name);
+
+ /* being the current pool takes 1 ref; this func expects the
+ * caller to always add the new pool as the current pool
+ */
+ kref_init(&pool->kref);
+ INIT_LIST_HEAD(&pool->list);
+
+ zswap_pool_debug("created", pool);
+
+ return pool;
+
+error:
+ free_percpu(pool->tfm);
+ if (pool->zpool)
+ zpool_destroy_pool(pool->zpool);
+ kfree(pool);
+ return NULL;
+}
+
+static struct zswap_pool *__zswap_pool_create_fallback(void)
+{
+ if (!crypto_has_comp(zswap_compressor, 0, 0)) {
+ pr_err("compressor %s not available, using default %s\n",
+ zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
+ strncpy(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT,
+ sizeof(zswap_compressor));
+ }
+ if (!zpool_has_pool(zswap_zpool_type)) {
+ pr_err("zpool %s not available, using default %s\n",
+ zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
+ strncpy(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT,
+ sizeof(zswap_zpool_type));
+ }
+
+ return zswap_pool_create(zswap_zpool_type, zswap_compressor);
+}
+
+static void zswap_pool_destroy(struct zswap_pool *pool)
+{
+ zswap_pool_debug("destroying", pool);
+
+ zswap_cpu_comp_destroy(pool);
+ free_percpu(pool->tfm);
+ zpool_destroy_pool(pool->zpool);
+ kfree(pool);
+}
+
+static int __must_check zswap_pool_get(struct zswap_pool *pool)
+{
+ return kref_get_unless_zero(&pool->kref);
+}
+
+static void __zswap_pool_release(struct rcu_head *head)
+{
+ struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head);
+
+ /* nobody should have been able to get a kref... */
+ WARN_ON(kref_get_unless_zero(&pool->kref));
+
+ /* pool is now off zswap_pools list and has no references. */
+ zswap_pool_destroy(pool);
+}
+
+static void __zswap_pool_empty(struct kref *kref)
+{
+ struct zswap_pool *pool;
+
+ pool = container_of(kref, typeof(*pool), kref);
+
+ spin_lock(&zswap_pools_lock);
+
+ WARN_ON(pool == zswap_pool_current());
+
+ list_del_rcu(&pool->list);
+ call_rcu(&pool->rcu_head, __zswap_pool_release);
+
+ spin_unlock(&zswap_pools_lock);
+}
+
+static void zswap_pool_put(struct zswap_pool *pool)
+{
+ kref_put(&pool->kref, __zswap_pool_empty);
+}
+
}
/*********************************
@@ -538,6 +784,7 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
pgoff_t offset;
struct zswap_entry *entry;
struct page *page;
+ struct crypto_comp *tfm;
u8 *src, *dst;
unsigned int dlen;
int ret;
@@ -578,13 +825,15 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
case ZSWAP_SWAPCACHE_NEW: /* page is locked */
/* decompress */
dlen = PAGE_SIZE;
- src = (u8 *)zpool_map_handle(zswap_pool, entry->handle,
+ src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
ZPOOL_MM_RO) + sizeof(struct zswap_header);
dst = kmap_atomic(page);
- ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src,
- entry->length, dst, &dlen);
+ tfm = *get_cpu_ptr(entry->pool->tfm);
+ ret = crypto_comp_decompress(tfm, src, entry->length,
+ dst, &dlen);
+ put_cpu_ptr(entry->pool->tfm);
kunmap_atomic(dst);
- zpool_unmap_handle(zswap_pool, entry->handle);
+ zpool_unmap_handle(entry->pool->zpool, entry->handle);
BUG_ON(ret);
BUG_ON(dlen != PAGE_SIZE);
@@ -633,6 +882,22 @@ end:
return ret;
}
+static int zswap_shrink(void)
+{
+ struct zswap_pool *pool;
+ int ret;
+
+ pool = zswap_pool_last_get();
+ if (!pool)
+ return -ENOENT;
+
+ ret = zpool_shrink(pool->zpool, 1, NULL);
+
+ zswap_pool_put(pool);
+
+ return ret;
+}
+
/*********************************
* frontswap hooks
**********************************/
@@ -642,6 +907,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
{
struct zswap_tree *tree = zswap_trees[type];
struct zswap_entry *entry, *dupentry;
+ struct crypto_comp *tfm;
int ret;
unsigned int dlen = PAGE_SIZE, len;
unsigned long handle;
@@ -657,7 +923,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
/* reclaim space if needed */
if (zswap_is_full()) {
zswap_pool_limit_hit++;
- if (zpool_shrink(zswap_pool, 1, NULL)) {
+ if (zswap_shrink()) {
zswap_reject_reclaim_fail++;
ret = -ENOMEM;
goto reject;
@@ -672,33 +938,42 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
goto reject;
}
+ /* if entry is successfully added, it keeps the reference */
+ entry->pool = zswap_pool_current_get();
+ if (!entry->pool) {
+ ret = -EINVAL;
+ goto freepage;
+ }
+
/* compress */
dst = get_cpu_var(zswap_dstmem);
+ tfm = *get_cpu_ptr(entry->pool->tfm);
src = kmap_atomic(page);
- ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
+ ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
kunmap_atomic(src);
+ put_cpu_ptr(entry->pool->tfm);
if (ret) {
ret = -EINVAL;
- goto freepage;
+ goto put_dstmem;
}
/* store */
len = dlen + sizeof(struct zswap_header);
- ret = zpool_malloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN,
- &handle);
+ ret = zpool_malloc(entry->pool->zpool, len,
+ __GFP_NORETRY | __GFP_NOWARN, &handle);
if (ret == -ENOSPC) {
zswap_reject_compress_poor++;
- goto freepage;
+ goto put_dstmem;
}
if (ret) {
zswap_reject_alloc_fail++;
- goto freepage;
+ goto put_dstmem;
}
- zhdr = zpool_map_handle(zswap_pool, handle, ZPOOL_MM_RW);
+ zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
zhdr->swpentry = swp_entry(type, offset);
buf = (u8 *)(zhdr + 1);
memcpy(buf, dst, dlen);
- zpool_unmap_handle(zswap_pool, handle);
+ zpool_unmap_handle(entry->pool->zpool, handle);
put_cpu_var(zswap_dstmem);
/* populate entry */
@@ -721,12 +996,14 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
/* update stats */
atomic_inc(&zswap_stored_pages);
- zswap_pool_total_size = zpool_get_total_size(zswap_pool);
+ zswap_update_total_size();
return 0;
-freepage:
+put_dstmem:
put_cpu_var(zswap_dstmem);
+ zswap_pool_put(entry->pool);
+freepage:
zswap_entry_cache_free(entry);
reject:
return ret;
@@ -741,6 +1018,7 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset,
{
struct zswap_tree *tree = zswap_trees[type];
struct zswap_entry *entry;
+ struct crypto_comp *tfm;
u8 *src, *dst;
unsigned int dlen;
int ret;
@@ -757,13 +1035,14 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset,
/* decompress */
dlen = PAGE_SIZE;
- src = (u8 *)zpool_map_handle(zswap_pool, entry->handle,
+ src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
ZPOOL_MM_RO) + sizeof(struct zswap_header);
dst = kmap_atomic(page);
- ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length,
- dst, &dlen);
+ tfm = *get_cpu_ptr(entry->pool->tfm);
+ ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
+ put_cpu_ptr(entry->pool->tfm);
kunmap_atomic(dst);
- zpool_unmap_handle(zswap_pool, entry->handle);
+ zpool_unmap_handle(entry->pool->zpool, entry->handle);
BUG_ON(ret);
spin_lock(&tree->lock);
@@ -816,10 +1095,6 @@ static void zswap_frontswap_invalidate_area(unsigned type)
zswap_trees[type] = NULL;
}
-static struct zpool_ops zswap_zpool_ops = {
- .evict = zswap_writeback_entry
-};
-
static void zswap_frontswap_init(unsigned type)
{
struct zswap_tree *tree;
@@ -900,49 +1175,40 @@ static void __exit zswap_debugfs_exit(void) { }
**********************************/
static int __init init_zswap(void)
{
- gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN;
+ struct zswap_pool *pool;
- pr_info("loading zswap\n");
-
- zswap_pool = zpool_create_pool(zswap_zpool_type, "zswap", gfp,
- &zswap_zpool_ops);
- if (!zswap_pool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
- pr_info("%s zpool not available\n", zswap_zpool_type);
- zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
- zswap_pool = zpool_create_pool(zswap_zpool_type, "zswap", gfp,
- &zswap_zpool_ops);
- }
- if (!zswap_pool) {
- pr_err("%s zpool not available\n", zswap_zpool_type);
- pr_err("zpool creation failed\n");
- goto error;
- }
- pr_info("using %s pool\n", zswap_zpool_type);
+ pr_info("loading\n");
if (zswap_entry_cache_create()) {
pr_err("entry cache creation failed\n");
- goto cachefail;
+ goto cache_fail;
}
- if (zswap_comp_init()) {
- pr_err("compressor initialization failed\n");
- goto compfail;
+
+ if (zswap_cpu_dstmem_init()) {
+ pr_err("dstmem alloc failed\n");
+ goto dstmem_fail;
}
- if (zswap_cpu_init()) {
- pr_err("per-cpu initialization failed\n");
- goto pcpufail;
+
+ pool = __zswap_pool_create_fallback();
+ if (!pool) {
+ pr_err("pool creation failed\n");
+ goto pool_fail;
}
+ pr_info("loaded using pool %s/%s\n", pool->tfm_name,
+ zpool_get_type(pool->zpool));
+
+ list_add(&pool->list, &zswap_pools);
frontswap_register_ops(&zswap_frontswap_ops);
if (zswap_debugfs_init())
pr_warn("debugfs initialization failed\n");
return 0;
-pcpufail:
- zswap_comp_exit();
-compfail:
+
+pool_fail:
+ zswap_cpu_dstmem_destroy();
+dstmem_fail:
zswap_entry_cache_destroy();
-cachefail:
- zpool_destroy_pool(zswap_pool);
-error:
+cache_fail:
return -ENOMEM;
}
/* must be late so crypto has time to come up */
--
2.1.0
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[toc] | [next] | [standalone]
| From | Dan Streetman <ddstreet@ieee.org> |
|---|---|
| Date | 2015-06-04 14:20 +0200 |
| Message-ID | <pxChR-xc-19@gated-at.bofh.it> |
| In reply to | #1158498 |
On Thu, Jun 4, 2015 at 8:02 AM, Dan Streetman <ddstreet@ieee.org> wrote:
> Add dynamic creation of pools. Move the static crypto compression
> per-cpu transforms into each pool. Add a pointer to zswap_entry to
> the pool it's in.
Seth, as far as the design, from your previous comments I assume you
were thinking of maintaining separate lists of zpools and compressors?
I do see how that will reduce duplication of zpools and compressors,
but it also requires adding a new pointer to each zswap_entry, and
increasing the amount of code to manage each list separately. And the
most common case in zswap will be just a single zpool and compressor,
not repeatedly changing params. What do you think?
>
> This is required by a separate patch which enables changing the
> zswap zpool and compressor params at runtime.
>
> Signed-off-by: Dan Streetman <ddstreet@ieee.org>
> ---
> mm/zswap.c | 550 +++++++++++++++++++++++++++++++++++++++++++++----------------
> 1 file changed, 408 insertions(+), 142 deletions(-)
>
> diff --git a/mm/zswap.c b/mm/zswap.c
> index 2d5727b..fc93770 100644
> --- a/mm/zswap.c
> +++ b/mm/zswap.c
> @@ -99,66 +99,19 @@ module_param_named(zpool, zswap_zpool_type, charp, 0444);
> static struct zpool *zswap_pool;
>
> /*********************************
> -* compression functions
> +* data structures
> **********************************/
> -/* per-cpu compression transforms */
> -static struct crypto_comp * __percpu *zswap_comp_pcpu_tfms;
>
> -enum comp_op {
> - ZSWAP_COMPOP_COMPRESS,
> - ZSWAP_COMPOP_DECOMPRESS
> +struct zswap_pool {
> + struct zpool *zpool;
> + struct kref kref;
> + struct list_head list;
> + struct rcu_head rcu_head;
> + struct notifier_block notifier;
> + char tfm_name[CRYPTO_MAX_ALG_NAME];
> + struct crypto_comp * __percpu *tfm;
> };
>
> -static int zswap_comp_op(enum comp_op op, const u8 *src, unsigned int slen,
> - u8 *dst, unsigned int *dlen)
> -{
> - struct crypto_comp *tfm;
> - int ret;
> -
> - tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, get_cpu());
> - switch (op) {
> - case ZSWAP_COMPOP_COMPRESS:
> - ret = crypto_comp_compress(tfm, src, slen, dst, dlen);
> - break;
> - case ZSWAP_COMPOP_DECOMPRESS:
> - ret = crypto_comp_decompress(tfm, src, slen, dst, dlen);
> - break;
> - default:
> - ret = -EINVAL;
> - }
> -
> - put_cpu();
> - return ret;
> -}
> -
> -static int __init zswap_comp_init(void)
> -{
> - if (!crypto_has_comp(zswap_compressor, 0, 0)) {
> - pr_info("%s compressor not available\n", zswap_compressor);
> - /* fall back to default compressor */
> - zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
> - if (!crypto_has_comp(zswap_compressor, 0, 0))
> - /* can't even load the default compressor */
> - return -ENODEV;
> - }
> - pr_info("using %s compressor\n", zswap_compressor);
> -
> - /* alloc percpu transforms */
> - zswap_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *);
> - if (!zswap_comp_pcpu_tfms)
> - return -ENOMEM;
> - return 0;
> -}
> -
> -static void __init zswap_comp_exit(void)
> -{
> - /* free percpu transforms */
> - free_percpu(zswap_comp_pcpu_tfms);
> -}
> -
> -/*********************************
> -* data structures
> -**********************************/
> /*
> * struct zswap_entry
> *
> @@ -166,22 +119,24 @@ static void __init zswap_comp_exit(void)
> * page within zswap.
> *
> * rbnode - links the entry into red-black tree for the appropriate swap type
> + * offset - the swap offset for the entry. Index into the red-black tree.
> * refcount - the number of outstanding reference to the entry. This is needed
> * to protect against premature freeing of the entry by code
> * concurrent calls to load, invalidate, and writeback. The lock
> * for the zswap_tree structure that contains the entry must
> * be held while changing the refcount. Since the lock must
> * be held, there is no reason to also make refcount atomic.
> - * offset - the swap offset for the entry. Index into the red-black tree.
> - * handle - zpool allocation handle that stores the compressed page data
> * length - the length in bytes of the compressed page data. Needed during
> * decompression
> + * pool - the zswap_pool the entry's data is in
> + * handle - zpool allocation handle that stores the compressed page data
> */
> struct zswap_entry {
> struct rb_node rbnode;
> pgoff_t offset;
> int refcount;
> unsigned int length;
> + struct zswap_pool *pool;
> unsigned long handle;
> };
>
> @@ -201,6 +156,44 @@ struct zswap_tree {
>
> static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
>
> +/* RCU-protected iteration */
> +static LIST_HEAD(zswap_pools);
> +/* protects zswap_pools list modification */
> +static DEFINE_SPINLOCK(zswap_pools_lock);
> +
> +/*********************************
> +* helpers and fwd declarations
> +**********************************/
> +
> +#define zswap_pool_debug(msg, p) \
> + pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
> + zpool_get_type((p)->zpool))
> +
> +static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
> +static int zswap_pool_get(struct zswap_pool *pool);
> +static void zswap_pool_put(struct zswap_pool *pool);
> +
> +static bool zswap_is_full(void)
> +{
> + return totalram_pages * zswap_max_pool_percent / 100 <
> + DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
> +}
> +
> +static void zswap_update_total_size(void)
> +{
> + struct zswap_pool *pool;
> + u64 total = 0;
> +
> + rcu_read_lock();
> +
> + list_for_each_entry_rcu(pool, &zswap_pools, list)
> + total += zpool_get_total_size(pool->zpool);
> +
> + rcu_read_unlock();
> +
> + zswap_pool_total_size = total;
> +}
> +
> /*********************************
> * zswap entry functions
> **********************************/
> @@ -294,10 +287,11 @@ static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
> */
> static void zswap_free_entry(struct zswap_entry *entry)
> {
> - zpool_free(zswap_pool, entry->handle);
> + zpool_free(entry->pool->zpool, entry->handle);
> + zswap_pool_put(entry->pool);
> zswap_entry_cache_free(entry);
> atomic_dec(&zswap_stored_pages);
> - zswap_pool_total_size = zpool_get_total_size(zswap_pool);
> + zswap_update_total_size();
> }
>
> /* caller must hold the tree lock */
> @@ -339,35 +333,21 @@ static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
> **********************************/
> static DEFINE_PER_CPU(u8 *, zswap_dstmem);
>
> -static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu)
> +static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu)
> {
> - struct crypto_comp *tfm;
> u8 *dst;
>
> switch (action) {
> case CPU_UP_PREPARE:
> - tfm = crypto_alloc_comp(zswap_compressor, 0, 0);
> - if (IS_ERR(tfm)) {
> - pr_err("can't allocate compressor transform\n");
> - return NOTIFY_BAD;
> - }
> - *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = tfm;
> dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
> if (!dst) {
> pr_err("can't allocate compressor buffer\n");
> - crypto_free_comp(tfm);
> - *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
> return NOTIFY_BAD;
> }
> per_cpu(zswap_dstmem, cpu) = dst;
> break;
> case CPU_DEAD:
> case CPU_UP_CANCELED:
> - tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu);
> - if (tfm) {
> - crypto_free_comp(tfm);
> - *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
> - }
> dst = per_cpu(zswap_dstmem, cpu);
> kfree(dst);
> per_cpu(zswap_dstmem, cpu) = NULL;
> @@ -378,43 +358,309 @@ static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu)
> return NOTIFY_OK;
> }
>
> -static int zswap_cpu_notifier(struct notifier_block *nb,
> - unsigned long action, void *pcpu)
> +static int zswap_cpu_dstmem_notifier(struct notifier_block *nb,
> + unsigned long action, void *pcpu)
> {
> - unsigned long cpu = (unsigned long)pcpu;
> - return __zswap_cpu_notifier(action, cpu);
> + return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu);
> }
>
> -static struct notifier_block zswap_cpu_notifier_block = {
> - .notifier_call = zswap_cpu_notifier
> +static struct notifier_block zswap_dstmem_notifier = {
> + .notifier_call = zswap_cpu_dstmem_notifier,
> };
>
> -static int __init zswap_cpu_init(void)
> +static int __init zswap_cpu_dstmem_init(void)
> {
> unsigned long cpu;
>
> cpu_notifier_register_begin();
> for_each_online_cpu(cpu)
> - if (__zswap_cpu_notifier(CPU_UP_PREPARE, cpu) != NOTIFY_OK)
> + if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) ==
> + NOTIFY_BAD)
> goto cleanup;
> - __register_cpu_notifier(&zswap_cpu_notifier_block);
> + __register_cpu_notifier(&zswap_dstmem_notifier);
> cpu_notifier_register_done();
> return 0;
>
> cleanup:
> for_each_online_cpu(cpu)
> - __zswap_cpu_notifier(CPU_UP_CANCELED, cpu);
> + __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
> cpu_notifier_register_done();
> return -ENOMEM;
> }
>
> +static void zswap_cpu_dstmem_destroy(void)
> +{
> + unsigned long cpu;
> +
> + cpu_notifier_register_begin();
> + for_each_online_cpu(cpu)
> + __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
> + __unregister_cpu_notifier(&zswap_dstmem_notifier);
> + cpu_notifier_register_done();
> +}
> +
> +static int __zswap_cpu_comp_notifier(struct zswap_pool *pool,
> + unsigned long action, unsigned long cpu)
> +{
> + struct crypto_comp *tfm;
> +
> + switch (action) {
> + case CPU_UP_PREPARE:
> + if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
> + break;
> + tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
> + if (IS_ERR_OR_NULL(tfm)) {
> + pr_err("could not alloc crypto comp %s : %ld\n",
> + pool->tfm_name, PTR_ERR(tfm));
> + return NOTIFY_BAD;
> + }
> + *per_cpu_ptr(pool->tfm, cpu) = tfm;
> + break;
> + case CPU_DEAD:
> + case CPU_UP_CANCELED:
> + tfm = *per_cpu_ptr(pool->tfm, cpu);
> + if (!IS_ERR_OR_NULL(tfm))
> + crypto_free_comp(tfm);
> + *per_cpu_ptr(pool->tfm, cpu) = NULL;
> + break;
> + default:
> + break;
> + }
> + return NOTIFY_OK;
> +}
> +
> +static int zswap_cpu_comp_notifier(struct notifier_block *nb,
> + unsigned long action, void *pcpu)
> +{
> + unsigned long cpu = (unsigned long)pcpu;
> + struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier);
> +
> + return __zswap_cpu_comp_notifier(pool, action, cpu);
> +}
> +
> +static int zswap_cpu_comp_init(struct zswap_pool *pool)
> +{
> + unsigned long cpu;
> +
> + memset(&pool->notifier, 0, sizeof(pool->notifier));
> + pool->notifier.notifier_call = zswap_cpu_comp_notifier;
> +
> + cpu_notifier_register_begin();
> + for_each_online_cpu(cpu)
> + if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) ==
> + NOTIFY_BAD)
> + goto cleanup;
> + __register_cpu_notifier(&pool->notifier);
> + cpu_notifier_register_done();
> + return 0;
> +
> +cleanup:
> + for_each_online_cpu(cpu)
> + __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
> + cpu_notifier_register_done();
> + return -ENOMEM;
> +}
> +
> +static void zswap_cpu_comp_destroy(struct zswap_pool *pool)
> +{
> + unsigned long cpu;
> +
> + cpu_notifier_register_begin();
> + for_each_online_cpu(cpu)
> + __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
> + __unregister_cpu_notifier(&pool->notifier);
> + cpu_notifier_register_done();
> +}
> +
> /*********************************
> -* helpers
> +* pool functions
> **********************************/
> -static bool zswap_is_full(void)
> +
> +static struct zswap_pool *__zswap_pool_current(void)
> {
> - return totalram_pages * zswap_max_pool_percent / 100 <
> - DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
> + struct zswap_pool *pool;
> +
> + pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
> + WARN_ON(!pool);
> +
> + return pool;
> +}
> +
> +static struct zswap_pool *zswap_pool_current(void)
> +{
> + assert_spin_locked(&zswap_pools_lock);
> +
> + return __zswap_pool_current();
> +}
> +
> +static struct zswap_pool *zswap_pool_current_get(void)
> +{
> + struct zswap_pool *pool;
> +
> + rcu_read_lock();
> +
> + pool = __zswap_pool_current();
> + if (!pool || !zswap_pool_get(pool))
> + pool = NULL;
> +
> + rcu_read_unlock();
> +
> + return pool;
> +}
> +
> +static struct zswap_pool *zswap_pool_last_get(void)
> +{
> + struct zswap_pool *pool, *last = NULL;
> +
> + rcu_read_lock();
> +
> + list_for_each_entry_rcu(pool, &zswap_pools, list)
> + last = pool;
> + if (!WARN_ON(!last) && !zswap_pool_get(last))
> + last = NULL;
> +
> + rcu_read_unlock();
> +
> + return last;
> +}
> +
> +static struct zpool_ops zswap_zpool_ops = {
> + .evict = zswap_writeback_entry
> +};
> +
> +static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
> +{
> + struct zswap_pool *pool;
> +
> + assert_spin_locked(&zswap_pools_lock);
> +
> + list_for_each_entry_rcu(pool, &zswap_pools, list) {
> + if (strncmp(pool->tfm_name, compressor, sizeof(pool->tfm_name)))
> + continue;
> + if (strncmp(zpool_get_type(pool->zpool), type,
> + sizeof(zswap_zpool_type)))
> + continue;
> + /* if we can't get it, it's about to be destroyed */
> + if (!zswap_pool_get(pool))
> + continue;
> + return pool;
> + }
> +
> + return NULL;
> +}
> +
> +static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
> +{
> + struct zswap_pool *pool;
> + gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN;
> +
> + pool = kzalloc(sizeof(*pool), GFP_KERNEL);
> + if (!pool) {
> + pr_err("pool alloc failed\n");
> + return NULL;
> + }
> +
> + pool->zpool = zpool_create_pool(type, "zswap", gfp, &zswap_zpool_ops);
> + if (!pool->zpool) {
> + pr_err("%s zpool not available\n", type);
> + goto error;
> + }
> + pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
> +
> + strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
> + pool->tfm = alloc_percpu(struct crypto_comp *);
> + if (!pool->tfm) {
> + pr_err("percpu alloc failed\n");
> + goto error;
> + }
> +
> + if (zswap_cpu_comp_init(pool))
> + goto error;
> + pr_debug("using %s compressor\n", pool->tfm_name);
> +
> + /* being the current pool takes 1 ref; this func expects the
> + * caller to always add the new pool as the current pool
> + */
> + kref_init(&pool->kref);
> + INIT_LIST_HEAD(&pool->list);
> +
> + zswap_pool_debug("created", pool);
> +
> + return pool;
> +
> +error:
> + free_percpu(pool->tfm);
> + if (pool->zpool)
> + zpool_destroy_pool(pool->zpool);
> + kfree(pool);
> + return NULL;
> +}
> +
> +static struct zswap_pool *__zswap_pool_create_fallback(void)
> +{
> + if (!crypto_has_comp(zswap_compressor, 0, 0)) {
> + pr_err("compressor %s not available, using default %s\n",
> + zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
> + strncpy(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT,
> + sizeof(zswap_compressor));
> + }
> + if (!zpool_has_pool(zswap_zpool_type)) {
> + pr_err("zpool %s not available, using default %s\n",
> + zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
> + strncpy(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT,
> + sizeof(zswap_zpool_type));
> + }
> +
> + return zswap_pool_create(zswap_zpool_type, zswap_compressor);
> +}
> +
> +static void zswap_pool_destroy(struct zswap_pool *pool)
> +{
> + zswap_pool_debug("destroying", pool);
> +
> + zswap_cpu_comp_destroy(pool);
> + free_percpu(pool->tfm);
> + zpool_destroy_pool(pool->zpool);
> + kfree(pool);
> +}
> +
> +static int __must_check zswap_pool_get(struct zswap_pool *pool)
> +{
> + return kref_get_unless_zero(&pool->kref);
> +}
> +
> +static void __zswap_pool_release(struct rcu_head *head)
> +{
> + struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head);
> +
> + /* nobody should have been able to get a kref... */
> + WARN_ON(kref_get_unless_zero(&pool->kref));
> +
> + /* pool is now off zswap_pools list and has no references. */
> + zswap_pool_destroy(pool);
> +}
> +
> +static void __zswap_pool_empty(struct kref *kref)
> +{
> + struct zswap_pool *pool;
> +
> + pool = container_of(kref, typeof(*pool), kref);
> +
> + spin_lock(&zswap_pools_lock);
> +
> + WARN_ON(pool == zswap_pool_current());
> +
> + list_del_rcu(&pool->list);
> + call_rcu(&pool->rcu_head, __zswap_pool_release);
> +
> + spin_unlock(&zswap_pools_lock);
> +}
> +
> +static void zswap_pool_put(struct zswap_pool *pool)
> +{
> + kref_put(&pool->kref, __zswap_pool_empty);
> +}
> +
> }
>
> /*********************************
> @@ -538,6 +784,7 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
> pgoff_t offset;
> struct zswap_entry *entry;
> struct page *page;
> + struct crypto_comp *tfm;
> u8 *src, *dst;
> unsigned int dlen;
> int ret;
> @@ -578,13 +825,15 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
> case ZSWAP_SWAPCACHE_NEW: /* page is locked */
> /* decompress */
> dlen = PAGE_SIZE;
> - src = (u8 *)zpool_map_handle(zswap_pool, entry->handle,
> + src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
> ZPOOL_MM_RO) + sizeof(struct zswap_header);
> dst = kmap_atomic(page);
> - ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src,
> - entry->length, dst, &dlen);
> + tfm = *get_cpu_ptr(entry->pool->tfm);
> + ret = crypto_comp_decompress(tfm, src, entry->length,
> + dst, &dlen);
> + put_cpu_ptr(entry->pool->tfm);
> kunmap_atomic(dst);
> - zpool_unmap_handle(zswap_pool, entry->handle);
> + zpool_unmap_handle(entry->pool->zpool, entry->handle);
> BUG_ON(ret);
> BUG_ON(dlen != PAGE_SIZE);
>
> @@ -633,6 +882,22 @@ end:
> return ret;
> }
>
> +static int zswap_shrink(void)
> +{
> + struct zswap_pool *pool;
> + int ret;
> +
> + pool = zswap_pool_last_get();
> + if (!pool)
> + return -ENOENT;
> +
> + ret = zpool_shrink(pool->zpool, 1, NULL);
> +
> + zswap_pool_put(pool);
> +
> + return ret;
> +}
> +
> /*********************************
> * frontswap hooks
> **********************************/
> @@ -642,6 +907,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
> {
> struct zswap_tree *tree = zswap_trees[type];
> struct zswap_entry *entry, *dupentry;
> + struct crypto_comp *tfm;
> int ret;
> unsigned int dlen = PAGE_SIZE, len;
> unsigned long handle;
> @@ -657,7 +923,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
> /* reclaim space if needed */
> if (zswap_is_full()) {
> zswap_pool_limit_hit++;
> - if (zpool_shrink(zswap_pool, 1, NULL)) {
> + if (zswap_shrink()) {
> zswap_reject_reclaim_fail++;
> ret = -ENOMEM;
> goto reject;
> @@ -672,33 +938,42 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
> goto reject;
> }
>
> + /* if entry is successfully added, it keeps the reference */
> + entry->pool = zswap_pool_current_get();
> + if (!entry->pool) {
> + ret = -EINVAL;
> + goto freepage;
> + }
> +
> /* compress */
> dst = get_cpu_var(zswap_dstmem);
> + tfm = *get_cpu_ptr(entry->pool->tfm);
> src = kmap_atomic(page);
> - ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
> + ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
> kunmap_atomic(src);
> + put_cpu_ptr(entry->pool->tfm);
> if (ret) {
> ret = -EINVAL;
> - goto freepage;
> + goto put_dstmem;
> }
>
> /* store */
> len = dlen + sizeof(struct zswap_header);
> - ret = zpool_malloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN,
> - &handle);
> + ret = zpool_malloc(entry->pool->zpool, len,
> + __GFP_NORETRY | __GFP_NOWARN, &handle);
> if (ret == -ENOSPC) {
> zswap_reject_compress_poor++;
> - goto freepage;
> + goto put_dstmem;
> }
> if (ret) {
> zswap_reject_alloc_fail++;
> - goto freepage;
> + goto put_dstmem;
> }
> - zhdr = zpool_map_handle(zswap_pool, handle, ZPOOL_MM_RW);
> + zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
> zhdr->swpentry = swp_entry(type, offset);
> buf = (u8 *)(zhdr + 1);
> memcpy(buf, dst, dlen);
> - zpool_unmap_handle(zswap_pool, handle);
> + zpool_unmap_handle(entry->pool->zpool, handle);
> put_cpu_var(zswap_dstmem);
>
> /* populate entry */
> @@ -721,12 +996,14 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset,
>
> /* update stats */
> atomic_inc(&zswap_stored_pages);
> - zswap_pool_total_size = zpool_get_total_size(zswap_pool);
> + zswap_update_total_size();
>
> return 0;
>
> -freepage:
> +put_dstmem:
> put_cpu_var(zswap_dstmem);
> + zswap_pool_put(entry->pool);
> +freepage:
> zswap_entry_cache_free(entry);
> reject:
> return ret;
> @@ -741,6 +1018,7 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset,
> {
> struct zswap_tree *tree = zswap_trees[type];
> struct zswap_entry *entry;
> + struct crypto_comp *tfm;
> u8 *src, *dst;
> unsigned int dlen;
> int ret;
> @@ -757,13 +1035,14 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset,
>
> /* decompress */
> dlen = PAGE_SIZE;
> - src = (u8 *)zpool_map_handle(zswap_pool, entry->handle,
> + src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
> ZPOOL_MM_RO) + sizeof(struct zswap_header);
> dst = kmap_atomic(page);
> - ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length,
> - dst, &dlen);
> + tfm = *get_cpu_ptr(entry->pool->tfm);
> + ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
> + put_cpu_ptr(entry->pool->tfm);
> kunmap_atomic(dst);
> - zpool_unmap_handle(zswap_pool, entry->handle);
> + zpool_unmap_handle(entry->pool->zpool, entry->handle);
> BUG_ON(ret);
>
> spin_lock(&tree->lock);
> @@ -816,10 +1095,6 @@ static void zswap_frontswap_invalidate_area(unsigned type)
> zswap_trees[type] = NULL;
> }
>
> -static struct zpool_ops zswap_zpool_ops = {
> - .evict = zswap_writeback_entry
> -};
> -
> static void zswap_frontswap_init(unsigned type)
> {
> struct zswap_tree *tree;
> @@ -900,49 +1175,40 @@ static void __exit zswap_debugfs_exit(void) { }
> **********************************/
> static int __init init_zswap(void)
> {
> - gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN;
> + struct zswap_pool *pool;
>
> - pr_info("loading zswap\n");
> -
> - zswap_pool = zpool_create_pool(zswap_zpool_type, "zswap", gfp,
> - &zswap_zpool_ops);
> - if (!zswap_pool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
> - pr_info("%s zpool not available\n", zswap_zpool_type);
> - zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
> - zswap_pool = zpool_create_pool(zswap_zpool_type, "zswap", gfp,
> - &zswap_zpool_ops);
> - }
> - if (!zswap_pool) {
> - pr_err("%s zpool not available\n", zswap_zpool_type);
> - pr_err("zpool creation failed\n");
> - goto error;
> - }
> - pr_info("using %s pool\n", zswap_zpool_type);
> + pr_info("loading\n");
>
> if (zswap_entry_cache_create()) {
> pr_err("entry cache creation failed\n");
> - goto cachefail;
> + goto cache_fail;
> }
> - if (zswap_comp_init()) {
> - pr_err("compressor initialization failed\n");
> - goto compfail;
> +
> + if (zswap_cpu_dstmem_init()) {
> + pr_err("dstmem alloc failed\n");
> + goto dstmem_fail;
> }
> - if (zswap_cpu_init()) {
> - pr_err("per-cpu initialization failed\n");
> - goto pcpufail;
> +
> + pool = __zswap_pool_create_fallback();
> + if (!pool) {
> + pr_err("pool creation failed\n");
> + goto pool_fail;
> }
> + pr_info("loaded using pool %s/%s\n", pool->tfm_name,
> + zpool_get_type(pool->zpool));
> +
> + list_add(&pool->list, &zswap_pools);
>
> frontswap_register_ops(&zswap_frontswap_ops);
> if (zswap_debugfs_init())
> pr_warn("debugfs initialization failed\n");
> return 0;
> -pcpufail:
> - zswap_comp_exit();
> -compfail:
> +
> +pool_fail:
> + zswap_cpu_dstmem_destroy();
> +dstmem_fail:
> zswap_entry_cache_destroy();
> -cachefail:
> - zpool_destroy_pool(zswap_pool);
> -error:
> +cache_fail:
> return -ENOMEM;
> }
> /* must be late so crypto has time to come up */
> --
> 2.1.0
>
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[toc] | [prev] | [standalone]
Back to top | Article view | linux.kernel
csiph-web