Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1326766 > unrolled thread

[RFC 0/4] lightnvm: add write buffering to rrpc

Back to article view | Back to linux.kernel

Contents

  [RFC 0/4] lightnvm: add write buffering to rrpc "Javier González" <jg@lightnvm.io> - 2016-02-04 14:10 +0100
    [RFC 4/4] lightnvm: add debug info for rrpc target "Javier González" <jg@lightnvm.io> - 2016-02-04 14:10 +0100
    [RFC 3/4] lightnvm: read from rrpc write buffer if possible "Javier González" <jg@lightnvm.io> - 2016-02-04 14:10 +0100
      Re: [RFC 3/4] lightnvm: read from rrpc write buffer if possible Matias Bjørling <mb@lightnvm.io> - 2016-02-05 16:00 +0100
    [RFC 1/4] lightnvm: precalculate controller write boundaries "Javier González" <jg@lightnvm.io> - 2016-02-04 14:10 +0100
      Re: [RFC 1/4] lightnvm: precalculate controller write boundaries Matias Bjørling <mb@lightnvm.io> - 2016-02-05 16:00 +0100

#1326766 — [RFC 0/4] lightnvm: add write buffering to rrpc

Hi,

This patchset adds write buffering to rrpc. This is necessary to support
writes that are smaller than the actual flash controller page size.
Typically, these are larger tahn 4K (e.g., 64K) since they are organized
organized in several 4K sectors and planes.

The code can be tested on the lightnvm enabled qemu, built on top of
Keith Busch's qemu-nvme [1]. At the moment, different flushing
strategies are enabled by flags. I am now working on flushing the writer
work queue when syncs are not aligned with the device's page size, but
this should not change the general architecture.

Any feedback on the architecture, design and implementation are more
than welcome.

Thanks,
Javier

[1] https://github.com/OpenChannelSSD/qemu-nvme

Javier González (4):
  lightnvm: precalculate controller write boundaries
  lightnvm: add write buffering for rrpc
  lightnvm: read from rrpc write buffer if possible
  lightnvm: add debug info for rrpc target

 drivers/lightnvm/core.c  |   10 +
 drivers/lightnvm/rrpc.c  | 1233 ++++++++++++++++++++++++++++++++++++++--------
 drivers/lightnvm/rrpc.h  |   93 +++-
 include/linux/lightnvm.h |   11 +-
 4 files changed, 1138 insertions(+), 209 deletions(-)

-- 
2.1.4

[toc] | [next] | [standalone]

#1326768 — [RFC 4/4] lightnvm: add debug info for rrpc target

Add a target to print its debug information when lightnvm debug
information is printed trough configure_debug.

In rrpc, we add statistics on inflight reads and writes, as well as
submitted and synced writes.

Signed-off-by: Javier González <javier@cnexlabs.com>
---
 drivers/lightnvm/core.c  | 10 +++++++++
 drivers/lightnvm/rrpc.c  | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/lightnvm/rrpc.h  |  8 +++++++
 include/linux/lightnvm.h |  4 ++++
 4 files changed, 79 insertions(+)

diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
index 5471cc5..ef64243 100644
--- a/drivers/lightnvm/core.c
+++ b/drivers/lightnvm/core.c
@@ -771,6 +771,8 @@ static int __nvm_configure_remove(struct nvm_ioctl_remove *remove)
 static int nvm_configure_show(const char *val)
 {
 	struct nvm_dev *dev;
+	struct nvm_target *t = NULL;
+	struct nvm_tgt_type *tt;
 	char opcode, devname[DISK_NAME_LEN];
 	int ret;
 
@@ -793,6 +795,14 @@ static int nvm_configure_show(const char *val)
 
 	dev->mt->lun_info_print(dev);
 
+	down_write(&nvm_lock);
+	list_for_each_entry(dev, &nvm_devices, devices)
+		list_for_each_entry(t, &dev->online_targets, list) {
+			tt = t->type;
+			tt->print_debug(t->disk->private_data);
+		}
+	up_write(&nvm_lock);
+
 	return 0;
 }
 
diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
index 6348d52..41bd9b0 100644
--- a/drivers/lightnvm/rrpc.c
+++ b/drivers/lightnvm/rrpc.c
@@ -784,6 +784,11 @@ static void rrpc_sync_buffer(struct rrpc *rrpc, struct rrpc_addr *p)
 
 	WARN_ON(test_and_set_bit((p->addr - bppa), buf->sync_bitmap));
 
+#ifdef CONFIG_NVM_DEBUG
+		atomic_dec(&rrpc->inflight_writes);
+		atomic_inc(&rrpc->sync_writes);
+#endif
+
 	if (unlikely(bitmap_full(buf->sync_bitmap, buf->nentries))) {
 		/* Write buffer out-of-bounds */
 		WARN_ON((buf->cur_mem != buf->nentries) &&
@@ -820,6 +825,9 @@ static void rrpc_end_io_read(struct rrpc *rrpc, struct nvm_rq *rqd,
 
 	rrpc_unlock_rq(rrpc, rrqd);
 	mempool_free(rrqd, rrpc->rrq_pool);
+#ifdef CONFIG_NVM_DEBUG
+	atomic_sub(nr_pages, &rrpc->inflight_reads);
+#endif
 }
 
 static void rrpc_end_io(struct nvm_rq *rqd)
@@ -919,6 +927,11 @@ static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 
 		rrqd->addr = p;
 
+#ifdef CONFIG_NVM_DEBUG
+		atomic_inc(&rrpc->inflight_writes);
+		atomic_inc(&rrpc->req_writes);
+#endif
+
 		err = rrpc_write_to_buffer(rrpc, bio, rrqd, p, w_buf, flags);
 		if (err) {
 			pr_err("rrpc: could not write to write buffer\n");
@@ -967,6 +980,11 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
 
 	rrqd->addr = p;
 
+#ifdef CONFIG_NVM_DEBUG
+	atomic_inc(&rrpc->inflight_writes);
+	atomic_inc(&rrpc->req_writes);
+#endif
+
 	err = rrpc_write_to_buffer(rrpc, bio, rrqd, p, w_buf, flags);
 	if (err) {
 		pr_err("rrpc: could not write to write buffer\n");
@@ -1023,6 +1041,10 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 		}
 
 		brrqd[i].addr = gp;
+
+#ifdef CONFIG_NVM_DEBUG
+		atomic_inc(&rrpc->inflight_reads);
+#endif
 	}
 
 	rqd->opcode = NVM_OP_HBREAD;
@@ -1055,6 +1077,10 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
 	rqd->opcode = NVM_OP_HBREAD;
 	rrqd->addr = gp;
 
+#ifdef CONFIG_NVM_DEBUG
+		atomic_inc(&rrpc->inflight_reads);
+#endif
+
 	return NVM_IO_OK;
 }
 
@@ -1615,6 +1641,9 @@ submit_io:
 			mempool_free(rqd, rrpc->rq_pool);
 			bio_put(bio);
 		}
+#ifdef CONFIG_NVM_DEBUG
+		atomic_add(pgs_to_sync, &rrpc->sub_writes);
+#endif
 	}
 
 	spin_unlock(&rlun->parent->lock);
@@ -2128,6 +2157,14 @@ static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk,
 	/* simple round-robin strategy */
 	atomic_set(&rrpc->next_lun, -1);
 
+#ifdef CONFIG_NVM_DEBUG
+	atomic_set(&rrpc->inflight_writes, 0);
+	atomic_set(&rrpc->req_writes, 0);
+	atomic_set(&rrpc->sub_writes, 0);
+	atomic_set(&rrpc->sync_writes, 0);
+	atomic_set(&rrpc->inflight_reads, 0);
+#endif
+
 	ret = rrpc_luns_init(rrpc, lun_begin, lun_end);
 	if (ret) {
 		pr_err("nvm: rrpc: could not initialize luns\n");
@@ -2182,6 +2219,24 @@ err:
 	return ERR_PTR(ret);
 }
 
+#ifdef CONFIG_NVM_DEBUG
+static void rrpc_print_debug(void *private)
+{
+	struct rrpc *rrpc = private;
+
+	pr_info("rrpc: %u\t%u\t%u\t%u\t%u\n",
+				atomic_read(&rrpc->inflight_writes),
+				atomic_read(&rrpc->inflight_reads),
+				atomic_read(&rrpc->req_writes),
+				atomic_read(&rrpc->sub_writes),
+				atomic_read(&rrpc->sync_writes));
+}
+#else
+static void rrpc_print_debug(void *private)
+{
+}
+#endif
+
 /* round robin, page-based FTL, and cost-based GC */
 static struct nvm_tgt_type tt_rrpc = {
 	.name		= "rrpc",
@@ -2193,6 +2248,8 @@ static struct nvm_tgt_type tt_rrpc = {
 
 	.init		= rrpc_init,
 	.exit		= rrpc_exit,
+
+	.print_debug	= rrpc_print_debug,
 };
 
 static int __init rrpc_module_init(void)
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
index 6e188b4..b7c40de 100644
--- a/drivers/lightnvm/rrpc.h
+++ b/drivers/lightnvm/rrpc.h
@@ -167,6 +167,14 @@ struct rrpc {
 			    * to point to the next write lun
 			    */
 
+#ifdef CONFIG_NVM_DEBUG
+	atomic_t inflight_writes;
+	atomic_t req_writes;
+	atomic_t sub_writes;
+	atomic_t sync_writes;
+	atomic_t inflight_reads;
+#endif
+
 	spinlock_t bio_lock;
 	struct bio_list requeue_bios;
 	struct work_struct ws_requeue;
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
index ae26ced..12a4296 100644
--- a/include/linux/lightnvm.h
+++ b/include/linux/lightnvm.h
@@ -428,6 +428,7 @@ typedef blk_qc_t (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *);
 typedef sector_t (nvm_tgt_capacity_fn)(void *);
 typedef void *(nvm_tgt_init_fn)(struct nvm_dev *, struct gendisk *, int, int);
 typedef void (nvm_tgt_exit_fn)(void *);
+typedef void (nvm_tgt_print_debug_fn)(void *);
 
 struct nvm_tgt_type {
 	const char *name;
@@ -442,6 +443,9 @@ struct nvm_tgt_type {
 	nvm_tgt_init_fn *init;
 	nvm_tgt_exit_fn *exit;
 
+	/* debugging */
+	nvm_tgt_print_debug_fn *print_debug;
+
 	/* For internal use */
 	struct list_head list;
 };
-- 
2.1.4

[toc] | [prev] | [next] | [standalone]

#1326773 — [RFC 3/4] lightnvm: read from rrpc write buffer if possible

Since writes are buffered in memory, incoming reads must retrieve
buffered pages instead of submitting the I/O to the media.

This patch implements this logic. When a read bio arrives to rrpc, valid
pages from the flash blocks residing in memory are copied. If there are
any "holes" in the bio, a new bio is submitted to the media to retrieve
the necessary pages. The original bio is updated accordingly.

Signed-off-by: Javier González <javier@cnexlabs.com>
---
 drivers/lightnvm/rrpc.c  | 451 ++++++++++++++++++++++++++++++++++++-----------
 include/linux/lightnvm.h |   1 +
 2 files changed, 346 insertions(+), 106 deletions(-)

diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
index e9fb19d..6348d52 100644
--- a/drivers/lightnvm/rrpc.c
+++ b/drivers/lightnvm/rrpc.c
@@ -827,10 +827,13 @@ static void rrpc_end_io(struct nvm_rq *rqd)
 	struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
 	uint8_t nr_pages = rqd->nr_pages;
 
-	if (bio_data_dir(rqd->bio) == WRITE)
+	if (bio_data_dir(rqd->bio) == WRITE) {
 		rrpc_end_io_write(rrpc, rqd, nr_pages);
-	else
+	} else {
+		if (rqd->flags & NVM_IOTYPE_SYNC)
+			return;
 		rrpc_end_io_read(rrpc, rqd, nr_pages);
+	}
 
 	bio_put(rqd->bio);
 
@@ -842,83 +845,6 @@ static void rrpc_end_io(struct nvm_rq *rqd)
 	mempool_free(rqd, rrpc->rq_pool);
 }
 
-static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
-			struct nvm_rq *rqd, struct rrpc_buf_rq *brrqd,
-			unsigned long flags, int nr_pages)
-{
-	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
-	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rrqd);
-	struct rrpc_addr *gp;
-	sector_t laddr = rrpc_get_laddr(bio);
-	int is_gc = flags & NVM_IOTYPE_GC;
-	int i;
-
-	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
-		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
-		mempool_free(rrqd, rrpc->rrq_pool);
-		mempool_free(rqd, rrpc->rq_pool);
-		return NVM_IO_REQUEUE;
-	}
-
-	for (i = 0; i < nr_pages; i++) {
-		/* We assume that mapping occurs at 4KB granularity */
-		BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
-		gp = &rrpc->trans_map[laddr + i];
-
-		if (gp->rblk) {
-			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
-								gp->addr);
-		} else {
-			BUG_ON(is_gc);
-			rrpc_unlock_laddr(rrpc, r);
-			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
-							rqd->dma_ppa_list);
-			mempool_free(rrqd, rrpc->rrq_pool);
-			mempool_free(rqd, rrpc->rq_pool);
-			return NVM_IO_DONE;
-		}
-
-		brrqd[i].addr = gp;
-	}
-
-	rqd->opcode = NVM_OP_HBREAD;
-
-	return NVM_IO_OK;
-}
-
-static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
-							unsigned long flags)
-{
-	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
-	int is_gc = flags & NVM_IOTYPE_GC;
-	sector_t laddr = rrpc_get_laddr(bio);
-	struct rrpc_addr *gp;
-
-	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
-		mempool_free(rrqd, rrpc->rrq_pool);
-		mempool_free(rqd, rrpc->rq_pool);
-		return NVM_IO_REQUEUE;
-	}
-
-	BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
-	gp = &rrpc->trans_map[laddr];
-
-	if (gp->rblk) {
-		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
-	} else {
-		BUG_ON(is_gc);
-		rrpc_unlock_rq(rrpc, rrqd);
-		mempool_free(rrqd, rrpc->rrq_pool);
-		mempool_free(rqd, rrpc->rq_pool);
-		return NVM_IO_DONE;
-	}
-
-	rqd->opcode = NVM_OP_HBREAD;
-	rrqd->addr = gp;
-
-	return NVM_IO_OK;
-}
-
 /*
  * Copy data from current bio to block write buffer. This if necessary
  * to guarantee durability if a flash block becomes bad before all pages
@@ -1051,14 +977,335 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
 	return NVM_IO_DONE;
 }
 
+static int rrpc_buffer_write(struct rrpc *rrpc, struct bio *bio,
+				struct rrpc_rq *rrqd, unsigned long flags)
+{
+	uint8_t nr_pages = rrpc_get_pages(bio);
+
+	rrqd->nr_pages = nr_pages;
+
+	if (nr_pages > 1)
+		return rrpc_write_ppalist_rq(rrpc, bio, rrqd, flags, nr_pages);
+	else
+		return rrpc_write_rq(rrpc, bio, rrqd, flags);
+}
+
+static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
+			struct nvm_rq *rqd, struct rrpc_buf_rq *brrqd,
+			unsigned long flags, int nr_pages)
+{
+	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
+	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rrqd);
+	struct rrpc_addr *gp;
+	sector_t laddr = rrpc_get_laddr(bio);
+	int is_gc = flags & NVM_IOTYPE_GC;
+	int i;
+
+	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
+		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
+		return NVM_IO_REQUEUE;
+	}
+
+	for (i = 0; i < nr_pages; i++) {
+		/* We assume that mapping occurs at 4KB granularity */
+		BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
+		gp = &rrpc->trans_map[laddr + i];
+
+		if (gp->rblk) {
+			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
+								gp->addr);
+		} else {
+			BUG_ON(is_gc);
+			rrpc_unlock_laddr(rrpc, r);
+			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
+							rqd->dma_ppa_list);
+			return NVM_IO_DONE;
+		}
+
+		brrqd[i].addr = gp;
+	}
+
+	rqd->opcode = NVM_OP_HBREAD;
+
+	return NVM_IO_OK;
+}
+
+static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
+							unsigned long flags)
+{
+	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
+	int is_gc = flags & NVM_IOTYPE_GC;
+	sector_t laddr = rrpc_get_laddr(bio);
+	struct rrpc_addr *gp;
+
+	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd))
+		return NVM_IO_REQUEUE;
+
+	BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
+	gp = &rrpc->trans_map[laddr];
+
+	if (gp->rblk) {
+		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
+	} else {
+		BUG_ON(is_gc);
+		rrpc_unlock_rq(rrpc, rrqd);
+		return NVM_IO_DONE;
+	}
+
+	rqd->opcode = NVM_OP_HBREAD;
+	rrqd->addr = gp;
+
+	return NVM_IO_OK;
+}
+
+static int rrpc_read_w_buf_entry(struct bio *bio, struct rrpc_block *rblk,
+					struct bvec_iter iter, int entry)
+{
+	struct buf_entry *read_entry;
+	struct bio_vec bv;
+	struct page *page;
+	void *kaddr;
+	void *data;
+	int read = 0;
+
+	lockdep_assert_held(&rblk->w_buf.s_lock);
+
+	spin_lock(&rblk->w_buf.w_lock);
+	if (entry >= rblk->w_buf.cur_mem) {
+		spin_unlock(&rblk->w_buf.w_lock);
+		goto out;
+	}
+	spin_unlock(&rblk->w_buf.w_lock);
+
+	read_entry = &rblk->w_buf.entries[entry];
+	data = read_entry->data;
+
+	bv = bio_iter_iovec(bio, iter);
+	page = bv.bv_page;
+	kaddr = kmap_atomic(page);
+	memcpy(kaddr + bv.bv_offset, data, RRPC_EXPOSED_PAGE_SIZE);
+	kunmap_atomic(kaddr);
+	read++;
+
+out:
+	return read;
+}
+
+static int rrpc_read_from_w_buf(struct rrpc *rrpc, struct nvm_rq *rqd,
+			struct rrpc_buf_rq *brrqd, unsigned long *read_bitmap)
+{
+	struct nvm_dev *dev = rrpc->dev;
+	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
+	struct rrpc_addr *addr;
+	struct bio *bio = rqd->bio;
+	struct bvec_iter iter = bio->bi_iter;
+	struct rrpc_block *rblk;
+	unsigned long blk_id;
+	int nr_pages = rqd->nr_pages;
+	int left = nr_pages;
+	int read = 0;
+	int entry;
+	int i;
+
+	if (nr_pages != bio->bi_vcnt)
+		goto out;
+
+	if (nr_pages == 1) {
+		rblk = rrqd->addr->rblk;
+
+		/* If the write buffer exists, the block is open in memory */
+		spin_lock(&rblk->w_buf.s_lock);
+		atomic_inc(&rblk->w_buf.refs);
+		if (rblk->w_buf.entries) {
+			blk_id = rblk->parent->id;
+			entry = rrqd->addr->addr -
+				(blk_id * dev->sec_per_pg * dev->pgs_per_blk);
+
+			read = rrpc_read_w_buf_entry(bio, rblk, iter, entry);
+
+			left -= read;
+			WARN_ON(test_and_set_bit(0, read_bitmap));
+		}
+		bio_advance_iter(bio, &iter, RRPC_EXPOSED_PAGE_SIZE);
+
+		atomic_dec(&rblk->w_buf.refs);
+		spin_unlock(&rblk->w_buf.s_lock);
+
+		goto out;
+	}
+
+	/* Iterate through all pages and copy those that are found in the write
+	 * buffer. We will complete the holes (if any) with a intermediate bio
+	 * later on
+	 */
+	for (i = 0; i < nr_pages; i++) {
+		addr = brrqd[i].addr;
+		rblk = addr->rblk;
+
+		/* If the write buffer exists, the block is open in memory */
+		spin_lock(&rblk->w_buf.s_lock);
+		atomic_inc(&rblk->w_buf.refs);
+		if (rblk->w_buf.entries) {
+			blk_id = rblk->parent->id;
+			entry = addr->addr - (blk_id * dev->sec_per_pg *
+							dev->pgs_per_blk);
+
+			read = rrpc_read_w_buf_entry(bio, rblk, iter, entry);
+
+			left -= read;
+			WARN_ON(test_and_set_bit(i, read_bitmap));
+		}
+		bio_advance_iter(bio, &iter, RRPC_EXPOSED_PAGE_SIZE);
+
+		atomic_dec(&rblk->w_buf.refs);
+		spin_unlock(&rblk->w_buf.s_lock);
+	}
+
+out:
+	return left;
+}
+
+static int rrpc_submit_read_io(struct rrpc *rrpc, struct bio *bio,
+				struct nvm_rq *rqd, unsigned long flags)
+{
+	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
+	int err;
+
+	err = nvm_submit_io(rrpc->dev, rqd);
+	if (err) {
+		pr_err("rrpc: I/O submission failed: %d\n", err);
+		bio_put(bio);
+		if (!(flags & NVM_IOTYPE_GC)) {
+			rrpc_unlock_rq(rrpc, rrqd);
+			if (rqd->nr_pages > 1)
+				nvm_dev_dma_free(rrpc->dev,
+			rqd->ppa_list, rqd->dma_ppa_list);
+		}
+		return NVM_IO_ERR;
+	}
+
+	return NVM_IO_OK;
+}
+
+static int rrpc_fill_partial_read_bio(struct rrpc *rrpc, struct bio *bio,
+				unsigned long *read_bitmap, struct nvm_rq *rqd,
+				struct rrpc_buf_rq *brrqd, uint8_t nr_pages)
+{
+	struct bio *new_bio;
+	struct page *page;
+	struct bio_vec src_bv, dst_bv;
+	void *src_p, *dst_p;
+	int nr_holes = nr_pages - bitmap_weight(read_bitmap, nr_pages);
+	int hole;
+	int i = 0;
+	int ret;
+	DECLARE_COMPLETION_ONSTACK(wait);
+
+	new_bio = bio_alloc(GFP_KERNEL, nr_holes);
+	if (!new_bio) {
+		pr_err("nvm: rrpc: could not alloc read bio\n");
+		return NVM_IO_ERR;
+	}
+
+	hole = find_first_zero_bit(read_bitmap, nr_pages);
+	do {
+		page = mempool_alloc(rrpc->page_pool, GFP_KERNEL);
+		if (!page) {
+			bio_put(new_bio);
+			pr_err("nvm: rrpc: could not alloc read page\n");
+			goto err;
+		}
+
+		ret = bio_add_page(new_bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
+		if (ret != RRPC_EXPOSED_PAGE_SIZE) {
+			pr_err("nvm: rrpc: could not add page to bio\n");
+			mempool_free(page, rrpc->page_pool);
+			goto err;
+		}
+
+		rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
+							brrqd[hole].addr->addr);
+
+		i++;
+		hole = find_next_zero_bit(read_bitmap, nr_pages, hole + 1);
+	} while (hole != nr_pages);
+
+	if (nr_holes != new_bio->bi_vcnt) {
+		pr_err("rrpc: malformed bio\n");
+		goto err;
+	}
+
+	new_bio->bi_iter.bi_sector = bio->bi_iter.bi_sector;
+	new_bio->bi_rw = READ;
+	new_bio->bi_private = &wait;
+	new_bio->bi_end_io = rrpc_end_sync_bio;
+
+	rqd->flags |= NVM_IOTYPE_SYNC;
+	rqd->bio = new_bio;
+	rqd->nr_pages = nr_holes;
+
+	rrpc_submit_read_io(rrpc, new_bio, rqd, rqd->flags);
+	wait_for_completion_io(&wait);
+
+	if (new_bio->bi_error)
+		goto err;
+
+	/* Fill the holes in the original bio */
+	i = 0;
+	hole = find_first_zero_bit(read_bitmap, nr_pages);
+	do {
+		src_bv = new_bio->bi_io_vec[i];
+		dst_bv = bio->bi_io_vec[hole];
+
+		src_p = kmap_atomic(src_bv.bv_page);
+		dst_p = kmap_atomic(dst_bv.bv_page);
+
+		memcpy(dst_p + dst_bv.bv_offset,
+			src_p + src_bv.bv_offset,
+			RRPC_EXPOSED_PAGE_SIZE);
+
+		kunmap_atomic(src_p);
+		kunmap_atomic(dst_p);
+
+		mempool_free(&src_bv.bv_page, rrpc->page_pool);
+
+		i++;
+		hole = find_next_zero_bit(read_bitmap, nr_pages, hole + 1);
+	} while (hole != nr_pages);
+
+	bio_put(new_bio);
+
+	/* Complete the original bio and associated request */
+	rqd->flags &= ~NVM_IOTYPE_SYNC;
+	rqd->bio = bio;
+	rqd->nr_pages = nr_pages;
+
+	bio_endio(bio);
+	rrpc_end_io(rqd);
+	return NVM_IO_OK;
+
+err:
+	/* Free allocated pages in new bio */
+	for (i = 0; i < new_bio->bi_vcnt; i++) {
+		src_bv = new_bio->bi_io_vec[i];
+		mempool_free(&src_bv.bv_page, rrpc->page_pool);
+	}
+	bio_endio(new_bio);
+	return NVM_IO_ERR;
+}
+
 static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
 				struct rrpc_rq *rrqd, unsigned long flags)
 {
 	struct nvm_rq *rqd;
 	struct rrpc_buf_rq brrqd[rrpc->max_write_pgs];
+	unsigned long read_bitmap; /* Max 64 ppas per request */
+	uint8_t left;
 	uint8_t nr_pages = rrpc_get_pages(bio);
 	int err;
 
+	bitmap_zero(&read_bitmap, nr_pages);
+
 	rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
 	if (!rqd) {
 		pr_err_ratelimited("rrpc: not able to queue bio.");
@@ -1073,22 +1320,25 @@ static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
 						&rqd->dma_ppa_list);
 		if (!rqd->ppa_list) {
 			pr_err("rrpc: not able to allocate ppa list\n");
-			mempool_free(rrqd, rrpc->rrq_pool);
 			mempool_free(rqd, rrpc->rq_pool);
+			mempool_free(rrqd, rrpc->rrq_pool);
 			return NVM_IO_ERR;
 		}
 
 		err = rrpc_read_ppalist_rq(rrpc, bio, rqd, brrqd, flags,
 								nr_pages);
 		if (err) {
-			mempool_free(rrqd, rrpc->rrq_pool);
 			mempool_free(rqd, rrpc->rq_pool);
+			mempool_free(rrqd, rrpc->rrq_pool);
 			return err;
 		}
 	} else {
 		err = rrpc_read_rq(rrpc, bio, rqd, flags);
-		if (err)
+		if (err) {
+			mempool_free(rrqd, rrpc->rrq_pool);
+			mempool_free(rqd, rrpc->rq_pool);
 			return err;
+		}
 	}
 
 	bio_get(bio);
@@ -1097,33 +1347,22 @@ static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
 	rqd->nr_pages = rrqd->nr_pages = nr_pages;
 	rqd->flags = flags;
 
-	err = nvm_submit_io(rrpc->dev, rqd);
-	if (err) {
-		pr_err("rrpc: I/O submission failed: %d\n", err);
-		bio_put(bio);
-		if (!(flags & NVM_IOTYPE_GC)) {
-			rrpc_unlock_rq(rrpc, rrqd);
-			if (rqd->nr_pages > 1)
-				nvm_dev_dma_free(rrpc->dev,
-			rqd->ppa_list, rqd->dma_ppa_list);
-		}
+	left = rrpc_read_from_w_buf(rrpc, rqd, brrqd, &read_bitmap);
+	if (left == 0) {
+		bio_endio(bio);
+		rrpc_end_io(rqd);
+		return NVM_IO_OK;
+	} else if (left < 0)
 		return NVM_IO_ERR;
-	}
 
-	return NVM_IO_OK;
-}
+	if (bitmap_empty(&read_bitmap, nr_pages))
+		return rrpc_submit_read_io(rrpc, bio, rqd, flags);
 
-static int rrpc_buffer_write(struct rrpc *rrpc, struct bio *bio,
-				struct rrpc_rq *rrqd, unsigned long flags)
-{
-	uint8_t nr_pages = rrpc_get_pages(bio);
-
-	rrqd->nr_pages = nr_pages;
-
-	if (nr_pages > 1)
-		return rrpc_write_ppalist_rq(rrpc, bio, rrqd, flags, nr_pages);
-	else
-		return rrpc_write_rq(rrpc, bio, rrqd, flags);
+	/* The read bio could not be completely read from the write buffer. This
+	 * case only occurs when several pages are sent in a single bio
+	 */
+	return rrpc_fill_partial_read_bio(rrpc, bio, &read_bitmap, rqd, brrqd,
+								nr_pages);
 }
 
 static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
index eda9743..ae26ced 100644
--- a/include/linux/lightnvm.h
+++ b/include/linux/lightnvm.h
@@ -11,6 +11,7 @@ enum {
 
 	NVM_IOTYPE_NONE = 0,
 	NVM_IOTYPE_GC = 1,
+	NVM_IOTYPE_SYNC = 2,
 };
 
 #define NVM_BLK_BITS (16)
-- 
2.1.4

[toc] | [prev] | [next] | [standalone]

#1327823 — Re: [RFC 3/4] lightnvm: read from rrpc write buffer if possible

On 02/04/2016 02:08 PM, Javier González wrote:
> Since writes are buffered in memory, incoming reads must retrieve
> buffered pages instead of submitting the I/O to the media.
> 
> This patch implements this logic. When a read bio arrives to rrpc, valid
> pages from the flash blocks residing in memory are copied. If there are
> any "holes" in the bio, a new bio is submitted to the media to retrieve
> the necessary pages. The original bio is updated accordingly.
> 
> Signed-off-by: Javier González <javier@cnexlabs.com>
> ---
>  drivers/lightnvm/rrpc.c  | 451 ++++++++++++++++++++++++++++++++++++-----------
>  include/linux/lightnvm.h |   1 +
>  2 files changed, 346 insertions(+), 106 deletions(-)
> 
> diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
> index e9fb19d..6348d52 100644
> --- a/drivers/lightnvm/rrpc.c
> +++ b/drivers/lightnvm/rrpc.c
> @@ -827,10 +827,13 @@ static void rrpc_end_io(struct nvm_rq *rqd)
>  	struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
>  	uint8_t nr_pages = rqd->nr_pages;
>  
> -	if (bio_data_dir(rqd->bio) == WRITE)
> +	if (bio_data_dir(rqd->bio) == WRITE) {
>  		rrpc_end_io_write(rrpc, rqd, nr_pages);
> -	else
> +	} else {
> +		if (rqd->flags & NVM_IOTYPE_SYNC)
> +			return;
>  		rrpc_end_io_read(rrpc, rqd, nr_pages);
> +	}
>  
>  	bio_put(rqd->bio);
>  
> @@ -842,83 +845,6 @@ static void rrpc_end_io(struct nvm_rq *rqd)
>  	mempool_free(rqd, rrpc->rq_pool);
>  }
>  
> -static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
> -			struct nvm_rq *rqd, struct rrpc_buf_rq *brrqd,
> -			unsigned long flags, int nr_pages)
> -{
> -	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> -	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rrqd);
> -	struct rrpc_addr *gp;
> -	sector_t laddr = rrpc_get_laddr(bio);
> -	int is_gc = flags & NVM_IOTYPE_GC;
> -	int i;
> -
> -	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
> -		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
> -		mempool_free(rrqd, rrpc->rrq_pool);
> -		mempool_free(rqd, rrpc->rq_pool);
> -		return NVM_IO_REQUEUE;
> -	}
> -
> -	for (i = 0; i < nr_pages; i++) {
> -		/* We assume that mapping occurs at 4KB granularity */
> -		BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
> -		gp = &rrpc->trans_map[laddr + i];
> -
> -		if (gp->rblk) {
> -			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
> -								gp->addr);
> -		} else {
> -			BUG_ON(is_gc);
> -			rrpc_unlock_laddr(rrpc, r);
> -			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
> -							rqd->dma_ppa_list);
> -			mempool_free(rrqd, rrpc->rrq_pool);
> -			mempool_free(rqd, rrpc->rq_pool);
> -			return NVM_IO_DONE;
> -		}
> -
> -		brrqd[i].addr = gp;
> -	}
> -
> -	rqd->opcode = NVM_OP_HBREAD;
> -
> -	return NVM_IO_OK;
> -}
> -
> -static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
> -							unsigned long flags)
> -{
> -	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> -	int is_gc = flags & NVM_IOTYPE_GC;
> -	sector_t laddr = rrpc_get_laddr(bio);
> -	struct rrpc_addr *gp;
> -
> -	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
> -		mempool_free(rrqd, rrpc->rrq_pool);
> -		mempool_free(rqd, rrpc->rq_pool);
> -		return NVM_IO_REQUEUE;
> -	}
> -
> -	BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
> -	gp = &rrpc->trans_map[laddr];
> -
> -	if (gp->rblk) {
> -		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
> -	} else {
> -		BUG_ON(is_gc);
> -		rrpc_unlock_rq(rrpc, rrqd);
> -		mempool_free(rrqd, rrpc->rrq_pool);
> -		mempool_free(rqd, rrpc->rq_pool);
> -		return NVM_IO_DONE;
> -	}
> -
> -	rqd->opcode = NVM_OP_HBREAD;
> -	rrqd->addr = gp;
> -
> -	return NVM_IO_OK;
> -}
> -
>  /*
>   * Copy data from current bio to block write buffer. This if necessary
>   * to guarantee durability if a flash block becomes bad before all pages
> @@ -1051,14 +977,335 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
>  	return NVM_IO_DONE;
>  }
>  
> +static int rrpc_buffer_write(struct rrpc *rrpc, struct bio *bio,
> +				struct rrpc_rq *rrqd, unsigned long flags)
> +{
> +	uint8_t nr_pages = rrpc_get_pages(bio);
> +
> +	rrqd->nr_pages = nr_pages;
> +
> +	if (nr_pages > 1)
> +		return rrpc_write_ppalist_rq(rrpc, bio, rrqd, flags, nr_pages);
> +	else
> +		return rrpc_write_rq(rrpc, bio, rrqd, flags);
> +}
> +
> +static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
> +			struct nvm_rq *rqd, struct rrpc_buf_rq *brrqd,
> +			unsigned long flags, int nr_pages)
> +{
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rrqd);
> +	struct rrpc_addr *gp;
> +	sector_t laddr = rrpc_get_laddr(bio);
> +	int is_gc = flags & NVM_IOTYPE_GC;
> +	int i;
> +
> +	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
> +		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
> +		return NVM_IO_REQUEUE;
> +	}
> +
> +	for (i = 0; i < nr_pages; i++) {
> +		/* We assume that mapping occurs at 4KB granularity */
> +		BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
> +		gp = &rrpc->trans_map[laddr + i];
> +
> +		if (gp->rblk) {
> +			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
> +								gp->addr);
> +		} else {
> +			BUG_ON(is_gc);
> +			rrpc_unlock_laddr(rrpc, r);
> +			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
> +							rqd->dma_ppa_list);
> +			return NVM_IO_DONE;
> +		}
> +
> +		brrqd[i].addr = gp;
> +	}
> +
> +	rqd->opcode = NVM_OP_HBREAD;
> +
> +	return NVM_IO_OK;
> +}
> +
> +static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
> +							unsigned long flags)
> +{
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	int is_gc = flags & NVM_IOTYPE_GC;
> +	sector_t laddr = rrpc_get_laddr(bio);
> +	struct rrpc_addr *gp;
> +
> +	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd))
> +		return NVM_IO_REQUEUE;
> +
> +	BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
> +	gp = &rrpc->trans_map[laddr];
> +
> +	if (gp->rblk) {
> +		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
> +	} else {
> +		BUG_ON(is_gc);
> +		rrpc_unlock_rq(rrpc, rrqd);
> +		return NVM_IO_DONE;
> +	}
> +
> +	rqd->opcode = NVM_OP_HBREAD;
> +	rrqd->addr = gp;
> +
> +	return NVM_IO_OK;
> +}
> +
> +static int rrpc_read_w_buf_entry(struct bio *bio, struct rrpc_block *rblk,
> +					struct bvec_iter iter, int entry)
> +{
> +	struct buf_entry *read_entry;
> +	struct bio_vec bv;
> +	struct page *page;
> +	void *kaddr;
> +	void *data;
> +	int read = 0;
> +
> +	lockdep_assert_held(&rblk->w_buf.s_lock);
> +
> +	spin_lock(&rblk->w_buf.w_lock);
> +	if (entry >= rblk->w_buf.cur_mem) {
> +		spin_unlock(&rblk->w_buf.w_lock);
> +		goto out;
> +	}
> +	spin_unlock(&rblk->w_buf.w_lock);
> +
> +	read_entry = &rblk->w_buf.entries[entry];
> +	data = read_entry->data;
> +
> +	bv = bio_iter_iovec(bio, iter);
> +	page = bv.bv_page;
> +	kaddr = kmap_atomic(page);
> +	memcpy(kaddr + bv.bv_offset, data, RRPC_EXPOSED_PAGE_SIZE);
> +	kunmap_atomic(kaddr);
> +	read++;
> +
> +out:
> +	return read;
> +}
> +
> +static int rrpc_read_from_w_buf(struct rrpc *rrpc, struct nvm_rq *rqd,
> +			struct rrpc_buf_rq *brrqd, unsigned long *read_bitmap)
> +{
> +	struct nvm_dev *dev = rrpc->dev;
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	struct rrpc_addr *addr;
> +	struct bio *bio = rqd->bio;
> +	struct bvec_iter iter = bio->bi_iter;
> +	struct rrpc_block *rblk;
> +	unsigned long blk_id;
> +	int nr_pages = rqd->nr_pages;
> +	int left = nr_pages;
> +	int read = 0;
> +	int entry;
> +	int i;
> +
> +	if (nr_pages != bio->bi_vcnt)
> +		goto out;
> +
> +	if (nr_pages == 1) {
> +		rblk = rrqd->addr->rblk;
> +
> +		/* If the write buffer exists, the block is open in memory */
> +		spin_lock(&rblk->w_buf.s_lock);
> +		atomic_inc(&rblk->w_buf.refs);
> +		if (rblk->w_buf.entries) {
> +			blk_id = rblk->parent->id;
> +			entry = rrqd->addr->addr -
> +				(blk_id * dev->sec_per_pg * dev->pgs_per_blk);
> +
> +			read = rrpc_read_w_buf_entry(bio, rblk, iter, entry);
> +
> +			left -= read;
> +			WARN_ON(test_and_set_bit(0, read_bitmap));
> +		}
> +		bio_advance_iter(bio, &iter, RRPC_EXPOSED_PAGE_SIZE);
> +
> +		atomic_dec(&rblk->w_buf.refs);
> +		spin_unlock(&rblk->w_buf.s_lock);
> +
> +		goto out;
> +	}
> +
> +	/* Iterate through all pages and copy those that are found in the write
> +	 * buffer. We will complete the holes (if any) with a intermediate bio
> +	 * later on
> +	 */
> +	for (i = 0; i < nr_pages; i++) {
> +		addr = brrqd[i].addr;
> +		rblk = addr->rblk;
> +
> +		/* If the write buffer exists, the block is open in memory */
> +		spin_lock(&rblk->w_buf.s_lock);
> +		atomic_inc(&rblk->w_buf.refs);
> +		if (rblk->w_buf.entries) {
> +			blk_id = rblk->parent->id;
> +			entry = addr->addr - (blk_id * dev->sec_per_pg *
> +							dev->pgs_per_blk);
> +
> +			read = rrpc_read_w_buf_entry(bio, rblk, iter, entry);
> +
> +			left -= read;
> +			WARN_ON(test_and_set_bit(i, read_bitmap));
> +		}
> +		bio_advance_iter(bio, &iter, RRPC_EXPOSED_PAGE_SIZE);
> +
> +		atomic_dec(&rblk->w_buf.refs);
> +		spin_unlock(&rblk->w_buf.s_lock);
> +	}
> +
> +out:
> +	return left;
> +}
> +
> +static int rrpc_submit_read_io(struct rrpc *rrpc, struct bio *bio,
> +				struct nvm_rq *rqd, unsigned long flags)
> +{
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	int err;
> +
> +	err = nvm_submit_io(rrpc->dev, rqd);
> +	if (err) {
> +		pr_err("rrpc: I/O submission failed: %d\n", err);
> +		bio_put(bio);
> +		if (!(flags & NVM_IOTYPE_GC)) {
> +			rrpc_unlock_rq(rrpc, rrqd);
> +			if (rqd->nr_pages > 1)
> +				nvm_dev_dma_free(rrpc->dev,
> +			rqd->ppa_list, rqd->dma_ppa_list);
> +		}
> +		return NVM_IO_ERR;
> +	}
> +
> +	return NVM_IO_OK;
> +}
> +
> +static int rrpc_fill_partial_read_bio(struct rrpc *rrpc, struct bio *bio,
> +				unsigned long *read_bitmap, struct nvm_rq *rqd,
> +				struct rrpc_buf_rq *brrqd, uint8_t nr_pages)
> +{
> +	struct bio *new_bio;
> +	struct page *page;
> +	struct bio_vec src_bv, dst_bv;
> +	void *src_p, *dst_p;
> +	int nr_holes = nr_pages - bitmap_weight(read_bitmap, nr_pages);
> +	int hole;
> +	int i = 0;
> +	int ret;
> +	DECLARE_COMPLETION_ONSTACK(wait);
> +
> +	new_bio = bio_alloc(GFP_KERNEL, nr_holes);
> +	if (!new_bio) {
> +		pr_err("nvm: rrpc: could not alloc read bio\n");
> +		return NVM_IO_ERR;
> +	}
> +
> +	hole = find_first_zero_bit(read_bitmap, nr_pages);
> +	do {
> +		page = mempool_alloc(rrpc->page_pool, GFP_KERNEL);
> +		if (!page) {
> +			bio_put(new_bio);
> +			pr_err("nvm: rrpc: could not alloc read page\n");
> +			goto err;
> +		}
> +
> +		ret = bio_add_page(new_bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
> +		if (ret != RRPC_EXPOSED_PAGE_SIZE) {
> +			pr_err("nvm: rrpc: could not add page to bio\n");
> +			mempool_free(page, rrpc->page_pool);
> +			goto err;
> +		}
> +
> +		rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
> +							brrqd[hole].addr->addr);
> +
> +		i++;
> +		hole = find_next_zero_bit(read_bitmap, nr_pages, hole + 1);
> +	} while (hole != nr_pages);
> +
> +	if (nr_holes != new_bio->bi_vcnt) {
> +		pr_err("rrpc: malformed bio\n");
> +		goto err;
> +	}
> +
> +	new_bio->bi_iter.bi_sector = bio->bi_iter.bi_sector;
> +	new_bio->bi_rw = READ;
> +	new_bio->bi_private = &wait;
> +	new_bio->bi_end_io = rrpc_end_sync_bio;
> +
> +	rqd->flags |= NVM_IOTYPE_SYNC;
> +	rqd->bio = new_bio;
> +	rqd->nr_pages = nr_holes;
> +
> +	rrpc_submit_read_io(rrpc, new_bio, rqd, rqd->flags);
> +	wait_for_completion_io(&wait);
> +
> +	if (new_bio->bi_error)
> +		goto err;
> +
> +	/* Fill the holes in the original bio */
> +	i = 0;
> +	hole = find_first_zero_bit(read_bitmap, nr_pages);
> +	do {
> +		src_bv = new_bio->bi_io_vec[i];
> +		dst_bv = bio->bi_io_vec[hole];
> +
> +		src_p = kmap_atomic(src_bv.bv_page);
> +		dst_p = kmap_atomic(dst_bv.bv_page);
> +
> +		memcpy(dst_p + dst_bv.bv_offset,
> +			src_p + src_bv.bv_offset,
> +			RRPC_EXPOSED_PAGE_SIZE);
> +
> +		kunmap_atomic(src_p);
> +		kunmap_atomic(dst_p);
> +
> +		mempool_free(&src_bv.bv_page, rrpc->page_pool);
> +
> +		i++;
> +		hole = find_next_zero_bit(read_bitmap, nr_pages, hole + 1);
> +	} while (hole != nr_pages);
> +
> +	bio_put(new_bio);
> +
> +	/* Complete the original bio and associated request */
> +	rqd->flags &= ~NVM_IOTYPE_SYNC;
> +	rqd->bio = bio;
> +	rqd->nr_pages = nr_pages;
> +
> +	bio_endio(bio);
> +	rrpc_end_io(rqd);
> +	return NVM_IO_OK;
> +
> +err:
> +	/* Free allocated pages in new bio */
> +	for (i = 0; i < new_bio->bi_vcnt; i++) {
> +		src_bv = new_bio->bi_io_vec[i];
> +		mempool_free(&src_bv.bv_page, rrpc->page_pool);
> +	}
> +	bio_endio(new_bio);
> +	return NVM_IO_ERR;
> +}
> +
>  static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
>  				struct rrpc_rq *rrqd, unsigned long flags)
>  {
>  	struct nvm_rq *rqd;
>  	struct rrpc_buf_rq brrqd[rrpc->max_write_pgs];
> +	unsigned long read_bitmap; /* Max 64 ppas per request */
> +	uint8_t left;
>  	uint8_t nr_pages = rrpc_get_pages(bio);
>  	int err;
>  
> +	bitmap_zero(&read_bitmap, nr_pages);
> +
>  	rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
>  	if (!rqd) {
>  		pr_err_ratelimited("rrpc: not able to queue bio.");
> @@ -1073,22 +1320,25 @@ static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
>  						&rqd->dma_ppa_list);
>  		if (!rqd->ppa_list) {
>  			pr_err("rrpc: not able to allocate ppa list\n");
> -			mempool_free(rrqd, rrpc->rrq_pool);
>  			mempool_free(rqd, rrpc->rq_pool);
> +			mempool_free(rrqd, rrpc->rrq_pool);
>  			return NVM_IO_ERR;
>  		}
>  
>  		err = rrpc_read_ppalist_rq(rrpc, bio, rqd, brrqd, flags,
>  								nr_pages);
>  		if (err) {
> -			mempool_free(rrqd, rrpc->rrq_pool);
>  			mempool_free(rqd, rrpc->rq_pool);
> +			mempool_free(rrqd, rrpc->rrq_pool);
>  			return err;
>  		}
>  	} else {
>  		err = rrpc_read_rq(rrpc, bio, rqd, flags);
> -		if (err)
> +		if (err) {
> +			mempool_free(rrqd, rrpc->rrq_pool);
> +			mempool_free(rqd, rrpc->rq_pool);
>  			return err;
> +		}
>  	}
>  
>  	bio_get(bio);
> @@ -1097,33 +1347,22 @@ static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
>  	rqd->nr_pages = rrqd->nr_pages = nr_pages;
>  	rqd->flags = flags;
>  
> -	err = nvm_submit_io(rrpc->dev, rqd);
> -	if (err) {
> -		pr_err("rrpc: I/O submission failed: %d\n", err);
> -		bio_put(bio);
> -		if (!(flags & NVM_IOTYPE_GC)) {
> -			rrpc_unlock_rq(rrpc, rrqd);
> -			if (rqd->nr_pages > 1)
> -				nvm_dev_dma_free(rrpc->dev,
> -			rqd->ppa_list, rqd->dma_ppa_list);
> -		}
> +	left = rrpc_read_from_w_buf(rrpc, rqd, brrqd, &read_bitmap);
> +	if (left == 0) {
> +		bio_endio(bio);
> +		rrpc_end_io(rqd);
> +		return NVM_IO_OK;
> +	} else if (left < 0)
>  		return NVM_IO_ERR;
> -	}
>  
> -	return NVM_IO_OK;
> -}
> +	if (bitmap_empty(&read_bitmap, nr_pages))
> +		return rrpc_submit_read_io(rrpc, bio, rqd, flags);
>  
> -static int rrpc_buffer_write(struct rrpc *rrpc, struct bio *bio,
> -				struct rrpc_rq *rrqd, unsigned long flags)
> -{
> -	uint8_t nr_pages = rrpc_get_pages(bio);
> -
> -	rrqd->nr_pages = nr_pages;
> -
> -	if (nr_pages > 1)
> -		return rrpc_write_ppalist_rq(rrpc, bio, rrqd, flags, nr_pages);
> -	else
> -		return rrpc_write_rq(rrpc, bio, rrqd, flags);
> +	/* The read bio could not be completely read from the write buffer. This
> +	 * case only occurs when several pages are sent in a single bio
> +	 */
> +	return rrpc_fill_partial_read_bio(rrpc, bio, &read_bitmap, rqd, brrqd,
> +								nr_pages);
>  }
>  
>  static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
> diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
> index eda9743..ae26ced 100644
> --- a/include/linux/lightnvm.h
> +++ b/include/linux/lightnvm.h
> @@ -11,6 +11,7 @@ enum {
>  
>  	NVM_IOTYPE_NONE = 0,
>  	NVM_IOTYPE_GC = 1,
> +	NVM_IOTYPE_SYNC = 2,
>  };
>  
>  #define NVM_BLK_BITS (16)
> 

Seems like this can be merged into the write buffer patch as well?

[toc] | [prev] | [next] | [standalone]

#1326780 — [RFC 1/4] lightnvm: precalculate controller write boundaries

Flash controllers typically define flash pages as a collection of flash
sectors of typically 4K. Moreover, flash controllers might program flash
pages across several planes. This defines the write granurality at which
flash can be programmed. This is different for each flash controller.

In order to simplify calculations, and avoid repeating them in a per-I/O
basis, this patch pre-calculates write granurality values as part of the
device characteristics in the bring up.

Signed-off-by: Javier González <javier@cnexlabs.com>
---
 drivers/lightnvm/rrpc.c | 4 ++++
 drivers/lightnvm/rrpc.h | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
index 775bf6c2..8187bf3 100644
--- a/drivers/lightnvm/rrpc.c
+++ b/drivers/lightnvm/rrpc.c
@@ -1149,6 +1149,10 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
 	if (!rrpc->luns)
 		return -ENOMEM;
 
+	rrpc->min_write_pgs = dev->sec_per_pl * (dev->sec_size / PAGE_SIZE);
+	/* assume max_phys_sect % dev->min_write_pgs == 0 */
+	rrpc->max_write_pgs = dev->ops->max_phys_sect;
+
 	/* 1:1 mapping */
 	for (i = 0; i < rrpc->nr_luns; i++) {
 		struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i);
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
index 3989d65..868e91a 100644
--- a/drivers/lightnvm/rrpc.h
+++ b/drivers/lightnvm/rrpc.h
@@ -107,6 +107,9 @@ struct rrpc {
 	unsigned long long nr_sects;
 	unsigned long total_blocks;
 
+	int min_write_pgs; /* minimum amount of pages required by controller */
+	int max_write_pgs; /* maximum amount of pages supported by controller */
+
 	/* Write strategy variables. Move these into each for structure for each
 	 * strategy
 	 */
-- 
2.1.4

[toc] | [prev] | [next] | [standalone]

#1327820 — Re: [RFC 1/4] lightnvm: precalculate controller write boundaries

On 02/04/2016 02:08 PM, Javier González wrote:
> Flash controllers typically define flash pages as a collection of flash
> sectors of typically 4K. Moreover, flash controllers might program flash
> pages across several planes. This defines the write granurality at which
> flash can be programmed. This is different for each flash controller.
> 
> In order to simplify calculations, and avoid repeating them in a per-I/O
> basis, this patch pre-calculates write granurality values as part of the
> device characteristics in the bring up.
> 
> Signed-off-by: Javier González <javier@cnexlabs.com>
> ---
>  drivers/lightnvm/rrpc.c | 4 ++++
>  drivers/lightnvm/rrpc.h | 3 +++
>  2 files changed, 7 insertions(+)
> 
> diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
> index 775bf6c2..8187bf3 100644
> --- a/drivers/lightnvm/rrpc.c
> +++ b/drivers/lightnvm/rrpc.c
> @@ -1149,6 +1149,10 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
>  	if (!rrpc->luns)
>  		return -ENOMEM;
>  
> +	rrpc->min_write_pgs = dev->sec_per_pl * (dev->sec_size / PAGE_SIZE);
> +	/* assume max_phys_sect % dev->min_write_pgs == 0 */
> +	rrpc->max_write_pgs = dev->ops->max_phys_sect;
> +
>  	/* 1:1 mapping */
>  	for (i = 0; i < rrpc->nr_luns; i++) {
>  		struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i);
> diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
> index 3989d65..868e91a 100644
> --- a/drivers/lightnvm/rrpc.h
> +++ b/drivers/lightnvm/rrpc.h
> @@ -107,6 +107,9 @@ struct rrpc {
>  	unsigned long long nr_sects;
>  	unsigned long total_blocks;
>  
> +	int min_write_pgs; /* minimum amount of pages required by controller */
> +	int max_write_pgs; /* maximum amount of pages supported by controller */
> +
>  	/* Write strategy variables. Move these into each for structure for each
>  	 * strategy
>  	 */
> 

This belongs to the write buffer patch.

[toc] | [prev] | [standalone]

Back to top | Article view | linux.kernel

csiph-web