summaryrefslogtreecommitdiffstats
path: root/block/blk-map.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/blk-map.c')
-rw-r--r--block/blk-map.c682
1 files changed, 682 insertions, 0 deletions
diff --git a/block/blk-map.c b/block/blk-map.c
new file mode 100644
index 000000000..9cb9605d1
--- /dev/null
+++ b/block/blk-map.c
@@ -0,0 +1,682 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Functions related to mapping data to requests
+ */
+#include <linux/kernel.h>
+#include <linux/sched/task_stack.h>
+#include <linux/module.h>
+#include <linux/bio.h>
+#include <linux/blkdev.h>
+#include <linux/uio.h>
+
+#include "blk.h"
+
+struct bio_map_data {
+ bool is_our_pages : 1;
+ bool is_null_mapped : 1;
+ struct iov_iter iter;
+ struct iovec iov[];
+};
+
+static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,
+ gfp_t gfp_mask)
+{
+ struct bio_map_data *bmd;
+
+ if (data->nr_segs > UIO_MAXIOV)
+ return NULL;
+
+ bmd = kmalloc_flex(*bmd, iov, data->nr_segs, gfp_mask);
+ if (!bmd)
+ return NULL;
+ bmd->iter = *data;
+ if (iter_is_iovec(data)) {
+ memcpy(bmd->iov, iter_iov(data), sizeof(struct iovec) * data->nr_segs);
+ bmd->iter.__iov = bmd->iov;
+ }
+ return bmd;
+}
+
+static inline void blk_mq_map_bio_put(struct bio *bio)
+{
+ bio_put(bio);
+}
+
+static struct bio *blk_rq_map_bio_alloc(struct request *rq,
+ unsigned int nr_vecs, gfp_t gfp_mask)
+{
+ struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL;
+ struct bio *bio;
+
+ bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask,
+ &fs_bio_set);
+ if (!bio)
+ return NULL;
+
+ return bio;
+}
+
+/**
+ * bio_copy_from_iter - copy all pages from iov_iter to bio
+ * @bio: The &struct bio which describes the I/O as destination
+ * @iter: iov_iter as source
+ *
+ * Copy all pages from iov_iter to bio.
+ * Returns 0 on success, or error on failure.
+ */
+static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
+{
+ struct bio_vec *bvec;
+ struct bvec_iter_all iter_all;
+
+ bio_for_each_segment_all(bvec, bio, iter_all) {
+ ssize_t ret;
+
+ ret = copy_page_from_iter(bvec->bv_page,
+ bvec->bv_offset,
+ bvec->bv_len,
+ iter);
+
+ if (!iov_iter_count(iter))
+ break;
+
+ if (ret < bvec->bv_len)
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+/**
+ * bio_copy_to_iter - copy all pages from bio to iov_iter
+ * @bio: The &struct bio which describes the I/O as source
+ * @iter: iov_iter as destination
+ *
+ * Copy all pages from bio to iov_iter.
+ * Returns 0 on success, or error on failure.
+ */
+static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
+{
+ struct bio_vec *bvec;
+ struct bvec_iter_all iter_all;
+
+ bio_for_each_segment_all(bvec, bio, iter_all) {
+ ssize_t ret;
+
+ ret = copy_page_to_iter(bvec->bv_page,
+ bvec->bv_offset,
+ bvec->bv_len,
+ &iter);
+
+ if (!iov_iter_count(&iter))
+ break;
+
+ if (ret < bvec->bv_len)
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+/**
+ * bio_uncopy_user - finish previously mapped bio
+ * @bio: bio being terminated
+ *
+ * Free pages allocated from bio_copy_user_iov() and write back data
+ * to user space in case of a read.
+ */
+static int bio_uncopy_user(struct bio *bio)
+{
+ struct bio_map_data *bmd = bio->bi_private;
+ int ret = 0;
+
+ if (!bmd->is_null_mapped) {
+ /*
+ * if we're in a workqueue, the request is orphaned, so
+ * don't copy into a random user address space, just free
+ * and return -EINTR so user space doesn't expect any data.
+ */
+ if (!current->mm)
+ ret = -EINTR;
+ else if (bio_data_dir(bio) == READ)
+ ret = bio_copy_to_iter(bio, bmd->iter);
+ if (bmd->is_our_pages)
+ bio_free_pages(bio);
+ }
+ kfree(bmd);
+ return ret;
+}
+
+static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data,
+ struct iov_iter *iter, gfp_t gfp_mask)
+{
+ struct bio_map_data *bmd;
+ struct page *page;
+ struct bio *bio;
+ int i = 0, ret;
+ int nr_pages;
+ unsigned int len = iter->count;
+ unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
+
+ bmd = bio_alloc_map_data(iter, gfp_mask);
+ if (!bmd)
+ return -ENOMEM;
+
+ /*
+ * We need to do a deep copy of the iov_iter including the iovecs.
+ * The caller provided iov might point to an on-stack or otherwise
+ * shortlived one.
+ */
+ bmd->is_our_pages = !map_data;
+ bmd->is_null_mapped = (map_data && map_data->null_mapped);
+
+ nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE));
+
+ ret = -ENOMEM;
+ bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask);
+ if (!bio)
+ goto out_bmd;
+
+ if (map_data) {
+ nr_pages = 1U << map_data->page_order;
+ i = map_data->offset / PAGE_SIZE;
+ }
+ while (len) {
+ unsigned int bytes = PAGE_SIZE;
+
+ bytes -= offset;
+
+ if (bytes > len)
+ bytes = len;
+
+ if (map_data) {
+ if (i == map_data->nr_entries * nr_pages) {
+ ret = -ENOMEM;
+ goto cleanup;
+ }
+
+ page = map_data->pages[i / nr_pages];
+ page += (i % nr_pages);
+
+ i++;
+ } else {
+ page = alloc_page(GFP_NOIO | gfp_mask);
+ if (!page) {
+ ret = -ENOMEM;
+ goto cleanup;
+ }
+ }
+
+ if (bio_add_page(bio, page, bytes, offset) < bytes) {
+ if (!map_data)
+ __free_page(page);
+ break;
+ }
+
+ len -= bytes;
+ offset = 0;
+ }
+
+ if (map_data)
+ map_data->offset += bio->bi_iter.bi_size;
+
+ /*
+ * success
+ */
+ if (iov_iter_rw(iter) == WRITE &&
+ (!map_data || !map_data->null_mapped)) {
+ ret = bio_copy_from_iter(bio, iter);
+ if (ret)
+ goto cleanup;
+ } else if (map_data && map_data->from_user) {
+ struct iov_iter iter2 = *iter;
+
+ /* This is the copy-in part of SG_DXFER_TO_FROM_DEV. */
+ iter2.data_source = ITER_SOURCE;
+ ret = bio_copy_from_iter(bio, &iter2);
+ if (ret)
+ goto cleanup;
+ } else {
+ if (bmd->is_our_pages)
+ zero_fill_bio(bio);
+ iov_iter_advance(iter, bio->bi_iter.bi_size);
+ }
+
+ bio->bi_private = bmd;
+
+ ret = blk_rq_append_bio(rq, bio);
+ if (ret)
+ goto cleanup;
+ return 0;
+cleanup:
+ if (!map_data)
+ bio_free_pages(bio);
+ blk_mq_map_bio_put(bio);
+out_bmd:
+ kfree(bmd);
+ return ret;
+}
+
+static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,
+ gfp_t gfp_mask)
+{
+ unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS);
+ struct bio *bio;
+ int ret;
+
+ if (!iov_iter_count(iter))
+ return -EINVAL;
+
+ bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask);
+ if (!bio)
+ return -ENOMEM;
+ /*
+ * No alignment requirements on our part to support arbitrary
+ * passthrough commands.
+ */
+ ret = bio_iov_iter_get_pages(bio, iter, 0, 0);
+ if (ret)
+ goto out_put;
+ ret = blk_rq_append_bio(rq, bio);
+ if (ret)
+ goto out_release;
+ return 0;
+
+out_release:
+ bio_release_pages(bio, false);
+out_put:
+ blk_mq_map_bio_put(bio);
+ return ret;
+}
+
+static void bio_invalidate_vmalloc_pages(struct bio *bio)
+{
+#ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE
+ if (bio->bi_private && !op_is_write(bio_op(bio))) {
+ unsigned long i, len = 0;
+
+ for (i = 0; i < bio->bi_vcnt; i++)
+ len += bio->bi_io_vec[i].bv_len;
+ invalidate_kernel_vmap_range(bio->bi_private, len);
+ }
+#endif
+}
+
+static void bio_map_kern_endio(struct bio *bio)
+{
+ bio_invalidate_vmalloc_pages(bio);
+ blk_mq_map_bio_put(bio);
+}
+
+static struct bio *bio_map_kern(struct request *rq, void *data, unsigned int len,
+ gfp_t gfp_mask)
+{
+ unsigned int nr_vecs = bio_add_max_vecs(data, len);
+ struct bio *bio;
+
+ bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask);
+ if (!bio)
+ return ERR_PTR(-ENOMEM);
+
+ if (is_vmalloc_addr(data)) {
+ bio->bi_private = data;
+ if (!bio_add_vmalloc(bio, data, len)) {
+ blk_mq_map_bio_put(bio);
+ return ERR_PTR(-EINVAL);
+ }
+ } else {
+ bio_add_virt_nofail(bio, data, len);
+ }
+ bio->bi_end_io = bio_map_kern_endio;
+ return bio;
+}
+
+static void bio_copy_kern_endio(struct bio *bio)
+{
+ bio_free_pages(bio);
+ blk_mq_map_bio_put(bio);
+}
+
+static void bio_copy_kern_endio_read(struct bio *bio)
+{
+ char *p = bio->bi_private;
+ struct bio_vec *bvec;
+ struct bvec_iter_all iter_all;
+
+ bio_for_each_segment_all(bvec, bio, iter_all) {
+ memcpy_from_bvec(p, bvec);
+ p += bvec->bv_len;
+ }
+
+ bio_copy_kern_endio(bio);
+}
+
+/**
+ * bio_copy_kern - copy kernel address into bio
+ * @rq: request to fill
+ * @data: pointer to buffer to copy
+ * @len: length in bytes
+ * @op: bio/request operation
+ * @gfp_mask: allocation flags for bio and page allocation
+ *
+ * copy the kernel address into a bio suitable for io to a block
+ * device. Returns an error pointer in case of error.
+ */
+static struct bio *bio_copy_kern(struct request *rq, void *data, unsigned int len,
+ gfp_t gfp_mask)
+{
+ enum req_op op = req_op(rq);
+ unsigned long kaddr = (unsigned long)data;
+ unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ unsigned long start = kaddr >> PAGE_SHIFT;
+ struct bio *bio;
+ void *p = data;
+ int nr_pages = 0;
+
+ /*
+ * Overflow, abort
+ */
+ if (end < start)
+ return ERR_PTR(-EINVAL);
+
+ nr_pages = end - start;
+ bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask);
+ if (!bio)
+ return ERR_PTR(-ENOMEM);
+
+ while (len) {
+ struct page *page;
+ unsigned int bytes = PAGE_SIZE;
+
+ if (bytes > len)
+ bytes = len;
+
+ page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask);
+ if (!page)
+ goto cleanup;
+
+ if (op_is_write(op))
+ memcpy(page_address(page), p, bytes);
+
+ __bio_add_page(bio, page, bytes, 0);
+
+ len -= bytes;
+ p += bytes;
+ }
+
+ if (op_is_write(op)) {
+ bio->bi_end_io = bio_copy_kern_endio;
+ } else {
+ bio->bi_end_io = bio_copy_kern_endio_read;
+ bio->bi_private = data;
+ }
+
+ return bio;
+
+cleanup:
+ bio_free_pages(bio);
+ blk_mq_map_bio_put(bio);
+ return ERR_PTR(-ENOMEM);
+}
+
+/*
+ * Append a bio to a passthrough request. Only works if the bio can be merged
+ * into the request based on the driver constraints.
+ */
+int blk_rq_append_bio(struct request *rq, struct bio *bio)
+{
+ const struct queue_limits *lim = &rq->q->limits;
+ unsigned int max_bytes = lim->max_hw_sectors << SECTOR_SHIFT;
+ unsigned int nr_segs = 0;
+ int ret;
+
+ /* check that the data layout matches the hardware restrictions */
+ ret = bio_split_io_at(bio, lim, &nr_segs, max_bytes, 0);
+ if (ret) {
+ /* if we would have to split the bio, copy instead */
+ if (ret > 0)
+ ret = -EREMOTEIO;
+ return ret;
+ }
+
+ if (rq->bio) {
+ if (!ll_back_merge_fn(rq, bio, nr_segs))
+ return -EINVAL;
+ rq->phys_gap_bit = bio_seg_gap(rq->q, rq->biotail, bio,
+ rq->phys_gap_bit);
+ rq->biotail->bi_next = bio;
+ rq->biotail = bio;
+ rq->__data_len += bio->bi_iter.bi_size;
+ bio_crypt_free_ctx(bio);
+ return 0;
+ }
+
+ rq->nr_phys_segments = nr_segs;
+ rq->bio = rq->biotail = bio;
+ rq->__data_len = bio->bi_iter.bi_size;
+ rq->phys_gap_bit = bio->bi_bvec_gap_bit;
+ return 0;
+}
+EXPORT_SYMBOL(blk_rq_append_bio);
+
+/* Prepare bio for passthrough IO given ITER_BVEC iter */
+static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter)
+{
+ unsigned int max_bytes = rq->q->limits.max_hw_sectors << SECTOR_SHIFT;
+ struct bio *bio;
+ int ret;
+
+ if (!iov_iter_count(iter) || iov_iter_count(iter) > max_bytes)
+ return -EINVAL;
+
+ /* reuse the bvecs from the iterator instead of allocating new ones */
+ bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL);
+ if (!bio)
+ return -ENOMEM;
+ bio_iov_iter_set(bio, iter);
+
+ ret = blk_rq_append_bio(rq, bio);
+ if (ret)
+ blk_mq_map_bio_put(bio);
+ return ret;
+}
+
+/**
+ * blk_rq_map_user_iov - map user data to a request, for passthrough requests
+ * @q: request queue where request should be inserted
+ * @rq: request to map data to
+ * @map_data: pointer to the rq_map_data holding pages (if necessary)
+ * @iter: iovec iterator
+ * @gfp_mask: memory allocation flags
+ *
+ * Description:
+ * Data will be mapped directly for zero copy I/O, if possible. Otherwise
+ * a kernel bounce buffer is used.
+ *
+ * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
+ * still in process context.
+ */
+int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
+ struct rq_map_data *map_data,
+ const struct iov_iter *iter, gfp_t gfp_mask)
+{
+ bool copy = false, map_bvec = false;
+ unsigned long align = blk_lim_dma_alignment_and_pad(&q->limits);
+ struct bio *bio = NULL;
+ struct iov_iter i;
+ int ret = -EINVAL;
+
+ if (map_data)
+ copy = true;
+ else if (iov_iter_alignment(iter) & align)
+ copy = true;
+ else if (iov_iter_is_bvec(iter))
+ map_bvec = true;
+ else if (!user_backed_iter(iter))
+ copy = true;
+ else if (queue_virt_boundary(q))
+ copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter);
+
+ if (map_bvec) {
+ ret = blk_rq_map_user_bvec(rq, iter);
+ if (!ret)
+ return 0;
+ if (ret != -EREMOTEIO)
+ goto fail;
+ /* fall back to copying the data on limits mismatches */
+ copy = true;
+ }
+
+ i = *iter;
+ do {
+ if (copy)
+ ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask);
+ else
+ ret = bio_map_user_iov(rq, &i, gfp_mask);
+ if (ret) {
+ if (ret == -EREMOTEIO)
+ ret = -EINVAL;
+ goto unmap_rq;
+ }
+ if (!bio)
+ bio = rq->bio;
+ } while (iov_iter_count(&i));
+
+ return 0;
+
+unmap_rq:
+ blk_rq_unmap_user(bio);
+fail:
+ rq->bio = NULL;
+ return ret;
+}
+EXPORT_SYMBOL(blk_rq_map_user_iov);
+
+int blk_rq_map_user(struct request_queue *q, struct request *rq,
+ struct rq_map_data *map_data, void __user *ubuf,
+ unsigned long len, gfp_t gfp_mask)
+{
+ struct iov_iter i;
+ int ret = import_ubuf(rq_data_dir(rq), ubuf, len, &i);
+
+ if (unlikely(ret < 0))
+ return ret;
+
+ return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask);
+}
+EXPORT_SYMBOL(blk_rq_map_user);
+
+int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data,
+ void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask,
+ bool vec, int iov_count, bool check_iter_count, int rw)
+{
+ int ret = 0;
+
+ if (vec) {
+ struct iovec fast_iov[UIO_FASTIOV];
+ struct iovec *iov = fast_iov;
+ struct iov_iter iter;
+
+ ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len,
+ UIO_FASTIOV, &iov, &iter);
+ if (ret < 0)
+ return ret;
+
+ if (iov_count) {
+ /* SG_IO howto says that the shorter of the two wins */
+ iov_iter_truncate(&iter, buf_len);
+ if (check_iter_count && !iov_iter_count(&iter)) {
+ kfree(iov);
+ return -EINVAL;
+ }
+ }
+
+ ret = blk_rq_map_user_iov(req->q, req, map_data, &iter,
+ gfp_mask);
+ kfree(iov);
+ } else if (buf_len) {
+ ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len,
+ gfp_mask);
+ }
+ return ret;
+}
+EXPORT_SYMBOL(blk_rq_map_user_io);
+
+/**
+ * blk_rq_unmap_user - unmap a request with user data
+ * @bio: start of bio list
+ *
+ * Description:
+ * Unmap a rq previously mapped by blk_rq_map_user(). The caller must
+ * supply the original rq->bio from the blk_rq_map_user() return, since
+ * the I/O completion may have changed rq->bio.
+ */
+int blk_rq_unmap_user(struct bio *bio)
+{
+ struct bio *next_bio;
+ int ret = 0, ret2;
+
+ while (bio) {
+ if (bio->bi_private) {
+ ret2 = bio_uncopy_user(bio);
+ if (ret2 && !ret)
+ ret = ret2;
+ } else {
+ bio_release_pages(bio, bio_data_dir(bio) == READ);
+ }
+
+ if (bio_integrity(bio))
+ bio_integrity_unmap_user(bio);
+
+ next_bio = bio;
+ bio = bio->bi_next;
+ blk_mq_map_bio_put(next_bio);
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(blk_rq_unmap_user);
+
+/**
+ * blk_rq_map_kern - map kernel data to a request, for passthrough requests
+ * @rq: request to fill
+ * @kbuf: the kernel buffer
+ * @len: length of user data
+ * @gfp_mask: memory allocation flags
+ *
+ * Description:
+ * Data will be mapped directly if possible. Otherwise a bounce
+ * buffer is used. Can be called multiple times to append multiple
+ * buffers.
+ */
+int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
+ gfp_t gfp_mask)
+{
+ unsigned long addr = (unsigned long) kbuf;
+ bool do_copy;
+ struct bio *bio;
+ int ret;
+
+ if (len > (queue_max_hw_sectors(rq->q) << SECTOR_SHIFT))
+ return -EINVAL;
+ if (!len || !kbuf)
+ return -EINVAL;
+
+ do_copy = !blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf);
+ if (do_copy)
+ bio = bio_copy_kern(rq, kbuf, len, gfp_mask);
+ else
+ bio = bio_map_kern(rq, kbuf, len, gfp_mask);
+
+ if (IS_ERR(bio))
+ return PTR_ERR(bio);
+
+ ret = blk_rq_append_bio(rq, bio);
+ if (unlikely(ret)) {
+ if (do_copy)
+ bio_free_pages(bio);
+ blk_mq_map_bio_put(bio);
+ }
+ return ret;
+}
+EXPORT_SYMBOL(blk_rq_map_kern);
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-10 10:13:05 +0000