aboutsummaryrefslogtreecommitdiff
path: root/fs
diff options
context:
space:
mode:
Diffstat (limited to 'fs')
-rw-r--r--fs/fuse/args.h2
-rw-r--r--fs/fuse/cuse.c8
-rw-r--r--fs/fuse/dev.c104
-rw-r--r--fs/fuse/dev.h2
-rw-r--r--fs/fuse/dev_uring.c633
-rw-r--r--fs/fuse/dev_uring_i.h56
-rw-r--r--fs/fuse/dir.c37
-rw-r--r--fs/fuse/file.c20
-rw-r--r--fs/fuse/fuse_dev_i.h5
-rw-r--r--fs/fuse/fuse_i.h2
-rw-r--r--fs/fuse/inode.c16
-rw-r--r--fs/fuse/readdir.c2
-rw-r--r--fs/fuse/req_timeout.c12
13 files changed, 734 insertions, 165 deletions
diff --git a/fs/fuse/args.h b/fs/fuse/args.h
index ecfe51a192af..5173264a1261 100644
--- a/fs/fuse/args.h
+++ b/fs/fuse/args.h
@@ -42,6 +42,8 @@ struct fuse_args {
bool is_pinned:1;
bool invalidate_vmap:1;
bool abort_on_kill:1;
+ /* server requested io-uring zero-copy for this op */
+ bool zero_copy:1;
struct fuse_in_arg in_args[4];
struct fuse_arg out_args[2];
void (*end)(struct fuse_args *args, int error);
diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c
index 3c15b5ba16d7..4079cf8e5974 100644
--- a/fs/fuse/cuse.c
+++ b/fs/fuse/cuse.c
@@ -530,7 +530,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
INIT_LIST_HEAD(&cc->list);
- cc->fc.chan->initialized = 1;
+ /* Pairs with smp_load_acquire() readers of fch->initialized */
+ smp_store_release(&cc->fc.chan->initialized, 1);
rc = cuse_send_init(cc);
if (rc) {
fuse_dev_put(fud);
@@ -653,6 +654,11 @@ static void __exit cuse_exit(void)
{
misc_deregister(&cuse_miscdev);
class_destroy(cuse_class);
+ /*
+ * Wait for pending call_rcu() callbacks that call back into
+ * this module via fc->release (cuse_fc_release).
+ */
+ rcu_barrier();
}
module_init(cuse_init);
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 5763a7cd3b37..4fec31fc0b84 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -75,17 +75,23 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa
fch->minor = param->minor;
fch->max_write = param->max_write;
fch->max_pages = param->max_pages;
+
+ if (param->io_uring_enabled)
+ fuse_uring_conn_init(fch);
}
- /* Make sure stores before this are seen on another CPU */
- smp_wmb();
- fch->initialized = 1;
+ /* Pairs with smp_load_acquire() readers of fch->initialized */
+ smp_store_release(&fch->initialized, 1);
wake_up_all(&fch->blocked_waitq);
}
static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background)
{
- return !fch->initialized || (for_background && fch->blocked) ||
+ /* Pairs with smp_store_release() in fuse_chan_set_initialized() */
+ if (!smp_load_acquire(&fch->initialized))
+ return true;
+
+ return (for_background && fch->blocked) ||
(fch->io_uring && fch->connected && !fuse_uring_ready(fch));
}
@@ -120,9 +126,6 @@ static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background)
goto out;
}
- /* Matches smp_wmb() in fuse_chan_set_initialized() */
- smp_rmb();
-
err = -ENOTCONN;
if (!fch->connected)
goto out;
@@ -210,10 +213,13 @@ EXPORT_SYMBOL_GPL(fuse_req_hash);
/*
* A new request is available, wake fiq->waitq
*/
-static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
+static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync)
__releases(fiq->lock)
{
- wake_up(&fiq->waitq);
+ if (sync)
+ wake_up_sync(&fiq->waitq);
+ else
+ wake_up(&fiq->waitq);
kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
spin_unlock(&fiq->lock);
}
@@ -230,7 +236,7 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
if (fiq->connected) {
fiq->forget_list_tail->next = forget;
fiq->forget_list_tail = forget;
- fuse_dev_wake_and_unlock(fiq);
+ fuse_dev_wake_and_unlock(fiq, false);
} else {
kfree(forget);
spin_unlock(&fiq->lock);
@@ -240,7 +246,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
{
spin_lock(&fiq->lock);
- if (list_empty(&req->intr_entry)) {
+ /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */
+ if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) {
list_add_tail(&req->intr_entry, &fiq->interrupts);
/*
* Pairs with smp_mb() implied by test_and_set_bit()
@@ -251,7 +258,7 @@ void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
list_del_init(&req->intr_entry);
spin_unlock(&fiq->lock);
} else {
- fuse_dev_wake_and_unlock(fiq);
+ fuse_dev_wake_and_unlock(fiq, false);
}
} else {
spin_unlock(&fiq->lock);
@@ -281,11 +288,13 @@ EXPORT_SYMBOL_GPL(fuse_request_assign_unique);
static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req)
{
+ bool sync = test_and_clear_bit(FR_SYNC_WAKEUP, &req->flags);
+
spin_lock(&fiq->lock);
if (fiq->connected) {
fuse_request_assign_unique_locked(fiq, req);
list_add_tail(&req->list, &fiq->pending);
- fuse_dev_wake_and_unlock(fiq);
+ fuse_dev_wake_and_unlock(fiq, sync);
} else {
spin_unlock(&fiq->lock);
req->out.h.error = -ENOTCONN;
@@ -397,7 +406,8 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val)
fch->max_background = val;
fch->blocked = fch->num_background >= fch->max_background;
if (!fch->blocked)
- wake_up(&fch->blocked_waitq);
+ wake_up_nr(&fch->blocked_waitq,
+ fch->max_background - fch->num_background);
spin_unlock(&fch->bg_lock);
}
@@ -411,11 +421,6 @@ void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc)
fch->conn = fc;
}
-void fuse_chan_io_uring_enable(struct fuse_chan *fch)
-{
- fch->io_uring = 1;
-}
-
void fuse_pqueue_init(struct fuse_pqueue *fpq)
{
spin_lock_init(&fpq->lock);
@@ -725,7 +730,7 @@ static void request_wait_answer(struct fuse_req *req)
if (req->args->abort_on_kill) {
fuse_chan_abort(fch, false);
- return;
+ goto wait_for_finish;
}
if (test_bit(FR_URING, &req->flags))
@@ -736,6 +741,7 @@ static void request_wait_answer(struct fuse_req *req)
return;
}
+wait_for_finish:
/*
* Either request is already in userspace, or it was forced.
* Wait it out.
@@ -752,6 +758,11 @@ static void __fuse_request_send(struct fuse_req *req)
/* acquire extra reference, since request is still needed after
fuse_request_end() */
__fuse_get_request(req);
+ /*
+ * This is a synchronous request: the caller will block waiting for
+ * the answer. Hint the scheduler via wake_up_sync().
+ */
+ set_bit(FR_SYNC_WAKEUP, &req->flags);
fuse_send_one(fiq, req);
request_wait_answer(req);
@@ -1249,11 +1260,25 @@ int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
if (folio) {
size = folio_size(folio);
- if (zeroing && count < size)
- folio_zero_range(folio, 0, size);
+ if (zeroing && count < size) {
+ /*
+ * When the copy is skipped the folio already holds the
+ * payload, so only the bytes outside [offset, offset +
+ * count) may be zeroed.
+ *
+ * Otherwise, the whole folio is cleared first so that a
+ * failed copy leaves zeros rather than stale folio
+ * contents.
+ */
+ if (cs->skip_folio_copy)
+ folio_zero_segments(folio, 0, offset,
+ offset + count, size);
+ else
+ folio_zero_range(folio, 0, size);
+ }
}
- while (count) {
+ while (!cs->skip_folio_copy && count) {
if (cs->write && cs->pipebufs && folio) {
/*
* Can't control lifetime of pipe buffers, so always
@@ -1346,6 +1371,10 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
for (i = 0; !err && i < numargs; i++) {
struct fuse_arg *arg = &args[i];
if (i == numargs - 1 && argpages)
+ /*
+ * if cs->skip_folio_copy is set, this just does any
+ * needed zeroing. No copying is involved.
+ */
err = fuse_copy_folios(cs, arg->size, zeroing);
else
err = fuse_copy_one(cs, arg->value, arg->size);
@@ -1760,7 +1789,7 @@ out:
void fuse_chan_resend(struct fuse_chan *fch)
{
struct fuse_dev *fud;
- struct fuse_req *req, *next;
+ struct fuse_req *req;
struct fuse_iqueue *fiq = &fch->iq;
LIST_HEAD(to_queue);
unsigned int i;
@@ -1775,24 +1804,20 @@ void fuse_chan_resend(struct fuse_chan *fch)
struct fuse_pqueue *fpq = &fud->pq;
spin_lock(&fpq->lock);
- for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
- list_splice_tail_init(&fpq->processing[i], &to_queue);
+ for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) {
+ struct list_head *this_queue = &fpq->processing[i];
+
+ list_for_each_entry(req, this_queue, list)
+ clear_bit(FR_SENT, &req->flags);
+ list_splice_tail_init(this_queue, &to_queue);
+ }
spin_unlock(&fpq->lock);
}
spin_unlock(&fch->lock);
- list_for_each_entry_safe(req, next, &to_queue, list) {
- set_bit(FR_PENDING, &req->flags);
- clear_bit(FR_SENT, &req->flags);
- /* mark the request as resend request */
- req->in.h.unique |= FUSE_UNIQUE_RESEND;
- }
-
spin_lock(&fiq->lock);
if (!fiq->connected) {
spin_unlock(&fiq->lock);
- list_for_each_entry(req, &to_queue, list)
- clear_bit(FR_PENDING, &req->flags);
fuse_dev_end_requests(&to_queue);
return;
}
@@ -1801,12 +1826,16 @@ void fuse_chan_resend(struct fuse_chan *fch)
* intr_entry on fiq->interrupts after the request is re-queued.
*/
list_for_each_entry(req, &to_queue, list) {
+ set_bit(FR_PENDING, &req->flags);
+ /* mark the request as resend request */
+ req->in.h.unique |= FUSE_UNIQUE_RESEND;
+
if (test_bit(FR_INTERRUPTED, &req->flags))
list_del_init(&req->intr_entry);
}
/* iq and pq requests are both oldest to newest */
list_splice(&to_queue, &fiq->pending);
- fuse_dev_wake_and_unlock(fiq);
+ fuse_dev_wake_and_unlock(fiq, false);
}
/* Look up request on processing list by unique ID */
@@ -1888,7 +1917,8 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
* initialized and connected state
*/
err = -EINVAL;
- if (!fch->initialized || !fch->connected)
+ /* Pairs with smp_store_release() in fuse_chan_set_initialized() */
+ if (!smp_load_acquire(&fch->initialized) || !fch->connected)
goto copy_finish;
/* Don't try to move folios (yet) */
diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h
index aed69fd14c41..8d25378c0918 100644
--- a/fs/fuse/dev.h
+++ b/fs/fuse/dev.h
@@ -22,6 +22,7 @@ struct fuse_chan_param {
unsigned int minor;
unsigned int max_write;
unsigned int max_pages;
+ bool io_uring_enabled;
};
struct fuse_chan *fuse_chan_new(void);
@@ -34,7 +35,6 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val);
unsigned int fuse_chan_num_waiting(struct fuse_chan *fch);
void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc);
void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param);
-void fuse_chan_io_uring_enable(struct fuse_chan *fch);
ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args);
int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags);
int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique);
diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index 77c8cec43d9c..c6dd420c4034 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -9,6 +9,7 @@
#include "dev_uring_i.h"
#include "fuse_trace.h"
+#include <linux/bitmap.h>
#include <linux/fs.h>
#include <linux/io_uring/cmd.h>
@@ -21,6 +22,8 @@ MODULE_PARM_DESC(enable_uring,
#define FUSE_URING_IOV_HEADERS 0
#define FUSE_URING_IOV_PAYLOAD 1
+#define FUSE_URING_ADD_QUEUE_FLAGS (FUSE_URING_ZERO_COPY)
+
bool fuse_uring_enabled(void)
{
return enable_uring;
@@ -30,6 +33,11 @@ struct fuse_uring_pdu {
struct fuse_ring_ent *ent;
};
+struct fuse_zero_copy_bvs {
+ unsigned int nr_bvs;
+ struct bio_vec bvs[];
+};
+
static const struct fuse_iqueue_ops fuse_io_uring_ops;
enum fuse_uring_header_type {
@@ -41,6 +49,32 @@ enum fuse_uring_header_type {
FUSE_URING_HEADER_RING_ENT,
};
+static inline bool bufpool_enabled(struct fuse_ring_queue *queue)
+{
+ return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL;
+}
+
+static inline bool bufpool_registered(struct fuse_ring_queue *queue)
+{
+ return queue->bufpool && queue->bufpool->registered;
+}
+
+/*
+ * For a registered bufpool, every sqe that drives a payload import (REGISTER,
+ * COMMIT_AND_FETCH) must carry the registered buffer index of the pool.
+ * This also must be called from the command's issue handler, where cmd->sqe is
+ * still valid
+ */
+static inline bool fuse_uring_cmd_index_ok(struct io_uring_cmd *cmd,
+ struct fuse_ring_queue *queue)
+{
+ if (!bufpool_registered(queue))
+ return true;
+
+ return (cmd->flags & IORING_URING_CMD_FIXED) &&
+ READ_ONCE(cmd->sqe->buf_index) == queue->bufpool->registered_index;
+}
+
static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,
struct fuse_ring_ent *ring_ent)
{
@@ -86,8 +120,36 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)
}
}
+static bool can_zero_copy_req(struct fuse_ring_ent *ent, struct fuse_req *req)
+{
+ struct fuse_args *args = req->args;
+
+ if (!ent->queue->zero_copy || !args->zero_copy)
+ return false;
+
+ if (args->opcode != FUSE_READ && args->opcode != FUSE_WRITE)
+ return false;
+
+ return args->in_pages || args->out_pages;
+}
+
+static void zero_copy_unregister(struct io_uring_cmd *cmd,
+ struct fuse_ring_ent *ent,
+ unsigned int issue_flags)
+{
+ if (ent->zero_copied) {
+ int err = io_buffer_unregister(cmd, ent->zero_copy_index,
+ issue_flags);
+
+ if (err)
+ pr_warn_ratelimited("qid=%d zero-copy unregister failed: %d\n",
+ ent->queue->qid, err);
+ ent->zero_copied = false;
+ }
+}
+
static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
- int error)
+ int error, unsigned int issue_flags)
{
struct fuse_ring_queue *queue = ent->queue;
struct fuse_ring *ring = queue->ring;
@@ -107,6 +169,8 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
spin_unlock(&queue->lock);
+ zero_copy_unregister(ent->cmd, ent, issue_flags);
+
if (error)
req->out.h.error = error;
@@ -204,7 +268,7 @@ void fuse_uring_destruct(struct fuse_chan *fch)
return;
for (qid = 0; qid < ring->nr_queues; qid++) {
- struct fuse_ring_queue *queue = ring->queues[qid];
+ struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);
struct fuse_ring_ent *ent, *next;
if (!queue)
@@ -222,8 +286,9 @@ void fuse_uring_destruct(struct fuse_chan *fch)
}
kfree(queue->fpq.processing);
+ kfree(queue->bufpool);
kfree(queue);
- ring->queues[qid] = NULL;
+ WRITE_ONCE(ring->queues[qid], NULL);
}
kfree(ring->queues);
@@ -238,7 +303,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch)
{
struct fuse_ring *ring;
size_t nr_queues = num_possible_cpus();
- struct fuse_ring *res = NULL;
size_t max_payload_size;
ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT);
@@ -258,12 +322,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch)
spin_unlock(&fch->lock);
goto out_err;
}
- if (fch->ring) {
- /* race, another thread created the ring in the meantime */
- spin_unlock(&fch->lock);
- res = fch->ring;
- goto out_err;
- }
init_waitqueue_head(&ring->stop_waitq);
@@ -278,11 +336,18 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch)
out_err:
kfree(ring->queues);
kfree(ring);
- return res;
+ return NULL;
+}
+
+void fuse_uring_conn_init(struct fuse_chan *fch)
+{
+ if (fuse_uring_create(fch))
+ fch->io_uring = 1;
}
static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
- int qid)
+ int qid, bool zero_copy,
+ bool fail_if_exists)
{
struct fuse_chan *fch = ring->chan;
struct fuse_ring_queue *queue;
@@ -290,16 +355,17 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT);
if (!queue)
- return NULL;
+ return ERR_PTR(-ENOMEM);
pq = fuse_pqueue_alloc();
if (!pq) {
kfree(queue);
- return NULL;
+ return ERR_PTR(-ENOMEM);
}
queue->qid = qid;
queue->ring = ring;
spin_lock_init(&queue->lock);
+ queue->zero_copy = zero_copy;
INIT_LIST_HEAD(&queue->ent_avail_queue);
INIT_LIST_HEAD(&queue->ent_commit_queue);
@@ -316,14 +382,17 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
if (ring->queues[qid]) {
spin_unlock(&fch->lock);
kfree(queue->fpq.processing);
+ kfree(queue->bufpool);
kfree(queue);
- return ring->queues[qid];
+ return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid];
}
/*
- * write_once and lock as the caller mostly doesn't take the lock at all
+ * fch->lock serializes concurrent creators for this qid.
+ * smp_store_release() are for the lockless readers who must see a
+ * fully initialized queue after &ring->queues[qid] is set
*/
- WRITE_ONCE(ring->queues[qid], queue);
+ smp_store_release(&ring->queues[qid], queue);
spin_unlock(&fch->lock);
return queue;
@@ -434,7 +503,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring)
struct fuse_ring_ent *ent;
for (qid = 0; qid < ring->nr_queues; qid++) {
- struct fuse_ring_queue *queue = ring->queues[qid];
+ struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);
if (!queue)
continue;
@@ -643,30 +712,57 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent,
return 0;
}
+static int fuse_uring_import_payload(struct fuse_ring_ent *ent, int dir,
+ struct iov_iter *iter,
+ unsigned int issue_flags)
+{
+ void __user *base = ent->payload.iov_base;
+ size_t len = ent->payload.iov_len;
+ int err = 0;
+
+ if (!base) {
+ memset(iter, 0, sizeof(*iter));
+ return 0;
+ }
+
+ if (bufpool_registered(ent->queue))
+ err = io_uring_cmd_import_fixed((u64)(uintptr_t)base, len, dir,
+ iter, ent->cmd, issue_flags);
+ else
+ err = import_ubuf(dir, base, len, iter);
+
+ if (err)
+ pr_info_ratelimited("fuse: Import of user buffer failed\n");
+
+ return err;
+}
+
static int setup_fuse_copy_state(struct fuse_copy_state *cs,
- struct fuse_ring *ring, struct fuse_req *req,
+ struct fuse_req *req,
struct fuse_ring_ent *ent, int dir,
- struct iov_iter *iter)
+ struct iov_iter *iter,
+ unsigned int issue_flags)
{
int err;
- err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter);
- if (err) {
- pr_info_ratelimited("fuse: Import of user buffer failed\n");
+ err = fuse_uring_import_payload(ent, dir, iter, issue_flags);
+ if (err)
return err;
- }
fuse_copy_init(cs, dir == ITER_DEST, iter);
+ if (ent->zero_copied)
+ cs->skip_folio_copy = true;
+
cs->is_uring = true;
cs->req = req;
return 0;
}
-static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
- struct fuse_req *req,
- struct fuse_ring_ent *ent)
+static int fuse_uring_copy_from_ring(struct fuse_req *req,
+ struct fuse_ring_ent *ent,
+ unsigned int issue_flags)
{
struct fuse_copy_state cs;
struct fuse_args *args = req->args;
@@ -679,7 +775,8 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
if (err)
return err;
- err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter);
+ err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter,
+ issue_flags);
if (err)
return err;
@@ -688,11 +785,68 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
return err;
}
+static void fuse_zero_copy_release(void *priv)
+{
+ struct fuse_zero_copy_bvs *zc_bvs = priv;
+ unsigned int i;
+
+ for (i = 0; i < zc_bvs->nr_bvs; i++)
+ folio_put(page_folio(zc_bvs->bvs[i].bv_page));
+
+ kvfree(zc_bvs);
+}
+
+static int fuse_uring_set_up_zero_copy(struct fuse_ring_ent *ent,
+ struct fuse_req *req,
+ unsigned int issue_flags)
+{
+ struct fuse_args_pages *ap;
+ int err, i, ddir = 0;
+ struct fuse_zero_copy_bvs *zc_bvs;
+ struct bio_vec *bvs;
+
+ /* out_pages indicates a read, in_pages indicates a write */
+ if (req->args->out_pages)
+ ddir |= IO_BUF_DEST;
+ if (req->args->in_pages)
+ ddir |= IO_BUF_SOURCE;
+
+ ap = container_of(req->args, typeof(*ap), args);
+
+ zc_bvs = kvmalloc_flex(*zc_bvs, bvs, ap->num_folios,
+ GFP_KERNEL_ACCOUNT);
+ if (!zc_bvs)
+ return -ENOMEM;
+
+ zc_bvs->nr_bvs = ap->num_folios;
+ bvs = zc_bvs->bvs;
+ for (i = 0; i < ap->num_folios; i++) {
+ bvs[i].bv_page = folio_page(ap->folios[i], 0);
+ bvs[i].bv_offset = ap->descs[i].offset;
+ bvs[i].bv_len = ap->descs[i].length;
+ folio_get(ap->folios[i]);
+ }
+
+ err = io_buffer_register_bvec(ent->cmd, bvs, ap->num_folios,
+ fuse_zero_copy_release, zc_bvs,
+ ddir, ent->zero_copy_index,
+ issue_flags);
+ if (err) {
+ fuse_zero_copy_release(zc_bvs);
+ return err;
+ }
+
+ ent->zero_copied = true;
+
+ return 0;
+}
+
/*
* Copy data from the req to the ring buffer
*/
-static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
- struct fuse_ring_ent *ent)
+static int fuse_uring_args_to_ring(struct fuse_req *req,
+ struct fuse_ring_ent *ent,
+ unsigned int issue_flags)
{
struct fuse_copy_state cs;
struct fuse_args *args = req->args;
@@ -705,7 +859,15 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
.commit_id = req->in.h.unique,
};
- err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter);
+ if (can_zero_copy_req(ent, req)) {
+ ent_in_out.flags |= FUSE_URING_ENT_ZERO_COPY;
+ err = fuse_uring_set_up_zero_copy(ent, req, issue_flags);
+ if (err)
+ return err;
+ }
+
+ err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter,
+ issue_flags);
if (err)
return err;
@@ -735,15 +897,32 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
}
ent_in_out.payload_sz = cs.ring.copied_sz;
+ /*
+ * on a zero-copied write the pages are registered for the server to
+ * read via a fixed-buffer op rather than copied into the payload
+ * buffer, so copied_sz does not account for it. The server still needs
+ * the total inbound size to know how many bytes to read from the
+ * registered buffer, so add the page arg (always the last in-arg) back
+ * in
+ */
+ if (cs.skip_folio_copy && args->in_pages)
+ ent_in_out.payload_sz +=
+ args->in_args[args->in_numargs - 1].size;
+
+ if (bufpool_enabled(ent->queue) && ent->payload.iov_base)
+ ent_in_out.offset =
+ (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr;
+
return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT,
&ent_in_out, sizeof(ent_in_out));
}
static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent,
- struct fuse_req *req)
+ struct fuse_req *req,
+ unsigned int issue_flags)
{
struct fuse_ring_queue *queue = ent->queue;
- struct fuse_ring *ring = queue->ring;
+ struct fuse_in_header in_header;
int err;
err = -EIO;
@@ -758,23 +937,124 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent,
return err;
/* copy the request */
- err = fuse_uring_args_to_ring(ring, req, ent);
+ err = fuse_uring_args_to_ring(req, ent, issue_flags);
if (unlikely(err)) {
pr_info_ratelimited("Copy to ring failed: %d\n", err);
return err;
}
/* copy fuse_in_header */
- return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h,
- sizeof(req->in.h));
+ in_header = req->in.h;
+ return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &in_header,
+ sizeof(in_header));
+}
+
+static bool fuse_uring_req_has_copyable_payload(struct fuse_ring_ent *ent,
+ struct fuse_req *req)
+{
+ struct fuse_args *args = req->args;
+
+ if (!can_zero_copy_req(ent, req))
+ return args->in_numargs > 1 || args->out_numargs;
+
+ /*
+ * the asymmetry between in_numargs > 2 and out_numargs > 1 is because
+ * the per-op header is extracted before fuse_copy_args() for inargs but
+ * not for outargs
+ */
+ if ((args->in_numargs > 1) && (!args->in_pages || args->in_numargs > 2))
+ return true;
+ if (args->out_numargs && (!args->out_pages || args->out_numargs > 1))
+ return true;
+
+ return false;
+}
+
+static int fuse_uring_select_buffer(struct fuse_ring_ent *ent)
+{
+ struct fuse_ring_queue *queue = ent->queue;
+ struct fuse_bufpool *pool = queue->bufpool;
+ unsigned int id;
+
+ lockdep_assert_held(&queue->lock);
+
+ id = find_first_bit(pool->free_map, pool->nr_bufs);
+ if (id >= pool->nr_bufs)
+ return -ENOBUFS;
+
+ WARN_ON_ONCE(ent->payload.iov_base);
+ __clear_bit(id, pool->free_map);
+
+ ent->buf_id = id;
+ ent->payload.iov_base =
+ (void __user *)(pool->base_uaddr + id * pool->buf_size);
+ ent->payload.iov_len = pool->buf_size;
+
+ return 0;
+}
+
+static void fuse_uring_recycle_buffer(struct fuse_ring_ent *ent)
+{
+ struct iovec *ent_payload = &ent->payload;
+ struct fuse_ring_queue *queue = ent->queue;
+ struct fuse_bufpool *pool;
+
+ lockdep_assert_held(&queue->lock);
+
+ if (!bufpool_enabled(queue) || !ent_payload->iov_base)
+ return;
+
+ pool = queue->bufpool;
+
+ /* a buffer should never be recycled twice */
+ WARN_ON_ONCE(test_bit(ent->buf_id, pool->free_map));
+ __set_bit(ent->buf_id, pool->free_map);
+
+ memset(ent_payload, 0, sizeof(*ent_payload));
+ ent->buf_id = 0;
+}
+
+static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent,
+ struct fuse_req *req)
+{
+ bool buffer_selected;
+ bool has_payload;
+
+ if (!bufpool_enabled(ent->queue))
+ return 0;
+
+ buffer_selected = !!ent->payload.iov_base;
+ has_payload = fuse_uring_req_has_copyable_payload(ent, req);
+
+ if (has_payload && !buffer_selected)
+ return fuse_uring_select_buffer(ent);
+
+ if (!has_payload && buffer_selected)
+ fuse_uring_recycle_buffer(ent);
+
+ return 0;
+}
+
+static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent,
+ struct fuse_req *req)
+{
+ if (!bufpool_enabled(ent->queue))
+ return 0;
+
+ /* no payload to copy, can skip selecting a buffer */
+ if (!fuse_uring_req_has_copyable_payload(ent, req))
+ return 0;
+
+ return fuse_uring_select_buffer(ent);
}
static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,
- struct fuse_req *req)
+ struct fuse_req *req,
+ unsigned int issue_flags)
{
int err;
- err = fuse_uring_copy_to_ring(ent, req);
+ err = fuse_uring_copy_to_ring(ent, req, issue_flags);
if (!err) {
set_bit(FR_SENT, &req->flags);
trace_fuse_request_sent(req);
@@ -788,7 +1068,7 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,
ent->state = FRRS_INVALID;
spin_unlock(&ent->queue->lock);
- fuse_uring_req_end(ent, req, err);
+ fuse_uring_req_end(ent, req, err, issue_flags);
}
return err;
@@ -856,9 +1136,12 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent)
/* get and assign the next entry while it is still holding the lock */
req = list_first_entry_or_null(req_queue, struct fuse_req, list);
- if (req)
- fuse_uring_add_req_to_ring_ent(ent, req);
+ if (!req || fuse_uring_next_req_update_buffer(ent, req)) {
+ fuse_uring_recycle_buffer(ent);
+ return NULL;
+ }
+ fuse_uring_add_req_to_ring_ent(ent, req);
return req;
}
@@ -870,12 +1153,13 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent)
static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
unsigned int issue_flags)
{
- struct fuse_ring *ring = ent->queue->ring;
+ struct fuse_out_header out_header;
ssize_t err = -EFAULT;
- if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h,
- sizeof(req->out.h)))
+ if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &out_header,
+ sizeof(out_header)))
goto out;
+ req->out.h = out_header;
err = fuse_uring_out_header_has_err(&req->out.h, req);
if (err) {
@@ -883,9 +1167,9 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
goto out;
}
- err = fuse_uring_copy_from_ring(ring, req, ent);
+ err = fuse_uring_copy_from_ring(req, ent, issue_flags);
out:
- fuse_uring_req_end(ent, req, err);
+ fuse_uring_req_end(ent, req, err, issue_flags);
}
/*
@@ -895,7 +1179,8 @@ out:
* Else, there is no next fuse request and this returns false.
*/
static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent,
- struct fuse_ring_queue *queue)
+ struct fuse_ring_queue *queue,
+ unsigned int issue_flags)
{
int err;
struct fuse_req *req;
@@ -907,7 +1192,7 @@ retry:
spin_unlock(&queue->lock);
if (req) {
- err = fuse_uring_prepare_send(ent, req);
+ err = fuse_uring_prepare_send(ent, req, issue_flags);
if (err)
goto retry;
}
@@ -967,7 +1252,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
if (qid >= ring->nr_queues)
return -EINVAL;
- queue = ring->queues[qid];
+ queue = READ_ONCE(ring->queues[qid]);
if (!queue)
return err;
fpq = &queue->fpq;
@@ -981,6 +1266,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
return err;
}
+ if (!fuse_uring_cmd_index_ok(cmd, queue)) {
+ spin_unlock(&queue->lock);
+ return -EINVAL;
+ }
+
/* Find a request based on the unique ID of the fuse request
* This should get revised, as it needs a hash calculation and list
* search. And full struct fuse_pqueue is needed (memory overhead).
@@ -1002,8 +1292,14 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
if (err != 0) {
pr_info_ratelimited("qid=%d commit_id %llu state %d",
queue->qid, commit_id, ent->state);
+ fuse_uring_recycle_buffer(ent);
spin_unlock(&queue->lock);
- fuse_uring_req_end(ent, req, err);
+ /*
+ * Unregister any zero copyable pages since ent->cmd is null
+ * when it hits fuse_uring_req_end() in this path
+ */
+ zero_copy_unregister(cmd, ent, issue_flags);
+ fuse_uring_req_end(ent, req, err, issue_flags);
return err;
}
@@ -1019,8 +1315,13 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
* fuse requests would otherwise not get processed - committing
* and fetching is done in one step vs legacy fuse, which has separated
* read (fetch request) and write (commit result).
+ *
+ * If there is no next request or if all buffers are busy (if using a
+ * bufpool), the cmd is not returned to userspace. The entry is left
+ * available and the cmd only returns to userspace when there's a
+ * next request and an available buffer.
*/
- if (fuse_uring_get_next_fuse_req(ent, queue))
+ if (fuse_uring_get_next_fuse_req(ent, queue, issue_flags))
fuse_uring_send(ent, cmd, 0, issue_flags);
return 0;
}
@@ -1035,7 +1336,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid)
if (current_qid == qid)
continue;
- queue = ring->queues[qid];
+ queue = READ_ONCE(ring->queues[qid]);
if (!queue) {