From 6648f54f3459c5d069b7ce294170d721ce9bac2f Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Mon, 6 Jul 2026 15:44:58 +0200 Subject: fuse: move "epoch" from dentry.d_time to fuse_dentry.epoch ...in hope of removing d_time one day. Fixes: 2396356a945b ("fuse: add more control over cache invalidation behaviour") Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 23 ++++++++++++++++------- fs/fuse/fuse_i.h | 2 ++ fs/fuse/readdir.c | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 0e2a1039fa43..b689503bc880 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -96,6 +96,7 @@ static void fuse_advise_use_readdirplus(struct inode *dir) struct fuse_dentry { u64 time; + u64 epoch; union { struct rcu_head rcu; struct rb_node node; @@ -236,6 +237,13 @@ void fuse_dentry_tree_cleanup(void) WARN_ON_ONCE(!RB_EMPTY_ROOT(&dentry_hash[i].tree)); } +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + + fd->epoch = epoch; +} + static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) { ((struct fuse_dentry *) dentry->d_fsdata)->time = time; @@ -387,10 +395,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, struct fuse_mount *fm; struct fuse_conn *fc; struct fuse_inode *fi; + struct fuse_dentry *fd = entry->d_fsdata; int ret; fc = get_fuse_conn_super(dir->i_sb); - if (entry->d_time < atomic_read(&fc->epoch)) + if (fd->epoch < atomic_read(&fc->epoch)) goto invalid; inode = d_inode_rcu(entry); @@ -480,10 +489,10 @@ static int fuse_dentry_init(struct dentry *dentry) RB_CLEAR_NODE(&fd->node); dentry->d_fsdata = fd; /* - * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * Initialising epoch to '0' ensures the dentry is invalid * if compared to fc->epoch, which is initialized to '1'. */ - dentry->d_time = 0; + fuse_dentry_set_epoch(dentry, 0); return 0; } @@ -641,7 +650,7 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, goto out_err; entry = newent ? newent : entry; - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); if (outarg_valid) fuse_change_entry_timeout(entry, &outarg); else @@ -898,7 +907,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, } kfree(forget); d_instantiate(entry, inode); - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outentry); fuse_dir_changed(dir); err = generic_file_open(inode, file); @@ -1028,10 +1037,10 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun return d; if (d) { - d->d_time = epoch; + fuse_dentry_set_epoch(d, epoch); fuse_change_entry_timeout(d, &outarg); } else { - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outarg); } fuse_dir_changed(dir); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 85f738c53122..c8d4c5f3af7e 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1054,6 +1054,8 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch); + /* * Initialize fuse_conn */ diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0e1321491747..5ca87151d70d 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -260,7 +260,7 @@ retry: } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); - dentry->d_time = epoch; + fuse_dentry_set_epoch(dentry, epoch); fuse_change_entry_timeout(dentry, o); dput(dentry); -- cgit v1.2.3 From cc6f804e785f9a7c04333cb55372723da482ae3f Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:48 -0700 Subject: fuse: don't clear folio uptodate on writethrough errors In the writethrough path (fuse_send_write_pages()), if the write to the server failed or was a short write, the uptodate flag on the folios are cleared. As explained by Matthew in [1], this is dangerous because the folio may be mapped into userspace. The mm code has the invariant that a non-uptodate folio must never be visible to userspace (to avoid potentially leaking confidental information to userspace) and has checks in place for this that if violated can bring down the whole machine. Practically speaking, the effect of this change for the fuse writethrough error path is that if an application does a write and then the server fails to persist the data or only services a short write, the page cache folio keeps the data the application wrote instead of being reverted to the server's contents on the next read. The failure is still reported to the application synchronously through the short count / error return of the write() syscall. Folios that were only partially written are unaffected since they were never marked uptodate in the first place (fuse_fill_write_page() only marks a folio as uptodate if the whole folio was written to). [1] https://lore.kernel.org/linux-fsdevel/ajtPMgO65FA1TXhi@casper.infradead.org/ Suggested-by: Matthew Wilcox Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Reviewed-by: Christoph Hellwig Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ceada75310b8..f5bcbfa8b6ae 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1219,8 +1219,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; - unsigned int offset, i; - bool short_write; + unsigned int i; int err; for (i = 0; i < ap->num_folios; i++) @@ -1235,24 +1234,9 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (!err && ia->write.out.size > count) err = -EIO; - short_write = ia->write.out.size < count; - offset = ap->descs[0].offset; - count = ia->write.out.size; for (i = 0; i < ap->num_folios; i++) { struct folio *folio = ap->folios[i]; - if (err) { - folio_clear_uptodate(folio); - } else { - if (count >= folio_size(folio) - offset) - count -= folio_size(folio) - offset; - else { - if (short_write) - folio_clear_uptodate(folio); - count = 0; - } - offset = 0; - } if (ia->write.folio_locked && (i == ap->num_folios - 1)) folio_unlock(folio); folio_put(folio); -- cgit v1.2.3 From 03e1dd35c206f24b6bc987198ac58e5138176cb7 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:49 -0700 Subject: iomap: add helper to mark folio uptodate Add an exported helper iomap_folio_mark_uptodate() to mark a folio as uptodate and update its uptodate bitmap if the folio has iomap state data attached. This is needed because there are some filesystems (eg fuse) that have paths outside of conventional iomap calls that need to mark a folio as uptodate (eg writing server-pushed data directly into the page cache) and need the iomap-internal uptodate bitmap to be in sync with the uptodate state of the folio. Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/iomap/buffered-io.c | 6 ++++++ include/linux/iomap.h | 1 + 2 files changed, 7 insertions(+) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 276720bc18dc..af415293e265 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -105,6 +105,12 @@ static void iomap_set_range_uptodate(struct folio *folio, size_t off, folio_mark_uptodate(folio); } +void iomap_folio_mark_uptodate(struct folio *folio) +{ + iomap_set_range_uptodate(folio, 0, folio_size(folio)); +} +EXPORT_SYMBOL_GPL(iomap_folio_mark_uptodate); + /* * Find the next dirty block in the folio. end_blk is inclusive. * If no dirty block is found, this will return end_blk + 1. diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 56b43d594e6e..40aa3476a351 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -365,6 +365,7 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len); bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio); +void iomap_folio_mark_uptodate(struct folio *folio); int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops); -- cgit v1.2.3 From 16f4be93c65ab3fc9c4c831849722581f8f3ca14 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:50 -0700 Subject: fuse: use iomap helper to mark folio uptodate When fuse enables large folios, a large folio will be backed by iomap_folio_state that keeps track of uptodate and dirty state in an internal bitmap. Fuse writethrough and notify store paths currently set folio uptodate state with folio_mark_uptodate(), which touches only the folio-level flag, but on an iomap-backed folio, that leaves the uptodate bitmap out of sync. Use the iomap_folio_mark_uptodate() helper to update both the folio uptodate state and the iomap uptodate bitmap. Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Reviewed-by: Christoph Hellwig Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 2 +- fs/fuse/notify.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f5bcbfa8b6ae..da5859e8159d 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1311,7 +1311,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); if (folio_test_uptodate(folio)) { folio_unlock(folio); diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c index 29578104ae6c..1ba763705d91 100644 --- a/fs/fuse/notify.c +++ b/fs/fuse/notify.c @@ -2,6 +2,8 @@ #include "dev.h" #include "fuse_i.h" + +#include #include static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, @@ -192,7 +194,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && (nr_bytes == folio_size(folio) || file_size == end)) { folio_zero_segment(folio, nr_bytes, folio_size(folio)); - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); } folio_unlock(folio); folio_put(folio); -- cgit v1.2.3 From ed9c881f3b498383f73c42712b359419da42a7b0 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Thu, 9 Jul 2026 08:37:05 +0200 Subject: fuse: fix race between interrupt and resend After commit f8fce75fedf7 ("fuse: clear intr_entry in fuse_resend and fuse_remove_pending_req") the WARN_ON(!list_empty(&req->intr_entry)) in fuse_request_free() still triggers due to the following race: In request_wait_answer() if (test_bit(FR_SENT, &req->flags)) -> returns true In fuse_chan_resend() clear_bit(FR_SENT, &req->flags) In request_wait_answer() queue_interrupt(req) Fix by: - move clearing FR_SENT inside fpq->lock - move setting FR_PENDING inside fiq->lock - recheck FR_SENT after acquiring fiq->lock in fuse_dev_queue_interrupt() Reported-by: zdi-disclosures@trendmicro.com Fixes: f8fce75fedf7 ("fuse: clear intr_entry in fuse_resend and fuse_remove_pending_req") Cc: stable@vger.kernel.org # 6.9 Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 5763a7cd3b37..7e81dea4f1c7 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -240,7 +240,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); - if (list_empty(&req->intr_entry)) { + /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ + if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) { list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() @@ -1760,7 +1761,7 @@ out: void fuse_chan_resend(struct fuse_chan *fch) { struct fuse_dev *fud; - struct fuse_req *req, *next; + struct fuse_req *req; struct fuse_iqueue *fiq = &fch->iq; LIST_HEAD(to_queue); unsigned int i; @@ -1775,24 +1776,20 @@ void fuse_chan_resend(struct fuse_chan *fch) struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - list_splice_tail_init(&fpq->processing[i], &to_queue); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + struct list_head *this_queue = &fpq->processing[i]; + + list_for_each_entry(req, this_queue, list) + clear_bit(FR_SENT, &req->flags); + list_splice_tail_init(this_queue, &to_queue); + } spin_unlock(&fpq->lock); } spin_unlock(&fch->lock); - list_for_each_entry_safe(req, next, &to_queue, list) { - set_bit(FR_PENDING, &req->flags); - clear_bit(FR_SENT, &req->flags); - /* mark the request as resend request */ - req->in.h.unique |= FUSE_UNIQUE_RESEND; - } - spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - list_for_each_entry(req, &to_queue, list) - clear_bit(FR_PENDING, &req->flags); fuse_dev_end_requests(&to_queue); return; } @@ -1801,6 +1798,10 @@ void fuse_chan_resend(struct fuse_chan *fch) * intr_entry on fiq->interrupts after the request is re-queued. */ list_for_each_entry(req, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; + if (test_bit(FR_INTERRUPTED, &req->flags)) list_del_init(&req->intr_entry); } -- cgit v1.2.3 From edb310bc27f0ad83e7fd558a3caf1a94ca511654 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:42 -0700 Subject: fuse: fix missing barrier when checking io-uring readiness fuse_block_alloc() reads fch->initialized and then fch->io_uring. fch->io_uring is set before fch->initialized, ordered by the smp_wmb() in fuse_chan_set_intialized(), but fuse_block_alloc() has no matching read barrier between the two loads. This may lead a CPU to observe fch->initialized=1 but fch->io_uring=0, and skip the check that blocks request allocation until the io-uring queues are ready. This can reintroduce the lock-order inversion deadlock that commit 3393ff964e0f prevents. Add an smp_rmb() barrier to pair with the smp_wmb() in fuse_chan_set_initialized() to prevent this. Fixes: 3393ff964e0f ("fuse: block request allocation until io-uring init is complete") Cc: stable@vger.kernel.org Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 7e81dea4f1c7..a27ea64d763a 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -85,7 +85,13 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - return !fch->initialized || (for_background && fch->blocked) || + if (!fch->initialized) + return true; + + /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ + smp_rmb(); + + return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -120,9 +126,6 @@ static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) goto out; } - /* Matches smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - err = -ENOTCONN; if (!fch->connected) goto out; -- cgit v1.2.3 From 4ef7c8cc9894fccc7aa5fdaf6b39faa45c58c23e Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:43 -0700 Subject: fuse: use release/acquire for fch->initialized fuse_chan_set_initialized() sets values for the connection state and then sets fch->initialized to true, but lockless readers read fch->initialized and if true, go to read the connection state values, without using any barriers. There are a few instances where this happens (fuse_uring_cmd() before dispatching register / commit-and-fetch cmds, fuse_dev_do_wriite() for handling notify retrieves, etc). To make this as simple as possible, use release/acquire semantics for writing/reading fch->initialized. Add the missing read barriers. This is not marked for stable as these are not realistically reachable on a well-behaved server, and buggy/malicious servers who trigger this path fail benignly rather than crash or deadlock the kernel. Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/cuse.c | 3 ++- fs/fuse/dev.c | 14 ++++++-------- fs/fuse/dev_uring.c | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 3c15b5ba16d7..96d57735a79f 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -530,7 +530,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file) INIT_LIST_HEAD(&cc->list); - cc->fc.chan->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&cc->fc.chan->initialized, 1); rc = cuse_send_init(cc); if (rc) { fuse_dev_put(fud); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index a27ea64d763a..27dafda2a841 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -77,20 +77,17 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->max_pages = param->max_pages; } - /* Make sure stores before this are seen on another CPU */ - smp_wmb(); - fch->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&fch->initialized, 1); wake_up_all(&fch->blocked_waitq); } static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - if (!fch->initialized) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized)) return true; - /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -1892,7 +1889,8 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * initialized and connected state */ err = -EINVAL; - if (!fch->initialized || !fch->connected) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized) || !fch->connected) goto copy_finish; /* Don't try to move folios (yet) */ diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 77c8cec43d9c..51f985154aa1 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1251,8 +1251,10 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) /* * fuse_uring_register() needs the ring to be initialized, * we need to know the max payload size + * + * Pairs with smp_store_release() in fuse_chan_set_initialized() */ - if (!fch->initialized) + if (!smp_load_acquire(&fch->initialized)) return -EAGAIN; switch (cmd_op) { -- cgit v1.2.3 From 42df916e5a5f8fb4b60c8cefb54318d1ec02c580 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:44 -0700 Subject: fuse: publish io-uring queues with release semantics fuse_uring_create_queue() initializes a fuse_ring_queue and then publishes the pointer into ring->queues[qid] with WRITE_ONCE() under the fch->lock. There are several readers that may concurrently be fetching that pointer locklessly and then deferencing it. WRITE_ONCE() doesn't ensure ordering of the queue's field initialization before the ring->queues[qid] pointer assignment. The queue must be published with smp_store_release() so the field initialization is guaranteed to happen before. Readers in paths where the read may happen concurrently with the store need to use READ_ONCE() because any race involving a plain access is undefined. Fixes: 24fe962c86f5 ("fuse: {io-uring} Handle SQEs - register commands") Cc: stable@vger.kernel.org Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 51f985154aa1..c8488ebc1d1f 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -204,7 +204,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) return; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); struct fuse_ring_ent *ent, *next; if (!queue) @@ -223,7 +223,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) kfree(queue->fpq.processing); kfree(queue); - ring->queues[qid] = NULL; + WRITE_ONCE(ring->queues[qid], NULL); } kfree(ring->queues); @@ -321,9 +321,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, } /* - * write_once and lock as the caller mostly doesn't take the lock at all + * fch->lock serializes concurrent creators for this qid. + * smp_store_release() are for the lockless readers who must see a + * fully initialized queue after &ring->queues[qid] is set */ - WRITE_ONCE(ring->queues[qid], queue); + smp_store_release(&ring->queues[qid], queue); spin_unlock(&fch->lock); return queue; @@ -434,7 +436,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) struct fuse_ring_ent *ent; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -967,7 +969,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (qid >= ring->nr_queues) return -EINVAL; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) return err; fpq = &queue->fpq; @@ -1035,7 +1037,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) if (current_qid == qid) continue; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { ready = false; break; @@ -1191,7 +1193,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return -EINVAL; } - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) @@ -1332,7 +1334,7 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) ring->nr_queues)) qid = 0; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); return queue; -- cgit v1.2.3 From 98b4ca2378e1f6b6c06a74f699623ebecfb3549d Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Mon, 22 Jun 2026 17:42:27 -0700 Subject: fuse: allow larger read requests by setting bdi->io_pages A FUSE server that advertises a large max_pages and max_write (e.g. max_pages=256, max_write=1MB) cannot currently obtain matching FUSE_READ request sizes from the kernel. Buffered sequential writes arrive at the server at the negotiated max_write size, but a large buffered read() is split into several smaller FUSE_READ requests. For a buffered read, filemap_get_pages() -> page_cache_sync_ra() sizes the read against ractl_max_pages(): max_pages = ractl->ra->ra_pages; if (req_size > max_pages && bdi->io_pages > max_pages) max_pages = min(req_size, bdi->io_pages); fuse leaves bdi->io_pages at the default VM_READAHEAD_PAGES (128KB), so a 1MB read() (req_size = 256 pages) is clamped to the readahead window (128KB, or 256KB for POSIX_FADV_SEQUENTIAL), producing four 256KB FUSE_READ round-trips instead of one. Set bdi->io_pages to fc->max_pages after feature negotiation. As the code above shows, io_pages only raises the limit when the request size already exceeds the readahead window, so it enlarges explicitly requested reads without enlarging the speculative readahead window. This avoids increasing speculative page-cache readahead on behalf of an unprivileged server. NFS does the same, setting io_pages from rpages while leaving ra_pages at the default. fc->max_pages is already bounded by fc->max_pages_limit (and, for virtio-fs, by the virtqueue descriptor count), so io_pages inherits the same bound. Suggested-by: Joanne Koong Signed-off-by: Jim Harris Assisted-by: Cursor:claude-opus-4.8 Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d975073c6029..f7a0a0860a04 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1416,6 +1416,7 @@ static void process_init_reply(struct fuse_args *args, int error) fm->sb->s_bdi->ra_pages = min(fm->sb->s_bdi->ra_pages, ra_pages); + fm->sb->s_bdi->io_pages = fc->max_pages; fc->minor = arg->minor; fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; fc->max_write = max_t(unsigned, 4096, fc->max_write); -- cgit v1.2.3 From 51e08eaf954de51b991889a5f5b5b5edcc712c6c Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:37 -0700 Subject: io_uring/rsrc: rename io_buffer_register_bvec()/io_buffer_unregister_bvec() Currently, io_buffer_register_bvec() takes in a request. In preparation for supporting kernel-populated buffers in fuse io-uring (which will need to register bvecs directly, not through a struct request), rename this to io_buffer_register_request(). A subsequent patch will commandeer the "io_buffer_register_bvec()" function name to support registering bvecs directly. Rename io_buffer_unregister_bvec() to a more generic name, io_buffer_unregister(), as both io_buffer_register_request() and io_buffer_register_bvec() callers will use it for unregistration. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-2-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- Documentation/block/ublk.rst | 14 +++++++------- drivers/block/ublk_drv.c | 22 +++++++++++----------- include/linux/io_uring/cmd.h | 25 +++++++++++++++++++------ io_uring/rsrc.c | 14 +++++++------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Documentation/block/ublk.rst b/Documentation/block/ublk.rst index 0413dcd9ef69..28300fee22bf 100644 --- a/Documentation/block/ublk.rst +++ b/Documentation/block/ublk.rst @@ -382,17 +382,17 @@ Zero copy --------- ublk zero copy relies on io_uring's fixed kernel buffer, which provides -two APIs: `io_buffer_register_bvec()` and `io_buffer_unregister_bvec`. +two APIs: `io_buffer_register_request()` and `io_buffer_unregister`. ublk adds IO command of `UBLK_IO_REGISTER_IO_BUF` to call -`io_buffer_register_bvec()` for ublk server to register client request +`io_buffer_register_request()` for ublk server to register client request buffer into io_uring buffer table, then ublk server can submit io_uring IOs with the registered buffer index. IO command of `UBLK_IO_UNREGISTER_IO_BUF` -calls `io_buffer_unregister_bvec()` to unregister the buffer, which is -guaranteed to be live between calling `io_buffer_register_bvec()` and -`io_buffer_unregister_bvec()`. Any io_uring operation which supports this -kind of kernel buffer will grab one reference of the buffer until the -operation is completed. +calls `io_buffer_unregister()` to unregister the buffer, which is guaranteed +to be live between calling `io_buffer_register_request()` and +`io_buffer_unregister()`. Any io_uring operation which supports this kind of +kernel buffer will grab one reference of the buffer until the operation is +completed. ublk server implementing zero copy or user copy has to be CAP_SYS_ADMIN and be trusted, because it is ublk server's responsibility to make sure IO buffer diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 4f6d9e652187..4036eb6be056 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1699,8 +1699,8 @@ ublk_auto_buf_register(const struct ublk_queue *ubq, struct request *req, { int ret; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, - io->buf.auto_reg.index, issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, + io->buf.auto_reg.index, issue_flags); if (ret) { if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) { ublk_auto_buf_reg_fallback(ubq, req->tag); @@ -1906,7 +1906,7 @@ static noinline void ublk_batch_dispatch_fail(struct ublk_queue *ubq, ublk_io_unlock(io); if (index != -1) - io_buffer_unregister_bvec(data->cmd, index, + io_buffer_unregister(data->cmd, index, data->issue_flags); } @@ -3194,8 +3194,8 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd, if (!req) return -EINVAL; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, - issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, index, + issue_flags); if (ret) { ublk_put_req_ref(io, req); return ret; @@ -3226,8 +3226,8 @@ ublk_daemon_register_io_buf(struct io_uring_cmd *cmd, if (!ublk_dev_support_zero_copy(ub) || !blk_rq_has_data(req)) return -EINVAL; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, - issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, index, + issue_flags); if (ret) return ret; @@ -3242,7 +3242,7 @@ static int ublk_unregister_io_buf(struct io_uring_cmd *cmd, if (!(ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)) return -EINVAL; - return io_buffer_unregister_bvec(cmd, index, issue_flags); + return io_buffer_unregister(cmd, index, issue_flags); } static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr) @@ -3383,7 +3383,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, goto out; /* - * io_buffer_unregister_bvec() doesn't access the ubq or io, + * io_buffer_unregister() doesn't access the ubq or io, * so no need to validate the q_id, tag, or task */ if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) @@ -3450,7 +3450,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, req = ublk_fill_io_cmd(io, cmd); ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); + io_buffer_unregister(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); if (req_op(req) == REQ_OP_ZONE_APPEND) @@ -3787,7 +3787,7 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, } if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(data->cmd, buf_idx, data->issue_flags); + io_buffer_unregister(data->cmd, buf_idx, data->issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = ublk_batch_zone_lba(uc, elem); if (compl) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index 331dcbefe72f..bbf57da1e4c8 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -91,6 +91,11 @@ struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, struct io_br_sel *sel, unsigned int issue_flags); +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags); +int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, + unsigned int issue_flags); #else static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, @@ -133,6 +138,20 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, { return true; } +static inline int io_buffer_register_request(struct io_uring_cmd *cmd, + struct request *rq, + void (*release)(void *), + unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} +static inline int io_buffer_unregister(struct io_uring_cmd *cmd, + unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} #endif static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req) @@ -182,10 +201,4 @@ static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); } -int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags); -int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, - unsigned int issue_flags); - #endif /* _LINUX_IO_URING_CMD_H */ diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 8d0f2ee24e0c..40807994a8f4 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1015,9 +1015,9 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } -int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags) +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; @@ -1076,10 +1076,10 @@ unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; } -EXPORT_SYMBOL_GPL(io_buffer_register_bvec); +EXPORT_SYMBOL_GPL(io_buffer_register_request); -int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, - unsigned int issue_flags) +int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, + unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; @@ -1109,7 +1109,7 @@ unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; } -EXPORT_SYMBOL_GPL(io_buffer_unregister_bvec); +EXPORT_SYMBOL_GPL(io_buffer_unregister); static int validate_fixed_range(u64 buf_addr, size_t len, const struct io_mapped_ubuf *imu) -- cgit v1.2.3 From fbc32d5f44d3e2134443909e816c5f9c657e81ad Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:38 -0700 Subject: io_uring/rsrc: split io_buffer_register_request() logic Split the main initialization logic in io_buffer_register_request() into a helper function. This is a preparatory patch for supporting kernel-populated buffers in fuse io-uring, which will be reusing this logic. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-3-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- io_uring/rsrc.c | 84 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 40807994a8f4..5d50b967645b 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1015,63 +1015,81 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } -int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags) +static struct io_mapped_ubuf *io_kernel_buffer_init(struct io_ring_ctx *ctx, + unsigned int nr_bvecs, + unsigned int total_bytes, + u8 dir, + void (*release)(void *), + void *priv, + unsigned int index) { - struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; - struct req_iterator rq_iter; struct io_mapped_ubuf *imu; struct io_rsrc_node *node; - struct bio_vec bv; - unsigned int nr_bvecs = 0; - int ret = 0; - io_ring_submit_lock(ctx, issue_flags); - if (index >= data->nr) { - ret = -EINVAL; - goto unlock; - } + if (index >= data->nr) + return ERR_PTR(-EINVAL); index = array_index_nospec(index, data->nr); - if (data->nodes[index]) { - ret = -EBUSY; - goto unlock; - } + if (data->nodes[index]) + return ERR_PTR(-EBUSY); node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); - if (!node) { - ret = -ENOMEM; - goto unlock; - } + if (!node) + return ERR_PTR(-ENOMEM); - /* - * blk_rq_nr_phys_segments() may overestimate the number of bvecs - * but avoids needing to iterate over the bvecs - */ - imu = io_alloc_imu(ctx, blk_rq_nr_phys_segments(rq)); + imu = io_alloc_imu(ctx, nr_bvecs); if (!imu) { io_cache_free(&ctx->node_cache, node); - ret = -ENOMEM; - goto unlock; + return ERR_PTR(-ENOMEM); } imu->ubuf = 0; - imu->len = blk_rq_bytes(rq); + imu->len = total_bytes; imu->folio_shift = PAGE_SHIFT; + imu->nr_bvecs = nr_bvecs; refcount_set(&imu->refs, 1); imu->release = release; - imu->priv = rq; + imu->priv = priv; + imu->dir = dir; imu->flags = IO_REGBUF_F_KBUF; - imu->dir = 1 << rq_data_dir(rq); + node->buf = imu; + data->nodes[index] = node; + + return imu; +} + +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags) +{ + struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; + struct req_iterator rq_iter; + struct io_mapped_ubuf *imu; + struct bio_vec bv; + /* + * blk_rq_nr_phys_segments() may overestimate the number of bvecs + * but avoids needing to iterate over the bvecs + */ + unsigned int nr_bvecs = blk_rq_nr_phys_segments(rq); + unsigned int total_bytes = blk_rq_bytes(rq); + int ret = 0; + + io_ring_submit_lock(ctx, issue_flags); + + imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, + 1 << rq_data_dir(rq), release, rq, index); + if (IS_ERR(imu)) { + ret = PTR_ERR(imu); + goto unlock; + } + + nr_bvecs = 0; rq_for_each_bvec(bv, rq, rq_iter) imu->bvec[nr_bvecs++] = bv; imu->nr_bvecs = nr_bvecs; - node->buf = imu; - data->nodes[index] = node; unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; -- cgit v1.2.3 From bd62a2cfff9f37d85927050f082ffc3a423e6b53 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:39 -0700 Subject: io_uring/rsrc: add io_buffer_register_bvec() Add io_buffer_register_bvec() for registering a bvec array. This is a preparatory patch for fuse-over-io-uring zero-copy. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-4-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- include/linux/io_uring/cmd.h | 13 +++++++++++++ io_uring/rsrc.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index bbf57da1e4c8..42801f0b6456 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -94,6 +94,10 @@ bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, void (*release)(void *), unsigned int index, unsigned int issue_flags); +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags); int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags); #else @@ -146,6 +150,15 @@ static inline int io_buffer_register_request(struct io_uring_cmd *cmd, { return -EOPNOTSUPP; } +static inline int io_buffer_register_bvec(struct io_uring_cmd *cmd, + const struct bio_vec *bvs, + unsigned int nr_bvecs, + void (*release)(void *), void *priv, + u8 dir, unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} static inline int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 5d50b967645b..819c5087d8d3 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1096,6 +1096,41 @@ unlock: } EXPORT_SYMBOL_GPL(io_buffer_register_request); +/* + * bvs is copied internally. caller may free it on return. + */ +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags) +{ + struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; + struct io_mapped_ubuf *imu; + struct bio_vec *bvec; + unsigned int i, total_bytes = 0; + int ret = 0; + + for (i = 0; i < nr_bvecs; i++) + total_bytes += bvs[i].bv_len; + + io_ring_submit_lock(ctx, issue_flags); + imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, dir, release, + priv, index); + if (IS_ERR(imu)) { + ret = PTR_ERR(imu); + goto unlock; + } + + bvec = imu->bvec; + for (i = 0; i < nr_bvecs; i++) + bvec[i] = bvs[i]; + +unlock: + io_ring_submit_unlock(ctx, issue_flags); + return ret; +} +EXPORT_SYMBOL_GPL(io_buffer_register_bvec); + int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags) { -- cgit v1.2.3 From 95961b72c57b29a96c14f86d16f1d32787f2e009 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:40 -0700 Subject: io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE Rename IO_IMU_DEST and IO_IMU_SOURCE to IO_BUF_DEST and IO_BUF_SOURCE and export it so subsystems may use it. This is needed by the io_buffer_register_bvec() path for callers who may need the buffer to be both readable and writable. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260612184840.4058966-5-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- include/linux/io_uring_types.h | 5 +++++ io_uring/io_uring.c | 2 +- io_uring/rsrc.c | 2 +- io_uring/rsrc.h | 5 ----- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1..db42a548c7a5 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -44,6 +44,11 @@ enum io_uring_cmd_flags { IO_URING_F_COMPAT = (1 << 12), }; +enum { + IO_BUF_DEST = 1 << ITER_DEST, + IO_BUF_SOURCE = 1 << ITER_SOURCE, +}; + struct iou_loop_params; struct io_wq_work_node { diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36..ba685b6052ed 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3245,7 +3245,7 @@ static int __init io_uring_init(void) io_uring_optable_init(); /* imu->dir is u8 */ - BUILD_BUG_ON((IO_IMU_DEST | IO_IMU_SOURCE) > U8_MAX); + BUILD_BUG_ON((IO_BUF_DEST | IO_BUF_SOURCE) > U8_MAX); /* * Allow user copy in the per-command field, which starts after the diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 819c5087d8d3..f3f01e0c8102 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -912,7 +912,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, imu->release = io_release_ubuf; imu->priv = imu; imu->flags = 0; - imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + imu->dir = IO_BUF_DEST | IO_BUF_SOURCE; if (coalesced) imu->folio_shift = data.folio_shift; refcount_set(&imu->refs, 1); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 98ae8ef51009..e503b02aa61a 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -23,11 +23,6 @@ struct io_rsrc_node { }; }; -enum { - IO_IMU_DEST = 1 << ITER_DEST, - IO_IMU_SOURCE = 1 << ITER_SOURCE, -}; - enum { IO_REGBUF_F_KBUF = 1, }; -- cgit v1.2.3 From 6330b1f61ed1d17850fc61bdb8920ca1056e2cf9 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:41 -0700 Subject: fuse: decouple fuse_ring creation from ent registration Currently, the connection's fuse_ring is created lazily on the first FUSE_IO_URING_CMD_REGISTER command. A server registers entries from one thread per queue (one per CPU) and those threads issue their first REGISTER command concurrently. They then race to create the single per-connection fuse_ring, which required open-coded handling in fuse_uring_create() to detect and protect against concurrent creations. Decouple fuse_ring creation from ent registration and move it to FUSE_INIT reply processing after a server has negotiated and set FUSE_OVER_IO_URING. The ring is published before the connection is marked initialized. fuse_uring_register() no longer creates the ring and it instead uses the ring set up at init time. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 8 +++----- fs/fuse/dev.h | 2 +- fs/fuse/dev_uring.c | 26 ++++++++++---------------- fs/fuse/dev_uring_i.h | 5 +++++ fs/fuse/inode.c | 4 +++- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 27dafda2a841..d8f97943e973 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -75,6 +75,9 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->minor = param->minor; fch->max_write = param->max_write; fch->max_pages = param->max_pages; + + if (param->io_uring_enabled) + fuse_uring_conn_init(fch); } /* Pairs with smp_load_acquire() readers of fch->initialized */ @@ -412,11 +415,6 @@ void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) fch->conn = fc; } -void fuse_chan_io_uring_enable(struct fuse_chan *fch) -{ - fch->io_uring = 1; -} - void fuse_pqueue_init(struct fuse_pqueue *fpq) { spin_lock_init(&fpq->lock); diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h index aed69fd14c41..8d25378c0918 100644 --- a/fs/fuse/dev.h +++ b/fs/fuse/dev.h @@ -22,6 +22,7 @@ struct fuse_chan_param { unsigned int minor; unsigned int max_write; unsigned int max_pages; + bool io_uring_enabled; }; struct fuse_chan *fuse_chan_new(void); @@ -34,7 +35,6 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val); unsigned int fuse_chan_num_waiting(struct fuse_chan *fch); void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc); void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param); -void fuse_chan_io_uring_enable(struct fuse_chan *fch); ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args); int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags); int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index c8488ebc1d1f..9616778505ba 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -238,7 +238,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) { struct fuse_ring *ring; size_t nr_queues = num_possible_cpus(); - struct fuse_ring *res = NULL; size_t max_payload_size; ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT); @@ -258,12 +257,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) spin_unlock(&fch->lock); goto out_err; } - if (fch->ring) { - /* race, another thread created the ring in the meantime */ - spin_unlock(&fch->lock); - res = fch->ring; - goto out_err; - } init_waitqueue_head(&ring->stop_waitq); @@ -278,7 +271,13 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) out_err: kfree(ring->queues); kfree(ring); - return res; + return NULL; +} + +void fuse_uring_conn_init(struct fuse_chan *fch) +{ + if (fuse_uring_create(fch)) + fch->io_uring = 1; } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, @@ -1178,15 +1177,10 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, struct fuse_ring *ring = smp_load_acquire(&fch->ring); struct fuse_ring_queue *queue; struct fuse_ring_ent *ent; - int err; unsigned int qid = READ_ONCE(cmd_req->qid); - err = -ENOMEM; - if (!ring) { - ring = fuse_uring_create(fch); - if (!ring) - return err; - } + if (!ring) + return -EINVAL; if (qid >= ring->nr_queues) { pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); @@ -1197,7 +1191,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) - return err; + return -ENOMEM; } /* diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 55f8d04e4b0b..d721a4fc0215 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -135,6 +135,7 @@ struct fuse_ring { bool ready; }; +void fuse_uring_conn_init(struct fuse_chan *fch); void fuse_uring_stop_queues(struct fuse_ring *ring); void fuse_uring_abort_end_requests(struct fuse_ring *ring); int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); @@ -174,6 +175,10 @@ static inline bool fuse_uring_ready(struct fuse_chan *fch) #else /* CONFIG_FUSE_IO_URING */ +static inline void fuse_uring_conn_init(struct fuse_chan *fch) +{ +} + static inline void fuse_uring_abort(struct fuse_chan *fch) { } diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index f7a0a0860a04..33773c7d129a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1272,6 +1272,7 @@ static void process_init_reply(struct fuse_args *args, int error) struct fuse_mount *fm = ia->fm; struct fuse_conn *fc = fm->fc; struct fuse_init_out *arg = &ia->out; + bool io_uring_enabled = false; bool ok = true; if (error || arg->major != FUSE_KERNEL_VERSION) @@ -1402,7 +1403,7 @@ static void process_init_reply(struct fuse_args *args, int error) ok = false; } if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) - fuse_chan_io_uring_enable(fc->chan); + io_uring_enabled = true; if (flags & FUSE_REQUEST_TIMEOUT) timeout = arg->request_timeout; @@ -1433,6 +1434,7 @@ static void process_init_reply(struct fuse_args *args, int error) .minor = fc->minor, .max_write = fc->max_write, .max_pages = fc->max_pages, + .io_uring_enabled = io_uring_enabled, }; fuse_chan_set_initialized(fc->chan, &cp); } -- cgit v1.2.3 From ebed9ea5b469588c6074f3ed5b8d8ec63c4ccf48 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:42 -0700 Subject: fuse: add FUSE_IO_URING_CMD_ADD_QUEUE fuse-over-io-uring queues are currently created lazily, as a side effect of the first FUSE_IO_URING_CMD_REGISTER command for a given qid. This ties queue creation to entry registration. Add a FUSE_IO_URING_CMD_ADD_QUEUE command so a server can create a queue explicitly, decoupling queue setup from entry registration. This is additionally a prerequisite for FUSE_IO_URING_CMD_ADD_BUFPOOL, which attaches a buffer pool to an existing queue and therefore needs the queue to have been created first. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 45 ++++++++++++++++++++++++++++++++++++++------- include/uapi/linux/fuse.h | 8 +++++++- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 9616778505ba..3b9fd0daef66 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -281,7 +281,8 @@ void fuse_uring_conn_init(struct fuse_chan *fch) } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, - int qid) + int qid, + bool fail_if_exists) { struct fuse_chan *fch = ring->chan; struct fuse_ring_queue *queue; @@ -289,11 +290,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT); if (!queue) - return NULL; + return ERR_PTR(-ENOMEM); pq = fuse_pqueue_alloc(); if (!pq) { kfree(queue); - return NULL; + return ERR_PTR(-ENOMEM); } queue->qid = qid; @@ -316,7 +317,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, spin_unlock(&fch->lock); kfree(queue->fpq.processing); kfree(queue); - return ring->queues[qid]; + return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } /* @@ -1189,9 +1190,9 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, queue = READ_ONCE(ring->queues[qid]); if (!queue) { - queue = fuse_uring_create_queue(ring, qid); - if (!queue) - return -ENOMEM; + queue = fuse_uring_create_queue(ring, qid, false); + if (IS_ERR(queue)) + return PTR_ERR(queue); } /* @@ -1206,6 +1207,30 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return fuse_uring_do_register(ent, cmd, issue_flags); } +static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + unsigned int qid = READ_ONCE(cmd_req->qid); + uint64_t flags = READ_ONCE(cmd_req->flags); + struct fuse_ring_queue *queue; + + if (!ring || flags) + return -EINVAL; + + if (qid >= ring->nr_queues) { + pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); + return -EINVAL; + } + + queue = fuse_uring_create_queue(ring, qid, true); + if (IS_ERR(queue)) + return PTR_ERR(queue); + + return 0; +} + /* * Entry function from io_uring to handle the given passthrough command * (op code IORING_OP_URING_CMD) @@ -1272,6 +1297,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return err; } break; + case FUSE_IO_URING_CMD_ADD_QUEUE: + err = fuse_uring_add_queue(cmd, fch); + if (err) + pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n", + err); + return err; default: return -EINVAL; } diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index c13e1f9a2f12..cfb055c0c764 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -240,6 +240,9 @@ * - add FUSE_COPY_FILE_RANGE_64 * - add struct fuse_copy_file_range_out * - add FUSE_NOTIFY_PRUNE + * + * 7.46 + * - add FUSE_IO_URING_CMD_ADD_QUEUE */ #ifndef _LINUX_FUSE_H @@ -275,7 +278,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 45 +#define FUSE_KERNEL_MINOR_VERSION 46 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -1292,6 +1295,9 @@ enum fuse_uring_cmd { /* commit fuse request result and fetch next request */ FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2, + + /* add a queue */ + FUSE_IO_URING_CMD_ADD_QUEUE = 3, }; /** -- cgit v1.2.3 From b45aaabc628bc7356e21bd2eb0c2ae9bdfa13894 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:43 -0700 Subject: fuse: add io-uring buffer pools Right now, ents and buffers are tightly coupled in fuse io-uring where each entry has its own dedicated payload buffer, requiring N buffers for N entries where each buffer must be large enough to accomodate the maximum payload size. This is suboptimal as most request types (lookup, open, release, getattr, etc) require vastly less bytes than the maximum payload size and some requests (unlink, rmdir, fsync, flush, etc) do not require payload buffers at all. Instead of requiring a 1:1 coupling between ents and payload buffers, allow the server to pass in a buffer pool (a contiguous chunk of memory) that the kernel will use as it wishes for servicing ents/requests. Entries only reserve a "buffer" from the pool while actively processing a request that requires a payload buffer. This decoupling and letting the kernel delegate memory from the pool for requests allows the kernel to optimize memory usage and reduces the memory usage requirements needed to use fuse-over-io-uring. A pool is registered per queue with the new FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}. Internally, the kernel splits the region into buffers of ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A queue commits to a payload mode on first use: registering an entry that carries its own payload selects the legacy per-entry mode, while ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so ADD_BUFPOOL must be issued before any payload-carrying entries are registered on that queue. The queue must have been created before the bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command. The kernel tracks free buffers with a bitmap (a set bit marks a free buffer). On dispatch, a request that needs a payload claims a free buffer (find_first_bit + clear). A request that needs none claims nothing. The buffer's byte offset within the pool is reported to the server in the new fuse_uring_ent_in_out.offset field so that the server can locate the payload. On completion the buffer is returned to the pool or reused directly if the next request on that entry also has a payload. The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the server for bufpools. Buffer pool request flow ~~~~~~~~~~~~~~~~~~~~~~~~ | Kernel | FUSE daemon | | | [request arrives] | | [claim a free pool buffer] | | >fuse_uring_select_buffer() | | [copy headers to ring] | | [copy payload to buffer] | | [report buffer offset in ent_in_out] | | >io_uring_cmd_done() | | | [read headers] | | [read/write payload at offset] | | [process request] | | >io_uring_submit() | | COMMIT_AND_FETCH | >fuse_uring_commit_fetch() | | [copy reply from ring] | | [return buffer to the pool] | | >fuse_uring_recycle_buffer() | Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 239 ++++++++++++++++++++++++++++++++++++++++------ fs/fuse/dev_uring_i.h | 37 ++++++- fs/fuse/inode.c | 2 +- include/uapi/linux/fuse.h | 21 +++- 4 files changed, 269 insertions(+), 30 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 3b9fd0daef66..d300c7f441c4 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -9,6 +9,7 @@ #include "dev_uring_i.h" #include "fuse_trace.h" +#include #include #include @@ -41,6 +42,11 @@ enum fuse_uring_header_type { FUSE_URING_HEADER_RING_ENT, }; +static inline bool bufpool_enabled(struct fuse_ring_queue *queue) +{ + return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL; +} + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -222,6 +228,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) } kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); WRITE_ONCE(ring->queues[qid], NULL); } @@ -316,6 +323,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, if (ring->queues[qid]) { spin_unlock(&fch->lock); kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } @@ -646,13 +654,14 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, } static int setup_fuse_copy_state(struct fuse_copy_state *cs, - struct fuse_ring *ring, struct fuse_req *req, + struct fuse_req *req, struct fuse_ring_ent *ent, int dir, struct iov_iter *iter) { int err; - err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len, + iter); if (err) { pr_info_ratelimited("fuse: Import of user buffer failed\n"); return err; @@ -666,8 +675,7 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, return 0; } -static int fuse_uring_copy_from_ring(struct fuse_ring *ring, - struct fuse_req *req, +static int fuse_uring_copy_from_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -681,7 +689,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, if (err) return err; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter); if (err) return err; @@ -693,7 +701,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, /* * Copy data from the req to the ring buffer */ -static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, +static int fuse_uring_args_to_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -707,7 +715,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter); if (err) return err; @@ -737,6 +745,10 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) + ent_in_out.offset = + (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, &ent_in_out, sizeof(ent_in_out)); } @@ -745,7 +757,6 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, struct fuse_req *req) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; int err; err = -EIO; @@ -760,7 +771,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, return err; /* copy the request */ - err = fuse_uring_args_to_ring(ring, req, ent); + err = fuse_uring_args_to_ring(req, ent); if (unlikely(err)) { pr_info_ratelimited("Copy to ring failed: %d\n", err); return err; @@ -771,6 +782,91 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } +static bool fuse_uring_req_has_payload(struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + return args->in_numargs > 1 || args->out_numargs; +} + +static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool = queue->bufpool; + unsigned int id; + + lockdep_assert_held(&queue->lock); + + id = find_first_bit(pool->free_map, pool->nr_bufs); + if (id >= pool->nr_bufs) + return -ENOBUFS; + + WARN_ON_ONCE(ent->payload.iov_base); + __clear_bit(id, pool->free_map); + + ent->buf_id = id; + ent->payload.iov_base = + (