path: root/fs/iomap/ioend.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2016-2025 Christoph Hellwig.
 */
#include <linux/bio-integrity.h>
#include <linux/iomap.h>
#include <linux/list_sort.h>
#include <linux/pagemap.h>
#include <linux/writeback.h>
#include <linux/fserror.h>
#include "internal.h"
#include "trace.h"

struct bio_set iomap_ioend_bioset;
EXPORT_SYMBOL_GPL(iomap_ioend_bioset);

struct iomap_ioend *iomap_init_ioend(struct inode *inode,
		struct bio *bio, loff_t file_offset, u16 ioend_flags)
{
	struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);

	atomic_set(&ioend->io_remaining, 1);
	ioend->io_error = 0;
	ioend->io_parent = NULL;
	INIT_LIST_HEAD(&ioend->io_list);
	ioend->io_flags = ioend_flags;
	ioend->io_inode = inode;
	ioend->io_offset = file_offset;
	ioend->io_size = bio->bi_iter.bi_size;
	ioend->io_sector = bio->bi_iter.bi_sector;
	ioend->io_private = NULL;
	return ioend;
}
EXPORT_SYMBOL_GPL(iomap_init_ioend);

/*
 * We're now finished for good with this ioend structure.  Update the folio
 * state, release holds on bios, and finally free up memory.  Do not use the
 * ioend after this.
 */
static u32 iomap_finish_ioend_buffered_write(struct iomap_ioend *ioend)
{
	struct inode *inode = ioend->io_inode;
	struct bio *bio = &ioend->io_bio;
	struct folio_iter fi;
	u32 folio_count = 0;

	if (ioend->io_error) {
		mapping_set_error(inode->i_mapping, ioend->io_error);
		if (!bio_flagged(bio, BIO_QUIET)) {
			pr_err_ratelimited(
"%s: writeback error on inode %llu, offset %lld, sector %llu",
				inode->i_sb->s_id, inode->i_ino,
				ioend->io_offset, ioend->io_sector);
		}
	}

	/* walk all folios in bio, ending page IO on them */
	bio_for_each_folio_all(fi, bio) {
		if (ioend->io_error)
			fserror_report_io(inode, FSERR_BUFFERED_WRITE,
					  folio_pos(fi.folio) + fi.offset,
					  fi.length, ioend->io_error,
					  GFP_ATOMIC);
		iomap_finish_folio_write(inode, fi.folio, fi.length);
		folio_count++;
	}

	if (bio_integrity(bio))
		fs_bio_integrity_free(bio);
	bio_put(bio);	/* frees the ioend */
	return folio_count;
}

static DEFINE_SPINLOCK(failed_ioend_lock);
static LIST_HEAD(failed_ioend_list);

static void
iomap_fail_ioends(
	struct work_struct	*work)
{
	struct iomap_ioend	*ioend;
	struct list_head	tmp;
	unsigned long		flags;

	spin_lock_irqsave(&failed_ioend_lock, flags);
	list_replace_init(&failed_ioend_list, &tmp);
	spin_unlock_irqrestore(&failed_ioend_lock, flags);

	while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
			io_list))) {
		list_del_init(&ioend->io_list);
		iomap_finish_ioend_buffered_write(ioend);
		cond_resched();
	}
}

static DECLARE_WORK(failed_ioend_work, iomap_fail_ioends);

static void iomap_fail_ioend_buffered(struct iomap_ioend *ioend)
{
	unsigned long flags;

	/*
	 * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions
	 * in the fserror code.  The caller no longer owns the ioend reference
	 * after the spinlock drops.
	 */
	spin_lock_irqsave(&failed_ioend_lock, flags);
	if (list_empty(&failed_ioend_list))
		WARN_ON_ONCE(!schedule_work(&failed_ioend_work));
	list_add_tail(&ioend->io_list, &failed_ioend_list);
	spin_unlock_irqrestore(&failed_ioend_lock, flags);
}

static void ioend_writeback_end_bio(struct bio *bio)
{
	struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);

	ioend->io_error = blk_status_to_errno(bio->bi_status);
	if (ioend->io_error) {
		iomap_fail_ioend_buffered(ioend);
		return;
	}

	iomap_finish_ioend_buffered_write(ioend);
}

/*
 * We cannot cancel the ioend directly in case of an error, so call the bio end
 * I/O handler with the error status here to run the normal I/O completion
 * handler.
 */
int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
{
	struct iomap_ioend *ioend = wpc->wb_ctx;

	if (!ioend->io_bio.bi_end_io)
		ioend->io_bio.bi_end_io = ioend_writeback_end_bio;

	if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
		error = -EIO;

	if (error) {
		ioend->io_bio.bi_status = errno_to_blk_status(error);
		bio_endio(&ioend->io_bio);
		return error;
	}

	if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
		fs_bio_integrity_generate(&ioend->io_bio);
	submit_bio(&ioend->io_bio);
	return 0;
}
EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit);

static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
		loff_t pos, u16 ioend_flags)
{
	struct bio *bio;

	bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
			       REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
			       GFP_NOFS, &iomap_ioend_bioset);
	bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
	bio->bi_write_hint = wpc->inode->i_write_hint;
	wbc_init_bio(wpc->wbc, bio);
	wpc->nr_folios = 0;
	return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
}

static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
		unsigned int map_len, u16 ioend_flags)
{
	struct iomap_ioend *ioend = wpc->wb_ctx;

	if (ioend->io_bio.bi_iter.bi_size >
	    iomap_max_bio_size(&wpc->iomap) - map_len)
		return false;
	if (ioend_flags & IOMAP_IOEND_BOUNDARY)
		return false;
	if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
	    (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
		return false;
	if (pos != ioend->io_offset + ioend->io_size)
		return false;
	if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
	    iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
		return false;
	/*
	 * Limit ioend bio chain lengths to minimise IO completion latency. This
	 * also prevents long tight loops ending page writeback on all the
	 * folios in the ioend.
	 */
	if (wpc->nr_folios >= IOEND_BATCH_SIZE)
		return false;
	return true;
}

/*
 * Test to see if we have an existing ioend structure that we could append to
 * first; otherwise finish off the current ioend and start another.
 *
 * If a new ioend is created and cached, the old ioend is submitted to the block
 * layer instantly.  Batching optimisations are provided by higher level block
 * plugging.
 *
 * At the end of a writeback pass, there will be a cached ioend remaining on the
 * writepage context that the caller will need to submit.
 */
ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
		loff_t pos, loff_t end_pos, unsigned int dirty_len)
{
	struct iomap_ioend *ioend = wpc->wb_ctx;
	size_t poff = offset_in_folio(folio, pos);
	unsigned int ioend_flags = 0;
	unsigned int map_len = min_t(u64, dirty_len,
		wpc->iomap.offset + wpc->iomap.length - pos);
	int error;

	trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);

	WARN_ON_ONCE(!folio->private && map_len < dirty_len);

	switch (wpc->iomap.type) {
	case IOMAP_UNWRITTEN:
		ioend_flags