path: root/fs/btrfs/bio.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 * Copyright (C) 2022 Christoph Hellwig.
 */

#include <linux/bio.h>
#include "bio.h"
#include "ctree.h"
#include "volumes.h"
#include "raid56.h"
#include "async-thread.h"
#include "dev-replace.h"
#include "zoned.h"
#include "file-item.h"
#include "raid-stripe-tree.h"

static struct bio_set btrfs_bioset;
static struct bio_set btrfs_clone_bioset;
static struct bio_set btrfs_repair_bioset;
static mempool_t btrfs_failed_bio_pool;

struct btrfs_failed_bio {
	struct btrfs_bio *bbio;
	int num_copies;
	atomic_t repair_count;
};

/* Is this a data path I/O that needs storage layer checksum and repair? */
static inline bool is_data_bbio(struct btrfs_bio *bbio)
{
	return bbio->inode && is_data_inode(bbio->inode);
}

static bool bbio_has_ordered_extent(struct btrfs_bio *bbio)
{
	return is_data_bbio(bbio) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE;
}

/*
 * Initialize a btrfs_bio structure.  This skips the embedded bio itself as it
 * is already initialized by the block layer.
 */
void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info,
		    btrfs_bio_end_io_t end_io, void *private)
{
	memset(bbio, 0, offsetof(struct btrfs_bio, bio));
	bbio->fs_info = fs_info;
	bbio->end_io = end_io;
	bbio->private = private;
	atomic_set(&bbio->pending_ios, 1);
	WRITE_ONCE(bbio->status, BLK_STS_OK);
}

/*
 * Allocate a btrfs_bio structure.  The btrfs_bio is the main I/O container for
 * btrfs, and is used for all I/O submitted through btrfs_submit_bbio().
 *
 * Just like the underlying bio_alloc_bioset it will not fail as it is backed by
 * a mempool.
 */
struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf,
				  struct btrfs_fs_info *fs_info,
				  btrfs_bio_end_io_t end_io, void *private)
{
	struct btrfs_bio *bbio;
	struct bio *bio;

	bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset);
	bbio = btrfs_bio(bio);
	btrfs_bio_init(bbio, fs_info, end_io, private);
	return bbio;
}

static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info,
					 struct btrfs_bio *orig_bbio,
					 u64 map_length)
{
	struct btrfs_bio *bbio;
	struct bio *bio;

	bio = bio_split(&orig_bbio->bio, map_length >> SECTOR_SHIFT, GFP_NOFS,
			&btrfs_clone_bioset);
	if (IS_ERR(bio))
		return ERR_CAST(bio);

	bbio = btrfs_bio(bio);
	btrfs_bio_init(bbio, fs_info, NULL, orig_bbio);
	bbio->inode = orig_bbio->inode;
	bbio->file_offset = orig_bbio->file_offset;
	orig_bbio->file_offset += map_length;
	if (bbio_has_ordered_extent(bbio)) {
		refcount_inc(&orig_bbio->ordered->refs);
		bbio->ordered = orig_bbio->ordered;
	}
	atomic_inc(&orig_bbio->pending_ios);
	return bbio;
}

/* Free a bio that was never submitted to the underlying device. */
static void btrfs_cleanup_bio(struct btrfs_bio *bbio)
{
	if (bbio_has_ordered_extent(bbio))
		btrfs_put_ordered_extent(bbio->ordered);
	bio_put(&bbio->bio);
}

static void __btrfs_bio_end_io(struct btrfs_bio *bbio)
{
	if (bbio_has_ordered_extent(bbio)) {
		struct btrfs_ordered_extent *ordered = bbio->ordered;

		bbio->end_io(bbio);
		btrfs_put_ordered_extent(ordered);
	} else {
		bbio->end_io(bbio);
	}
}

void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status)
{
	bbio->bio.bi_status = status;
	if (bbio->bio.bi_pool == &btrfs_clone_bioset) {
		struct btrfs_bio *orig_bbio = bbio->private;

		btrfs_cleanup_bio(bbio);
		bbio = orig_bbio;
	}

	/*
	 * At this point, bbio always points to the original btrfs_bio. Save
	 * the first error in it.
	 */
	if (status != BLK_STS_OK)
		cmpxchg(&bbio->status, BLK_STS_OK, status);

	if (atomic_dec_and_test(&bbio->pending_ios)) {
		/* Load split bio's error which might be set above. */
		if (status == BLK_STS_OK)
			bbio->bio.bi_status = READ_ONCE(bbio->status);
		__btrfs_bio_end_io(bbio);
	}
}

static int next_repair_mirror(struct btrfs_failed_bio *fbio, int cur_mirror)
{
	if (cur_mirror == fbio->num_copies)
		return cur_mirror + 1 - fbio->num_copies;
	return cur_mirror + 1;
}

static int prev_repair_mirror(struct btrfs_failed_bio *fbio, int cur_mirror