// SPDX-License-Identifier: GPL-2.0
/*
* Some low level IO code, and hacks for various block layer limitations
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include "bcachefs.h"
#include "alloc_background.h"
#include "alloc_foreground.h"
#include "btree_update.h"
#include "buckets.h"
#include "checksum.h"
#include "clock.h"
#include "compress.h"
#include "data_update.h"
#include "disk_groups.h"
#include "ec.h"
#include "error.h"
#include "io_read.h"
#include "io_misc.h"
#include "io_write.h"
#include "subvolume.h"
#include "trace.h"
#include <linux/sched/mm.h>
#ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
static bool bch2_target_congested(struct bch_fs *c, u16 target)
{
const struct bch_devs_mask *devs;
unsigned d, nr = 0, total = 0;
u64 now = local_clock(), last;
s64 congested;
struct bch_dev *ca;
if (!target)
return false;
rcu_read_lock();
devs = bch2_target_to_mask(c, target) ?:
&c->rw_devs[BCH_DATA_user];
for_each_set_bit(d, devs->d, BCH_SB_MEMBERS_MAX) {
ca = rcu_dereference(c->devs[d]);
if (!ca)
continue;
congested = atomic_read(&ca->congested);
last = READ_ONCE(ca->congested_last);
if (time_after64(now, last))
congested -= (now - last) >> 12;
total += max(congested, 0LL);
nr++;
}
rcu_read_unlock();
return bch2_rand_range(nr * CONGESTED_MAX) < total;
}
#else
static bool bch2_target_congested(struct bch_fs *c, u16 target)
{
return false;
}
#endif
/* Cache promotion on read */
struct promote_op {
struct rcu_head rcu;
u64 start_time;
struct rhash_head hash;
struct bpos pos;
struct data_update write;
struct bio_vec bi_inline_vecs[]; /* must be last */
};
static const struct rhashtable_params bch_promote_params = {
.head_offset = offsetof(struct promote_op, hash),
.key_offset = offsetof(struct promote_op, pos),
.key_len = sizeof(struct bpos),
.automatic_shrinking = true,
};
static inline int should_promote