path: root/fs/bcachefs/btree_key_cache.c

// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"
#include "btree_cache.h"
#include "btree_iter.h"
#include "btree_key_cache.h"
#include "btree_locking.h"
#include "btree_update.h"
#include "errcode.h"
#include "error.h"
#include "journal.h"
#include "journal_reclaim.h"
#include "trace.h"

#include <linux/sched/mm.h>

static inline bool btree_uses_pcpu_readers(enum btree_id id)
{
	return id == BTREE_ID_subvolumes;
}

static struct kmem_cache *bch2_key_cache;

static int bch2_btree_key_cache_cmp_fn(struct rhashtable_compare_arg *arg,
				       const void *obj)
{
	const struct bkey_cached *ck = obj;
	const struct bkey_cached_key *key = arg->key;

	return ck->key.btree_id != key->btree_id ||
		!bpos_eq(ck->key.pos, key->pos);
}

static const struct rhashtable_params bch2_btree_key_cache_params = {
	.head_offset		= offsetof(struct bkey_cached, hash),
	.key_offset		= offsetof(struct bkey_cached, key),
	.key_len		= sizeof(struct bkey_cached_key),
	.obj_cmpfn		= bch2_btree_key_cache_cmp_fn,
	.automatic_shrinking	= true,
};

static inline void btree_path_cached_set(struct btree_trans *trans, struct btree_path *path,
					 struct bkey_cached *ck,
					 enum btree_node_locked_type lock_held)
{
	path->l[0].lock_seq	= six_lock_seq(&ck->c.lock);
	path->l[0].b		= (void *) ck;
	mark_btree_node_locked(trans, path, 0, lock_held);
}

__flatten
inline struct bkey_cached *
bch2_btree_key_cache_find(struct bch_fs *c, enum btree_id btree_id, struct bpos pos)
{
	struct bkey_cached_key key = {
		.btree_id	= btree_id,
		.pos		= pos,
	};

	return rhashtable_lookup_fast(&c->btree_key_cache.table, &key,
				      bch2_btree_key_cache_params);
}

static bool bkey_cached_lock_for_evict(struct bkey_cached *ck)
{
	if (!six_trylock_intent(&ck->c.lock))
		return false;

	if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) {
		six_unlock_intent(&ck->c.lock);
		return false;
	}

	if (!six_trylock_write(&ck->c.lock)) {
		six_unlock_intent(&ck->c.lock);
		return false;
	}

	return true;
}

static bool bkey_cached_evict(struct btree_key_cache *c,
			      struct bkey_cached *ck)
{
	bool ret = !rhashtable_remove_fast(&c->table, &ck->hash,
				      bch2_btree_key_cache_params);
	if (ret) {
		memset(&ck->key, ~0, sizeof(ck->key));
		atomic_long_dec(&c->nr_keys);
	}

	return ret;
}

static void __bkey_cached_free(struct rcu_pending *pending, struct rcu_head *rcu)
{
	struct bch_fs *c = container_of(pending->srcu, struct bch_fs, btree_trans_barrier);
	struct bkey_cached *ck = container_of(rcu, struct bkey_cached, rcu);

	this_cpu_dec(*c->btree_key_cache.nr_pending);
	kmem_cache_free(bch2_key_cache, ck);
}

static void bkey_cached_free(struct btree_key_cache *bc,
			     struct bkey_cached *ck)
{
	kfree(ck->k);
	ck->k		= NULL;
	ck->u64s	= 0;

	six_unlock_write(&ck->c.lock);
	six_unlock_intent(&ck->c.lock);

	bool pcpu_readers = ck->c.lock.readers != NULL;
	rcu_pending_enqueue(&bc->pending[pcpu_readers], &ck->rcu);
	this_cpu_inc(*bc->nr_pending);
}

static struct bkey_cached *__bkey_cached_alloc(unsigned key_u64s, gfp_t gfp)
{
	gfp |= __GFP_ACCOUNT|__GFP_RECLAIMABLE;

	struct bkey_cached *ck = kmem_cache_zalloc(bch2_key_cache, gfp);
	if (unlikely(!ck))
		return NULL;
	ck->k = kmalloc(key_u64s * sizeof(u64), gfp);
	if (unlikely(!ck->k)) {
		kmem_cache_free(bch2_key_cache, ck);
		return NULL;
	}
	ck->u64s = key_u64s;
	return ck;
}

static struct bkey_cached *
bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, unsigned key_u64s)
{
	struct bch_fs *c = trans->c;
	struct btree_key_cache *bc = &c->btree_key_cache;
	bool pcpu_readers = btree_uses_pcpu_readers(path->btree_id);
	int ret;

	struct bkey_cached *ck = container_of_or_null(
				rcu_pending_dequeue(&bc->pending[pcpu_readers]),
				struct bkey_cached, rcu);
	if (ck)
		goto lock;

	ck = allocate_dropping_locks(trans, ret,
				     __bkey_cached_alloc(key_u64s, _gfp));
	if (ret) {
		<