path: root/kernel/bpf/bpf_local_storage.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 Facebook  */
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/btf_ids.h>
#include <linux/bpf_local_storage.h>
#include <net/sock.h>
#include <uapi/linux/sock_diag.h>
#include <uapi/linux/btf.h>
#include <linux/rcupdate.h>
#include <linux/rcupdate_trace.h>
#include <linux/rcupdate_wait.h>

#define BPF_LOCAL_STORAGE_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_CLONE)

static struct bpf_local_storage_map_bucket *
select_bucket(struct bpf_local_storage_map *smap,
	      struct bpf_local_storage_elem *selem)
{
	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];
}

static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)
{
	struct bpf_map *map = &smap->map;

	if (!map->ops->map_local_storage_charge)
		return 0;

	return map->ops->map_local_storage_charge(smap, owner, size);
}

static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner,
			 u32 size)
{
	struct bpf_map *map = &smap->map;

	if (map->ops->map_local_storage_uncharge)
		map->ops->map_local_storage_uncharge(smap, owner, size);
}

static struct bpf_local_storage __rcu **
owner_storage(struct bpf_local_storage_map *smap, void *owner)
{
	struct bpf_map *map = &smap->map;

	return map->ops->map_owner_storage_ptr(owner);
}

static bool selem_linked_to_storage_lockless(const struct bpf_local_storage_elem *selem)
{
	return !hlist_unhashed_lockless(&selem->snode);
}

static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)
{
	return !hlist_unhashed(&selem->snode);
}

static bool selem_linked_to_map_lockless(const struct bpf_local_storage_elem *selem)
{
	return !hlist_unhashed_lockless(&selem->map_node);
}

static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)
{
	return !hlist_unhashed(&selem->map_node);
}

struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
		void *value, bool charge_mem, bool swap_uptrs, gfp_t gfp_flags)
{
	struct bpf_local_storage_elem *selem;

	if (charge_mem && mem_charge(smap, owner, smap->elem_size))
		return NULL;

	if (smap->bpf_ma) {
		migrate_disable();
		selem = bpf_mem_cache_alloc_flags(&smap->selem_ma, gfp_flags);
		migrate_enable();
		if (selem)
			/* Keep the original bpf_map_kzalloc behavior
			 * before started using the bpf_mem_cache_alloc.
			 *
			 * No need to use zero_map_value. The bpf_selem_free()
			 * only does bpf_mem_cache_free when there is
			 * no other bpf prog is using the selem.
			 */
			memset(SDATA(selem)->data, 0, smap->map.value_size);
	} else {
		selem = bpf_map_kzalloc(&smap->map, smap->elem_size,
					gfp_flags | __GFP_NOWARN);
	}

	if (selem) {
		if (value) {
			/* No need to call check_and_init_map_value as memory is zero init */
			copy_map_value(&smap->map, SDATA(selem)->data, value);
			if (swap_uptrs)
				bpf_obj_swap_uptrs(smap->map.record, SDATA(selem)->data, value);
		}
		return selem;
	}

	if (charge_mem)
		mem_uncharge(smap, owner, smap->elem_size);

	return NULL;
}

/* rcu tasks trace callback for bpf_ma == false */
static void __bpf_local_storage_free_trace_rcu(struct rcu_head *rcu)
{
	struct bpf_local_storage *local_storage;

	/* If RCU Tasks Trace grace period implies RCU grace period, do
	 * kfree(), else do kfree_rcu().
	 */
	local_storage = container_of(rcu, struct bpf_local_storage, rcu);
	if (rcu_trace_implies_rcu_gp())
		kfree(local_storage);
	else
		kfree_rcu(local_storage, rcu);
}

static void bpf_local_storage_free_rcu(struct rcu_head *rcu)
{
	struct bpf_local_storage *local_storage;

	local_storage = container_of(rcu, struct bpf_local_storage, rcu);
	bpf_mem_cache_raw_free(local_storage);
}

static void bpf_local_storage_free_trace_rcu(struct rcu_head *rcu)
{
	if (rcu_trace_implies_rcu_gp())
		bpf_local_storage_free_rcu(rcu);
	else
		call_rcu(rcu, bpf_local_storage_free_rcu);
}

/* Handle bpf_ma == false */
static void __bpf_local_storage_free(struct bpf_local_storage *local_storage,
				     bool vanilla_rcu)
{
	if (vanilla_rcu)
		kfree_rcu(local_storage, rcu);
	else
		call_rcu_tasks_trace(&local_storage->rcu,
				     __bpf_local_stor