path: root/kernel/trace/fprobe.c

// SPDX-License-Identifier: GPL-2.0
/*
 * fprobe - Simple ftrace probe wrapper for function entry.
 */
#define pr_fmt(fmt) "fprobe: " fmt

#include <linux/cleanup.h>
#include <linux/err.h>
#include <linux/fprobe.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rhashtable.h>
#include <linux/slab.h>
#include <linux/sort.h>

#include <asm/fprobe.h>

#include "trace.h"

#define FPROBE_IP_HASH_BITS 8
#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS)

#define FPROBE_HASH_BITS 6
#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS)

#define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2))

/*
 * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still
 *   exists. The key is the address of fprobe instance.
 * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe
 *   instance related to the function address. The key is the ftrace IP
 *   address.
 *
 * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp
 * are set NULL and delete those from both hash tables (by hlist_del_rcu).
 * After an RCU grace period, the fprobe_hlist itself will be released.
 *
 * fprobe_table and fprobe_ip_table can be accessed from either
 *  - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held.
 *  - RCU hlist traversal under disabling preempt
 */
static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
static struct rhltable fprobe_ip_table;
static DEFINE_MUTEX(fprobe_mutex);
static struct fgraph_ops fprobe_graph_ops;

static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed)
{
	return hash_ptr(*(unsigned long **)data, 32);
}

static int fprobe_node_cmp(struct rhashtable_compare_arg *arg,
			   const void *ptr)
{
	unsigned long key = *(unsigned long *)arg->key;
	const struct fprobe_hlist_node *n = ptr;

	return n->addr != key;
}

static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed)
{
	const struct fprobe_hlist_node *n = data;

	return hash_ptr((void *)n->addr, 32);
}

static const struct rhashtable_params fprobe_rht_params = {
	.head_offset		= offsetof(struct fprobe_hlist_node, hlist),
	.key_offset		= offsetof(struct fprobe_hlist_node, addr),
	.key_len		= sizeof_field(struct fprobe_hlist_node, addr),
	.hashfn			= fprobe_node_hashfn,
	.obj_hashfn		= fprobe_node_obj_hashfn,
	.obj_cmpfn		= fprobe_node_cmp,
	.automatic_shrinking	= true,
};

/* Node insertion and deletion requires the fprobe_mutex */
static int __insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)
{
	int ret;

	lockdep_assert_held(&fprobe_mutex);

	ret = rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params);
	/* Set the fprobe pointer if insertion was successful. */
	if (!ret)
		WRITE_ONCE(node->fp, fp);
	return ret;
}

static void __delete_fprobe_node(struct fprobe_hlist_node *node)
{
	lockdep_assert_held(&fprobe_mutex);

	/* Avoid double deleting and non-inserted nodes */
	if (READ_ONCE(node->fp) != NULL) {
		WRITE_ONCE(node->fp, NULL);
		rhltable_remove(&fprobe_ip_table, &node->hlist,
				fprobe_rht_params);
	}
}

/* Check existence of the fprobe */
static bool fprobe_registered(struct fprobe *fp)
{
	struct hlist_head *head;
	struct fprobe_hlist *fph;

	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
	hlist_for_each_entry_rcu(fph, head, hlist,
				 lockdep_is_held(&fprobe_mutex)) {
		if (fph->fp == fp)
			return true;
	}
	return false;
}
NOKPROBE_SYMBOL(fprobe_registered);