path: root/kernel/trace/fprobe.c

// SPDX-License-Identifier: GPL-2.0
/*
 * fprobe - Simple ftrace probe wrapper for function entry.
 */
#define pr_fmt(fmt) "fprobe: " fmt

#include <linux/err.h>
#include <linux/fprobe.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rhashtable.h>
#include <linux/slab.h>
#include <linux/sort.h>

#include <asm/fprobe.h>

#include "trace.h"

#define FPROBE_IP_HASH_BITS 8
#define FPROBE_IP_TABLE_SIZE (1 << FPROBE_IP_HASH_BITS)

#define FPROBE_HASH_BITS 6
#define FPROBE_TABLE_SIZE (1 << FPROBE_HASH_BITS)

#define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2))

/*
 * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still
 *   exists. The key is the address of fprobe instance.
 * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe
 *   instance related to the function address. The key is the ftrace IP
 *   address.
 *
 * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp
 * are set NULL and delete those from both hash tables (by hlist_del_rcu).
 * After an RCU grace period, the fprobe_hlist itself will be released.
 *
 * fprobe_table and fprobe_ip_table can be accessed from either
 *  - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held.
 *  - RCU hlist traversal under disabling preempt
 */
static struct hlist_head fprobe_table[FPROBE_TABLE_SIZE];
static struct rhltable fprobe_ip_table;
static DEFINE_MUTEX(fprobe_mutex);
static struct fgraph_ops fprobe_graph_ops;

static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed)
{
	return hash_ptr(*(unsigned long **)data, 32);
}

static int fprobe_node_cmp(struct rhashtable_compare_arg *arg,
			   const void *ptr)
{
	unsigned long key = *(unsigned long *)arg->key;
	const struct fprobe_hlist_node *n = ptr;

	return n->addr != key;
}

static u32 fprobe_node_obj_hashfn(const void *data, u32 len, u32 seed)
{
	const struct fprobe_hlist_node *n = data;

	return hash_ptr((void *)n->addr, 32);
}

static const struct rhashtable_params fprobe_rht_params = {
	.head_offset		= offsetof(struct fprobe_hlist_node, hlist),
	.key_offset		= offsetof(struct fprobe_hlist_node, addr),
	.key_len		= sizeof_field(struct fprobe_hlist_node, addr),
	.hashfn			= fprobe_node_hashfn,
	.obj_hashfn		= fprobe_node_obj_hashfn,
	.obj_cmpfn		= fprobe_node_cmp,
	.automatic_shrinking	= true,
};

/* Node insertion and deletion requires the fprobe_mutex */
static int insert_fprobe_node(struct fprobe_hlist_node *node)
{
	lockdep_assert_held(&fprobe_mutex);

	return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params);
}

/* Return true if there are synonims */
static bool delete_fprobe_node(struct fprobe_hlist_node *node)
{
	lockdep_assert_held(&fprobe_mutex);
	bool ret;

	/* Avoid double deleting */
	if (READ_ONCE(node->fp) != NULL) {
		WRITE_ONCE(node->fp, NULL);
		rhltable_remove(&fprobe_ip_table, &node->hlist,
				fprobe_rht_params);
	}

	rcu_read_lock();
	ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr,
				fprobe_rht_params);
	rcu_read_unlock();

	return ret;
}

/* Check existence of the fprobe */
static bool is_fprobe_still_exist(struct fprobe *fp)
{
	struct hlist_head *head;
	struct fprobe_hlist *fph;

	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
	hlist_for_each_entry_rcu(fph, head, hlist,
				 lockdep_is_held(&fprobe_mutex)) {
		if (fph->fp == fp)
			return true;
	}
	return false;
}
NOKPROBE_SYMBOL(is_fprobe_still_exist);

static int add_fprobe_hash(struct fprobe *fp)
{
	struct fprobe_hlist *fph = fp->hlist_array;
	struct hlist_head *head;

	lockdep_assert_held(&fprobe_mutex);

	if (WARN_ON_ONCE(!fph))
		return -EINVAL;

	if (is_fprobe_still_exist(fp))
		return -EEXIST;

	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];
	hlist_add_head_rcu(&fp->hlist_array->hlist, head);
	return 0;
}

static int del_fprobe_hash(struct fprobe *fp)
{
	struct fprobe_hlist *fph = fp->hlist_array;

	lockdep_assert_held(&fprobe_mutex);

	if (WARN_ON_ONCE(!fph))
		return -EINVAL;

	if (!is_fprobe_still_exist(fp))
		return -ENOENT;

	fph->fp = NULL;
	hlist_del_rcu(&fph->hlist);
	return 0;
}

#ifdef ARCH_DEFINE_ENCODE_FPROBE_HEADER

/* The arch should encode fprobe_header info into one unsigned long */
#define FPROBE_HEADER_SIZE_IN_LONG	1

static inline bool write_fprobe_header(unsigned long *stack,
					struct fprobe *fp, unsigned int size_words)