path: root/kernel/events/hw_breakpoint.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright (C) 2007 Alan Stern
 * Copyright (C) IBM Corporation, 2009
 * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
 *
 * Thanks to Ingo Molnar for his many suggestions.
 *
 * Authors: Alan Stern <stern@rowland.harvard.edu>
 *          K.Prasad <prasad@linux.vnet.ibm.com>
 *          Frederic Weisbecker <fweisbec@gmail.com>
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 * This file contains the arch-independent routines.
 */

#include <linux/hw_breakpoint.h>

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/cpu.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/irqflags.h>
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/notifier.h>
#include <linux/percpu-rwsem.h>
#include <linux/percpu.h>
#include <linux/rhashtable.h>
#include <linux/sched.h>
#include <linux/slab.h>

/*
 * Datastructure to track the total uses of N slots across tasks or CPUs;
 * bp_slots_histogram::count[N] is the number of assigned N+1 breakpoint slots.
 */
struct bp_slots_histogram {
#ifdef hw_breakpoint_slots
	atomic_t count[hw_breakpoint_slots(0)];
#else
	atomic_t *count;
#endif
};

/*
 * Per-CPU constraints data.
 */
struct bp_cpuinfo {
	/* Number of pinned CPU breakpoints in a CPU. */
	unsigned int			cpu_pinned;
	/* Histogram of pinned task breakpoints in a CPU. */
	struct bp_slots_histogram	tsk_pinned;
};

static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]);

static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)
{
	return per_cpu_ptr(bp_cpuinfo + type, cpu);
}

/* Number of pinned CPU breakpoints globally. */
static struct bp_slots_histogram cpu_pinned[TYPE_MAX];
/* Number of pinned CPU-independent task breakpoints. */
static struct bp_slots_histogram tsk_pinned_all[TYPE_MAX];

/* Keep track of the breakpoints attached to tasks */
static struct rhltable task_bps_ht;
static const struct rhashtable_params task_bps_ht_params = {
	.head_offset = offsetof(struct hw_perf_event, bp_list),
	.key_offset = offsetof(struct hw_perf_event, target),
	.key_len = sizeof_field(struct hw_perf_event, target),
	.automatic_shrinking = true,
};

static bool constraints_initialized __ro_after_init;

/*
 * Synchronizes accesses to the per-CPU constraints; the locking rules are:
 *
 *  1. Atomic updates to bp_cpuinfo::tsk_pinned only require a held read-lock
 *     (due to bp_slots_histogram::count being atomic, no update are lost).
 *
 *  2. Holding a write-lock is required for computations that require a
 *     stable snapshot of all bp_cpuinfo::tsk_pinned.
 *
 *  3. In all other cases, non-atomic accesses require the appropriately held
 *     lock (read-lock for read-only accesses; write-lock for reads/writes).
 */
DEFINE_STATIC_PERCPU_RWSEM(bp_cpuinfo_sem);

/*
 * Return mutex to serialize accesses to per-task lists in task_bps_ht. Since
 * rhltable synchronizes concurrent insertions/deletions, independent tasks may
 * insert/delete concurrently; therefore, a mutex per task is sufficient.
 *
 * Uses task_struct::perf_event_mutex, to avoid extending task_struct with a
 * hw_breakpoint-only mutex, which may be infrequently used. The caveat here is
 * that hw_breakpoint may contend with per-task perf event list management. The
 * assumption is that perf usecases involving hw_breakpoints are very unlikely
 * to result in unnecessary contention.
 */
static inline struct mutex *get_task_bps_mutex(struct perf_event *bp)
{
	struct task_struct *tsk = bp->hw.target;

	return tsk ? &tsk->perf_event_mutex : NULL;
}

static struct mutex *bp_constraints_lock(struct perf_event *bp)
{
	struct mutex *tsk_mtx = get_task_bps_mutex(bp);

	if (tsk_mtx) {
		/*
		 * Fully analogous to the perf_try_init_event() nesting
		 * argument in the comment near perf_event_ctx_lock_nested();
		 * this child->perf_event_mutex cannot ever deadlock against
		 * the parent->perf_event_mutex usage from
		 * perf_event_task_{en,dis}able().
		 *
		 * Specifically, inherited events will never occur on
		 * ->perf_event_list.
		 */
		mutex_lock_nested(tsk_mtx, SINGLE_DEPTH_NESTING);
		percpu_down_read(&bp_cpuinfo_sem);
	} else {
		percpu_down_write(&bp_cpuinfo_sem);
	}

	return tsk_mtx;
}

static void bp_constraints_unlock(struct mutex *tsk_mtx)
{
	if (tsk_mtx) {
		percpu_up_read(&bp_cpuinfo_sem);
		mutex_unlock(tsk_mtx);
	} else {
		percpu_up_write(&bp_cpuinfo_sem);
	}
}

static bool bp_constraints_is_locked(struct perf_event *bp)
{
	struct mutex *tsk_mtx = get_task_bps_mutex(bp);

	return percpu_is_write_locked(&bp_cpuinfo_sem) ||
	       (tsk_mtx ? mutex_is_locked(tsk_mtx) :
			  percpu_is_read_locked(&bp_cpuinfo_sem));
}

static inline void assert_bp_constraints_lock_held(struct perf_event *bp)
{
	struct mutex *tsk_mtx = get_task_bps_mutex(bp);

	if (tsk_mtx)
		lockdep_assert_held(tsk_mtx);
	lockdep_assert_held(&bp_cpuinfo_sem);
}

#ifdef hw_breakpoint_slots
/*
 * Number of breakpoint slots is constant, and the same for all types.
 */
static_assert(hw_breakpoint_slots(TYPE_INST) == hw_breakpoint_slots(TYPE_DATA));
static inline int hw_breakpoint_slots_cached(int type)	{ return hw_breakpoint_slots(type); }
static inline int init_breakpoint_slots(void)		{ return 0