path: root/kernel/rcu/tree_stall.h

// SPDX-License-Identifier: GPL-2.0+
/*
 * RCU CPU stall warnings for normal RCU grace periods
 *
 * Copyright IBM Corporation, 2019
 *
 * Author: Paul E. McKenney <paulmck@linux.ibm.com>
 */

#include <linux/console.h>
#include <linux/kvm_para.h>
#include <linux/rcu_notifier.h>
#include <linux/smp.h>

//////////////////////////////////////////////////////////////////////////////
//
// Controlling CPU stall warnings, including delay calculation.

/* panic() on RCU Stall sysctl. */
static int sysctl_panic_on_rcu_stall __read_mostly;
static int sysctl_max_rcu_stall_to_panic __read_mostly;

static const struct ctl_table rcu_stall_sysctl_table[] = {
	{
		.procname	= "panic_on_rcu_stall",
		.data		= &sysctl_panic_on_rcu_stall,
		.maxlen		= sizeof(sysctl_panic_on_rcu_stall),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= SYSCTL_ZERO,
		.extra2		= SYSCTL_ONE,
	},
	{
		.procname	= "max_rcu_stall_to_panic",
		.data		= &sysctl_max_rcu_stall_to_panic,
		.maxlen		= sizeof(sysctl_max_rcu_stall_to_panic),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_minmax,
		.extra1		= SYSCTL_ONE,
		.extra2		= SYSCTL_INT_MAX,
	},
};

static int __init init_rcu_stall_sysctl(void)
{
	register_sysctl_init("kernel", rcu_stall_sysctl_table);
	return 0;
}

subsys_initcall(init_rcu_stall_sysctl);

#ifdef CONFIG_SYSFS

static unsigned int rcu_stall_count;

static ssize_t rcu_stall_count_show(struct kobject *kobj, struct kobj_attribute *attr,
				    char *page)
{
	return sysfs_emit(page, "%u\n", rcu_stall_count);
}

static struct kobj_attribute rcu_stall_count_attr = __ATTR_RO(rcu_stall_count);

static __init int kernel_rcu_stall_sysfs_init(void)
{
	sysfs_add_file_to_group(kernel_kobj, &rcu_stall_count_attr.attr, NULL);
	return 0;
}

late_initcall(kernel_rcu_stall_sysfs_init);

#endif // CONFIG_SYSFS

#ifdef CONFIG_PROVE_RCU
#define RCU_STALL_DELAY_DELTA		(5 * HZ)
#else
#define RCU_STALL_DELAY_DELTA		0
#endif
#define RCU_STALL_MIGHT_DIV		8
#define RCU_STALL_MIGHT_MIN		(2 * HZ)

int rcu_exp_jiffies_till_stall_check(void)
{
	int cpu_stall_timeout = READ_ONCE(rcu_exp_cpu_stall_timeout);
	int exp_stall_delay_delta = 0;
	int till_stall_check;

	// Zero says to use rcu_cpu_stall_timeout, but in milliseconds.
	if (!cpu_stall_timeout)
		cpu_stall_timeout = jiffies_to_msecs(rcu_jiffies_till_stall_check());

	// Limit check must be consistent with the Kconfig limits for
	// CONFIG_RCU_EXP_CPU_STALL_TIMEOUT, so check the allowed range.
	// The minimum clamped value is "2UL", because at least one full
	// tick has to be guaranteed.
	till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 300UL * HZ);

	if (cpu_stall_timeout && jiffies_to_msecs(till_stall_check) != cpu_stall_timeout)
		WRITE_ONCE(rcu_exp_cpu_stall_timeout, jiffies_to_msecs(till_stall_check));

#ifdef CONFIG_PROVE_RCU
	/* Add extra ~25% out of till_stall_check. */
	exp_stall_delay_delta = ((till_stall_check * 25) / 100) + 1;
#endif

	return till_stall_check + exp_stall_delay_delta;
}
EXPORT_SYMBOL_GPL(rcu_exp_jiffies_till_stall_check);

/* Limit-check stall timeouts specified at boottime and runtime. */
int rcu_jiffies_till_stall_check(void)
{
	int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);

	/*
	 * Limit check must be consistent with the Kconfig limits
	 * for CONFIG_RCU_CPU_STALL_TIMEOUT.
	 */
	if (till_stall_check < 3) {
		WRITE_ONCE(rcu_cpu_stall_timeout, 3);
		till_stall_check = 3;
	} els