// SPDX-License-Identifier: GPL-2.0
/*
* KFENCE guarded object allocator and fault handling.
*
* Copyright (C) 2020, Google LLC.
*/
#define pr_fmt(fmt) "kfence: " fmt
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/debugfs.h>
#include <linux/hash.h>
#include <linux/irq_work.h>
#include <linux/jhash.h>
#include <linux/kcsan-checks.h>
#include <linux/kfence.h>
#include <linux/kmemleak.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/log2.h>
#include <linux/memblock.h>
#include <linux/moduleparam.h>
#include <linux/nodemask.h>
#include <linux/notifier.h>
#include <linux/panic_notifier.h>
#include <linux/random.h>
#include <linux/rcupdate.h>
#include <linux/reboot.h>
#include <linux/sched/clock.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <asm/kfence.h>
#include "kfence.h"
/* Disables KFENCE on the first warning assuming an irrecoverable error. */
#define KFENCE_WARN_ON(cond) \
({ \
const bool __cond = WARN_ON(cond); \
if (unlikely(__cond)) { \
WRITE_ONCE(kfence_enabled, false); \
disabled_by_warn = true; \
} \
__cond; \
})
/* === Data ================================================================= */
static bool kfence_enabled __read_mostly;
static bool disabled_by_warn __read_mostly;
unsigned long kfence_sample_interval __read_mostly = CONFIG_KFENCE_SAMPLE_INTERVAL;
EXPORT_SYMBOL_GPL(kfence_sample_interval); /* Export for test modules. */
#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
#define MODULE_PARAM_PREFIX "kfence."
static int kfence_enable_late(void);
static int param_set_sample_interval(const char *val, const struct kernel_param *kp)
{
unsigned long num;
int ret = kstrtoul(val, 0, &num);
if (ret < 0)
return ret;
/* Using 0 to indicate KFENCE is disabled. */
if (!num && READ_ONCE(kfence_enabled)) {
pr_info("disabled\n");
WRITE_ONCE(kfence_enabled, false);
}
*((unsigned long *)kp->arg) = num;
if (num && !READ_ONCE(kfence_enabled) && system_state != SYSTEM_BOOTING)
return disabled_by_warn ? -EINVAL : kfence_enable_late();
return 0;
}
static int param_get_sample_interval(char *buffer, const struct kernel_param *kp)
{
if (!READ_ONCE(kfence_enabled))
return sprintf(buffer, "0\n");
