path: root/mm/page_owner.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/debugfs.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/memblock.h>
#include <linux/stacktrace.h>
#include <linux/page_owner.h>
#include <linux/jump_label.h>
#include <linux/migrate.h>
#include <linux/stackdepot.h>
#include <linux/seq_file.h>
#include <linux/memcontrol.h>
#include <linux/sched/clock.h>

#include "internal.h"

/*
 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
 * to use off stack temporal storage
 */
#define PAGE_OWNER_STACK_DEPTH (16)

struct page_owner {
	unsigned short order;
	short last_migrate_reason;
	gfp_t gfp_mask;
	depot_stack_handle_t handle;
	depot_stack_handle_t free_handle;
	u64 ts_nsec;
	u64 free_ts_nsec;
	char comm[TASK_COMM_LEN];
	pid_t pid;
	pid_t tgid;
	pid_t free_pid;
	pid_t free_tgid;
};

struct stack {
	struct stack_record *stack_record;
	struct stack *next;
};
static struct stack dummy_stack;
static struct stack failure_stack;
static struct stack *stack_list;
static DEFINE_SPINLOCK(stack_list_lock);

#define STACK_PRINT_FLAG_STACK		0x1
#define STACK_PRINT_FLAG_PAGES		0x2
#define STACK_PRINT_FLAG_HANDLE		0x4

struct stack_print_ctx {
	struct stack *stack;
	u8 flags;
};

static bool page_owner_enabled __initdata;
DEFINE_STATIC_KEY_FALSE(page_owner_inited);

static depot_stack_handle_t dummy_handle;
static depot_stack_handle_t failure_handle;
static depot_stack_handle_t early_handle;

static void init_early_allocated_pages(void);

static inline void set_current_in_page_owner(void)
{
	/*
	 * Avoid recursion.
	 *
	 * We might need to allocate more memory from page_owner code, so make
	 * sure to signal it in order to avoid recursion.
	 */
	current->in_page_owner = 1;
}

static inline void unset_current_in_page_owner(void)
{
	current->in_page_owner = 0;
}

static int __init early_page_owner_param(char *buf)
{
	int ret = kstrtobool(buf, &page_owner_enabled);

	if (page_owner_enabled)
		stack_depot_request_early_init();

	return ret;
}
early_param("page_owner", early_page_owner_param);

static __init bool need_page_owner(void)
{
	return page_owner_enabled;
}

static __always_inline depot_stack_handle_t create_dummy_stack(void)
{
	unsigned long entries[4];
	unsigned int nr_entries;

	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
	return stack_depot_save(entries, nr_entries, GFP_KERNEL);
}

static noinline void register_dummy_stack(void)
{
	dummy_handle = create_dummy_stack();
}

static noinline void register_failure_stack(void)
{
	failure_handle = create_dummy_stack();
}

static noinline void register_early_stack(void)
{
	early_handle = create_dummy_stack();
}

static __init void init_page_owner(void)
{
	if (!page_owner_enabled)
		return;

	register_dummy_stack();
	register_failure_stack();
	register_early_stack();
	init_early_allocated_pages();
	/* Initialize dummy and failure stacks and link them to stack_list */
	dummy_stack.stack_record = __stack_depot_get_stack_record(dummy_handle);
	failure_stack.stack_record = __stack_depot_get_stack_record(failure_handle);
	if (dummy_stack.stack_record)
		refcount_set(&dummy_stack.stack_record->count, 1);
	if (failure_stack.stack_record)
		refcount_set(&failure_stack.stack_record->count, 1);
	dummy_stack.next = &failure_stack;
	stack_list = &dummy_stack;
	static_branch_enable(&page_owner_inited);
}

struct page_ext_operations page_owner_ops = {
	.size = sizeof(struct page_owner),
	.need = need_page_owner,
	.init = init_page_owner,
	.need_shared_flags = true,
};

static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
{
	return page_ext_data(page_ext, &page_owner_ops);
}

static noinline depot_stack_handle_t save_stack(gfp_t flags)
{
	unsigned long entries[PAGE_OWNER_STACK_DEPTH];
	depot_stack_handle_t handle;
	unsigned int nr_entries;

	if (current->in_page_owner)
		return dummy_handle;

	set_current_in_page_owner();
	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
	handle = stack_depot_save(entries, nr_entries, flags);
	if (!handle)
		handle = failure_handle;
	unset_current_in_page_owner();

	return handle;
}

static void add_stack_record_to_list(struct stack_record *stack_record,
				     gfp_t