path: root/kernel/cgroup/dmem.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright 2023-2024 Intel Corporation (Maarten Lankhorst <dev@lankhorst.se>)
 * Copyright 2024 Red Hat (Maxime Ripard <mripard@kernel.org>)
 * Partially based on the rdma and misc controllers, which bear the following copyrights:
 *
 * Copyright 2020 Google LLC
 * Copyright (C) 2016 Parav Pandit <pandit.parav@gmail.com>
 */

#include <linux/cgroup.h>
#include <linux/cgroup_dmem.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/page_counter.h>
#include <linux/parser.h>
#include <linux/refcount.h>
#include <linux/rculist.h>
#include <linux/slab.h>
#include <linux/srcu.h>

/* Maximum reclaim attempts before giving up when lowering dmem.max. */
#define DMEM_MAX_RECLAIM_RETRIES 16

/* SRCU domain serialising reclaim callbacks against region unregistration. */
DEFINE_STATIC_SRCU(dmemcg_srcu);

struct dmem_cgroup_region {
	/**
	 * @ref: References keeping the region alive.
	 * Keeps the region reference alive after a succesful RCU lookup.
	 */
	struct kref ref;

	/** @rcu: RCU head for freeing */
	struct rcu_head rcu;

	/**
	 * @region_node: Linked into &dmem_cgroup_regions list.
	 * Protected by RCU and global spinlock.
	 */
	struct list_head region_node;

	/**
	 * @pools: List of pools linked to this region.
	 * Protected by global spinlock only
	 */
	struct list_head pools;

	/** @size: Size of region, in bytes */
	u64 size;

	/** @name: Name describing the node, set by dmem_cgroup_register_region */
	char *name;

	/**
	 * @unregistered: Whether the region is unregistered by its caller.
	 * No new pools should be added to the region afterwards, and no new
	 * reclaim callbacks should be invoked.
	 */
	bool unregistered;

	/**
	 * @ops: Optional driver operations for this region.
	 */
	const struct dmem_cgroup_ops *ops;

	/** @reclaim_priv: Private data passed to @ops->reclaim. */
	void *reclaim_priv;
};

struct dmemcg_state {
	struct cgroup_subsys_state css;

	struct list_head pools;
};

struct dmem_cgroup_pool_state {
	struct dmem_cgroup_region *region;
	struct dmemcg_state *cs;

	/* css node, RCU protected against region teardown */
	struct list_head	css_node;

	/* dev node, no RCU protection required */
	struct list_head	region_node;

	struct rcu_head rcu;

	struct page_counter cnt;
	struct dmem_cgroup_pool_state *parent;

	refcount_t ref;
	bool inited;
};

/*
 * 3 operations require locking protection:
 * - Registering and unregistering region to/from list, requires global lock.
 * - Adding a dmem_cgroup_pool_state to a CSS, removing when CSS is freed.
 * - Adding a dmem_cgroup_pool_state to a region list.
 *
 * Since for the most common operations RCU provides enough protection, I
 * do not think more granular locking makes sense. Most protection is offered
 * by RCU and the lockless operating page_counter.
 */
static DEFINE_SPINLOCK(dmemcg_lock);
static LIST_HEAD(dmem_cgroup_regions);

static void dmemcg_free_region(struct kref *ref);
static void dmemcg_pool_free_rcu(struct rcu_head *rcu);

static inline struct dmemcg_state *
css_to_dmemcs(struct cgroup_subsys_state *css)
{
	return container_of(css, struct dmemcg_state, css);
}

static inline struct dmemcg_state *get_current_dmemcs(void)
{
	return css_to_dmemcs(task_get_css(current, dmem_cgrp_id));
}

static struct dmemcg_state *parent_dmemcs(struct dmemcg_state *cg)
{
	return cg->css.parent ? css_to_dmemcs(cg->css.parent) : NULL;
}

static void dmemcg_pool_get(struct dmem_cgroup_pool_state *pool)
{
	refcount_inc(&pool->ref);
}

static bool dmemcg_pool_tryget(struct dmem_cgroup_pool_state *pool)
{
	return refcount_inc_not_zero(&pool->ref);
}

static void dmemcg_pool_put(struct dmem_cgroup_pool_state *pool)
{
	if (!refcount_dec_and_test(&pool->ref))
		return;

	call_rcu(&pool->rcu, dmemcg_pool_free_rcu);
}

static void dmemcg_pool_free_rcu(struct rcu_head *rcu)
{
	struct dmem_cgroup_pool_state *pool = container_of(rcu, typeof(*pool), rcu);

	if (pool->parent)
		dmemcg_pool_put(pool->parent);
	kref_put(&pool->region->ref, dmemcg_free_region);
	kfree(pool);
}

static void free_cg_pool(struct dmem_cgroup_pool_state *pool)
{
	list_del(&pool->region_node);
	dmemcg_pool_put(pool);
}

static void
set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
{
	page_counter_set_min(&pool->cnt, val);
}

static void
set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
{
	page_counter_set_low(&pool->cnt, val);
}

static void
set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
{
	struct dmem_cgroup_region *region = pool->region;
	unsigned long limit =