path: root/mm/shrinker.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/memcontrol.h>
#include <linux/rwsem.h>
#include <linux/shrinker.h>
#include <linux/rculist.h>
#include <trace/events/vmscan.h>

#include "internal.h"

LIST_HEAD(shrinker_list);
DEFINE_MUTEX(shrinker_mutex);

#ifdef CONFIG_MEMCG
static int shrinker_nr_max;

static inline int shrinker_unit_size(int nr_items)
{
	return (DIV_ROUND_UP(nr_items, SHRINKER_UNIT_BITS) * sizeof(struct shrinker_info_unit *));
}

static inline void shrinker_unit_free(struct shrinker_info *info, int start)
{
	struct shrinker_info_unit **unit;
	int nr, i;

	if (!info)
		return;

	unit = info->unit;
	nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS);

	for (i = start; i < nr; i++) {
		if (!unit[i])
			break;

		kfree(unit[i]);
		unit[i] = NULL;
	}
}

static inline int shrinker_unit_alloc(struct shrinker_info *new,
				       struct shrinker_info *old, int nid)
{
	struct shrinker_info_unit *unit;
	int nr = DIV_ROUND_UP(new->map_nr_max, SHRINKER_UNIT_BITS);
	int start = old ? DIV_ROUND_UP(old->map_nr_max, SHRINKER_UNIT_BITS) : 0;
	int i;

	for (i = start; i < nr; i++) {
		unit = kzalloc_node(sizeof(*unit), GFP_KERNEL, nid);
		if (!unit) {
			shrinker_unit_free(new, start);
			return -ENOMEM;
		}

		new->unit[i] = unit;
	}

	return 0;
}

void free_shrinker_info(struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_node *pn;
	struct shrinker_info *info;
	int nid;

	for_each_node(nid) {
		pn = memcg->nodeinfo[nid];
		info = rcu_dereference_protected(pn->shrinker_info, true);
		shrinker_unit_free(info, 0);
		kvfree(info);
		rcu_assign_pointer(pn->shrinker_info, NULL);
	}
}

int alloc_shrinker_info(struct mem_cgroup *memcg)
{
	int nid, ret = 0;
	int array_size = 0;

	mutex_lock(&shrinker_mutex);
	array_size = shrinker_unit_size(shrinker_nr_max);
	for_each_node(nid) {
		struct shrinker_info *info = kvzalloc_node(sizeof(*info) + array_size,
							   GFP_KERNEL, nid);
		if (!info)
			goto err;
		info->map_nr_max = shrinker_nr_max;
		if (shrinker_unit_alloc(info, NULL, nid)) {
			kvfree(info);
			goto err;
		}
		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
	}
	mutex_unlock(&shrinker_mutex);

	return ret;

err:
	mutex_unlock(&shrinker_mutex);
	free_shrinker_info(memcg);
	return -ENOMEM;
}

static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
						     int nid)
{
	return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
					 lockdep_is_held(&shrinker_mutex));
}

static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size,
				    int old_size, int new_nr_max)
{
	struct shrinker_info *new, *old;
	struct mem_cgroup_per_node *pn;
	int nid;

	for_each_node(nid) {
		pn = memcg->nodeinfo[nid];
		old = shrinker_info_protected(memcg, nid);
		/* Not yet online memcg */
		if (!old)
			return 0;

		/* Already expanded this shrinker_info */
		if (new_nr_max <= old->map_nr_max)
			continue;

		new = kvzalloc_node(sizeof(*new) + new_size, GFP_KERNEL, nid);
		if (!new)
			return -ENOMEM;

		new->map_nr_max = new_nr_max;

		memcpy(new->unit, old->unit, old_size);
		if (shrinker_unit_alloc(new, old, nid)) {
			kvfree(new);
			return -ENOMEM;
		}

		rcu_assign_pointer(pn->shrinker_info, new);
		kvfree_rcu(old, rcu);
	}

	return 0;
}

static int expand_shrinker_info(int new_id)
{
	int ret = 0;
	int new_nr_max = round_up(new_id + 1, SHRINKER_UNIT_BITS);
	int new_size, old_size = 0;
	struct mem_cgroup *memcg;

	if (!root_mem_cgroup)
		goto out;

	lockdep_assert_held(&shrinker_mutex);

	new_size = shrinker_unit_size(new_nr_max);
	old_size = shrinker_unit_size(shrinker_nr_max);

	memcg = mem_cgroup_iter(NULL, NULL, NULL);
	do {
		ret = expand_one_shrinker_info(memcg, new_size, old_size,
					       new_nr_max);
		if (ret) {
			mem_cgroup_iter_break(NULL, memcg);
			goto out;
		}
	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) !=