path: root/tools/testing/selftests/kvm/access_tracking_perf_test.c

// SPDX-License-Identifier: GPL-2.0
/*
 * access_tracking_perf_test
 *
 * Copyright (C) 2021, Google, Inc.
 *
 * This test measures the performance effects of KVM's access tracking.
 * Access tracking is driven by the MMU notifiers test_young, clear_young, and
 * clear_flush_young. These notifiers do not have a direct userspace API,
 * however the clear_young notifier can be triggered either by
 *   1. marking a pages as idle in /sys/kernel/mm/page_idle/bitmap OR
 *   2. adding a new MGLRU generation using the lru_gen debugfs file.
 * This test leverages page_idle to enable access tracking on guest memory
 * unless MGLRU is enabled, in which case MGLRU is used.
 *
 * To measure performance this test runs a VM with a configurable number of
 * vCPUs that each touch every page in disjoint regions of memory. Performance
 * is measured in the time it takes all vCPUs to finish touching their
 * predefined region.
 *
 * Note that a deterministic correctness test of access tracking is not possible
 * by using page_idle or MGLRU aging as it exists today. This is for a few
 * reasons:
 *
 * 1. page_idle and MGLRU only issue clear_young notifiers, which lack a TLB flush.
 *    This means subsequent guest accesses are not guaranteed to see page table
 *    updates made by KVM until some time in the future.
 *
 * 2. page_idle only operates on LRU pages. Newly allocated pages are not
 *    immediately allocated to LRU lists. Instead they are held in a "pagevec",
 *    which is drained to LRU lists some time in the future. There is no
 *    userspace API to force this drain to occur.
 *
 * These limitations are worked around in this test by using a large enough
 * region of memory for each vCPU such that the number of translations cached in
 * the TLB and the number of pages held in pagevecs are a small fraction of the
 * overall workload. And if either of those conditions are not true (for example
 * in nesting, where TLB size is unlimited) this test will print a warning
 * rather than silently passing.
 */
#include <inttypes.h>
#include <limits.h>
#include <pthread.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>

#include "kvm_util.h"
#include "test_util.h"
#include "memstress.h"
#include "guest_modes.h"
#include "processor.h"
#include "ucall_common.h"

#include "cgroup_util.h"
#include "lru_gen_util.h"

static const char *TEST_MEMCG_NAME = "access_tracking_perf_test";

/* Global variable used to synchronize all of the vCPU threads. */
static int iteration;

/* The cgroup memory controller root. Needed for lru_gen-based aging. */
char cgroup_root[PATH_MAX];

/* Defines what vCPU threads should do during a given iteration. */
static enum {
	/* Run the vCPU to access all its memory. */
	ITERATION_ACCESS_MEMORY,
	/* Mark the vCPU's memory idle in page_idle. */
	ITERATION_MARK_IDLE,
} iteration_work;

/* The iteration that was last completed by each vCPU. */
static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];

/* Whether to overlap the regions of memory vCPUs access. */
static bool overlap_memory_access;

/*
 * If the test should only warn if there are too many idle pages (i.e., it is
 * expected).
 * -1: Not yet set.
 *  0: We do not expect too many idle pages, so FAIL if too many idle pages.
 *  1: Having too many idle pages is expected, so merely print a warning if
 *     too many idle pages are found.
 */
static int idle_pages_warn_only = -1;

/* Whether or not to use MGLRU instead of page_idle for access tracking */
static bool use_lru_gen;

/* Total number of pages to expect in the memcg after touching everything */
static long test_pages;

/* Last generation we found the pages in */
static int lru_gen_last_gen = -1;

struct test_params {
	/* The backing source for the region of memory. */
	enum vm_mem_backing_src_type backing_src;

	/* The amount of memory to allocate for each vCPU. */
	u64 vcpu_memory_bytes;

	/* The number of vCPUs to create in the VM. */
	int nr_vcpus;
};

static u64 pread_u64(int fd, const char *filename, u64 index)
{
	u64 value;
	off_t offset = index * sizeof(value);

	TEST_ASSERT(pread(fd, &value, sizeof(value), offset) == sizeof(value),
		    "pread from %s offset 0x%" PRIx64 " failed!",
		    filename, offset);

	return value;

}

#define PAGEMAP_PRESENT (1ULL << 63)
#define PAGEMAP_PFN_MASK ((1ULL << 55) - 1)

static u64 lookup_pfn(int pagemap_fd, struct kvm_vm *vm, gva_t gva)
{
	u64 hva = (u64)addr_gva2hva(vm, gva);
	u64 entry;
	u64 pfn;

	entry = pread_u64(pagemap_fd, "pagemap", hva / getpagesize());
	if (!(entry & PAGEMAP_PRESENT))
		return 0;

	pfn = entry & PAGEMAP_PFN_MASK;
	__TEST_REQUIRE(pfn, "Looking up PFNs requires CAP_SYS_ADMIN");

	return pfn;
}

static bool is_page_idle(int page_idle_fd, u64 pfn)
{
	u64 bits = pread_u64(page_idle_fd, "page_idle", pfn / 64);

	return !!((bits >> (pfn % 64)) & 1);
}

static void mark_page_idle(int page_idle_fd, u64 pfn)
{
	u64 bits = 1ULL << (pfn % 64);

	TEST_ASSERT(pwrite(page_idle_fd, &bits, 8, 8 * (pfn / 64)) == 8,
		    "Set page_idle bits for PFN 0x%" PRIx64, pfn);
}

static void too_many_idle_pages(long idle_pages, long total_pages, int vcpu_idx)
{
	char prefix[18] = {};

	if (vcpu_idx >= 0)
		snprintf(prefix, 18, "vCPU%d: ", vcpu_idx);

	TEST_ASSERT(idle_pages_warn_only,
		    "%sToo many pages still idle (%lu out of %lu)",
		    prefix, idle_pages, total_pages);

	printf("WARNING: %sToo many pages still idle (%lu out of %lu), "
	       "this will affect performance results.\n",
	       prefix, idle_pages, total_pages);
}

static void pageidle_mark_vcpu_memory_idle(struct kvm_vm *vm,
					   struct memstress_vcpu_args *vcpu_args)
{
	int vcpu_idx = vcpu_args->vcpu_idx;
	gva_t base_gva = vcpu_args->gva;
	u64 pages = vcpu_args->pages;
	u64 page;
	u64 still_idle = 0;
	u64 no_pfn = 0;
	int page_idle_fd;
	int pagemap_fd;

	/* If vCPUs are using an overlapping region, let vCPU 0 mark it idle. */
	if (overlap_memory_access && vcpu_idx)
		return;

	page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR);
	TEST_ASSERT(page_idle_fd > 0, "Failed to open page_idle.");

	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
	TEST_ASSERT(pagemap_fd > 0, "Failed to open pagemap.");

	for (page = 0; page < pages; page++) {
		gva_t gva = base_gva + page * memstress_args.guest_page_size;
		u64 pfn = lookup_pfn(pagemap_fd, vm, gva);

		if (!pfn) {
			no_pfn++;
			continue;
		}

		if (is_page_idle(page_idle_fd, pfn)) {
			still_idle++;
			continue;
		}

		mark_page_idle(page_idle_fd, pfn);
	}

	/*
	 * Assumption: Less than 1% of pages are going to be swapped out from
	 * under us during this test.
	 */
	TEST_ASSERT(no_pfn < pages / 100,
		    "vCPU %d: No PFN for %" PRIu64 " out of %" PRIu64 " pages.",
		    vcpu_idx, no_pfn, pages);

	/*
	 * Check that at least 90% of memory has been marked idle (the rest
	 * might not be marked idle because the pages have not yet made it to an
	 * LRU list or the translations are still cached in the TLB). 90% is
	 * arbitrary; high enough that we ensure most memory access went through
	 * access tracking but low enough as to not make the test too brittle
	 * over time and across architectures.
	 */
	if (still_idle >= pages / 10)
		too_many_idle_pages(still_idle, pages,
				    overlap_memory_access ? -1 : vcpu_idx);

	close(page_idle_fd);
	close(pagemap_fd);
}

int find_generation(struct memcg_stats *stats, long total_pages)
{
	/*
	 * For finding the generation that contains our pages, use the same
	 * 90% threshold that page_idle uses.
	 */
	int gen = lru_gen_find_generation(stats, total_pages * 9 / 10);

	if (gen >= 0)
		return gen;

	if (!idle_pages_warn_only) {
		TEST_FAIL("Could not find a generation with 90%% of guest memory (%ld pages).",
			   total_pages * 9 / 10);
		return gen;
	}

	/*
	 * We couldn't find a generation with 90% of guest memory, which can
	 * happen if access tracking is unreliable. Simply look for a majority
	 * of pages.
	 */
	puts("WARNING: Couldn't find a generation with 90% of guest memory. "
	     "Performance results may not be accurate.");
	gen = lru_gen_find_generation(stats, total_pages / 2);
	TEST_ASSERT(gen >= 0,
		    "Could not find a generation with 50%% of guest memory (%ld pages).",
		    total_pages / 2);
	return gen;
}

static void