path: root/tools/testing/selftests/membarrier/membarrier_rseq_stress.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Membarrier stress test for CFS throttle interactions.
 *
 * Reproducer for the interaction between CFS throttle and expedited membarrier.
 */

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <pthread.h>
#include <syscall.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <stdint.h>
#include <errno.h>
#include <sched.h>
#include <time.h>
#include <signal.h>
#include <stdatomic.h>
#include <dirent.h>
#include <sys/prctl.h>
#include <sys/mman.h>

#include "../kselftest.h"

/* -- Architecture-specific rseq signature -- */
#if defined(__x86_64__) || defined(__i386__)
# define RSEQ_SIG  0x53053053U
#elif defined(__aarch64__)
# define RSEQ_SIG  0xd428bc00U
#elif defined(__powerpc__) || defined(__powerpc64__)
# define RSEQ_SIG  0x0f000000U
#elif defined(__s390__) || defined(__s390x__)
# define RSEQ_SIG  0x0c000000U
#else
# define RSEQ_SIG  0
# define UNSUPPORTED_ARCH 1
#endif

/* -- rseq ABI (kernel uapi; define locally for portability) -- */
#define RSEQ_CPU_ID_UNINITIALIZED       ((__u32)-1)

#include <linux/compiler.h>

struct rseq_abi {
	__u32 cpu_id_start;
	__u32 cpu_id;
	__u64 rseq_cs;
	__u32 flags;
	__u32 node_id;
	__u32 mm_cid;
	char  end[0];
} __aligned(32);

/* -- membarrier constants (not in all distro headers) -- */
#ifndef MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
# define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ          (1 << 7)
#endif
#ifndef MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ
# define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ (1 << 8)
#endif
#ifndef MEMBARRIER_CMD_FLAG_CPU
# define MEMBARRIER_CMD_FLAG_CPU  (1 << 0)
#endif

/* -- Test parameters -- */
#define N_SIBLINGS          2000
#define NEST_DEPTH		5
static char g_cgroup_path[4096];
static int use_cgroup_v2;

#define CFS_QUOTA_US        1000
#define CFS_PERIOD_US       5000
#define N_HAMMER_PER_CPU    25
#define N_BURNER_PER_CPU    50
#define MAX_STRESS_CPUS     1024
#define TEST_DURATION_SEC   20

/* Latency thresholds for the sentinel */
#define LATENCY_WARN_MS     50
#define LATENCY_CRITICAL_MS 200

/* Sentinel sampling interval */
#define SENTINEL_INTERVAL_US  500

/* -- Shared globals -- */
static atomic_int  g_stop;
static atomic_int  g_stop_sentinel;
static atomic_long g_max_latency_us;
static atomic_long g_interval_max_latency_us;
static atomic_long g_mb_ok;
static atomic_long g_mb_err;
static int         g_ncpus_stress;
static int *g_stress_cpus;

static atomic_int  g_test_ready;

/* Per-thread rseq ABI block registered with the kernel */
static __thread struct rseq_abi tls_rseq
	__attribute__((tls_model("initial-exec"))) __aligned(32) = {
	.cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
};

/* -- Utility -- */
static int write_file(const char *path, const char *val)
{
	int fd = open(path, O_WRONLY | O_CLOEXEC);

	if (fd < 0)
		return -errno;

	size_t len = strlen(val);
	ssize_t r = write(fd, val, len);

	close(fd);
	if (r < 0)
		return -errno;
	if ((size_t)r != len)
		return -EIO;
	return 0;
}

static uint64_t monotonic_us(void)
{
	struct timespec ts;

	clock_gettime(CLOCK_MONOTONIC, &ts);
	return (uint64_t)ts.tv_sec * 1000000ULL + ts.tv_nsec / 1000ULL;
}

static void update_max_latency(long lat)
{
	long old = atomic_load_explicit(&g_max_latency_us, memory_order_relaxed);

	while (lat > old) {
		if (atomic_compare_exchange_weak_explicit(&g_max_latency_us, &old, lat,
				memory_order_relaxed, memory_order_relaxed))
			break;
	}

	old = atomic_load_explicit(&g_interval_max_latency_us, memory_order_relaxed);
	while (lat > old) {
		if (atomic_compare_exchange_weak_explicit(&g_interval_max_latency_us, &old, lat,
				memory_order_relaxed, memory_order_relaxed))
			break;
	}
}

static void init_stress_cpus(void)
{
	cpu_set_t set;
	int capacity = MAX_STRESS_CPUS;

	g_stress_cpus = malloc(capacity * sizeof(int));
	if (!g_stress_cpus)
		ksft_exit_fail_msg("malloc failed for g_stress_cpus\n");

	if (sched_getaffinity(0, sizeof(set), &set) < 0)
		ksft_exit_fail_msg("sched_getaffinity failed\n");

	for (int i = 0; i < CPU_SETSIZE && g_ncpus_stress < capacity; i++) {
		if (CPU_ISSET(i, &set))
			g_stress_cpus[g_ncpus_stress++] = i;
	}

	if (g_ncpus_stress == 0)
		ksft_exit_skip("No CPUs available for stress test\n");

	ksft_print_msg("Stressing %d CPUs discovered via affinity\n", g_ncpus_stress);
}

/* -- rseq / membarrier helpers -- */
static int rseq_register_thread(void)
{
	int r = syscall(SYS_rseq, &tls_rseq, sizeof(tls_rseq), 0, RSEQ_SIG);

	return (r == 0 || errno == EBUSY || errno == EINVAL) ? 0 : -1;
}

static int rseq_register_thread_at(struct