path: root/net/sched/sch_taprio.c

// SPDX-License-Identifier: GPL-2.0

/* net/sched/sch_taprio.c	 Time Aware Priority Scheduler
 *
 * Authors:	Vinicius Costa Gomes <vinicius.gomes@intel.com>
 *
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/math64.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/sch_generic.h>

static LIST_HEAD(taprio_list);
static DEFINE_SPINLOCK(taprio_list_lock);

#define TAPRIO_ALL_GATES_OPEN -1

struct sched_entry {
	struct list_head list;

	/* The instant that this entry "closes" and the next one
	 * should open, the qdisc will make some effort so that no
	 * packet leaves after this time.
	 */
	ktime_t close_time;
	atomic_t budget;
	int index;
	u32 gate_mask;
	u32 interval;
	u8 command;
};

struct sched_gate_list {
	struct rcu_head rcu;
	struct list_head entries;
	size_t num_entries;
	ktime_t cycle_close_time;
	s64 cycle_time;
	s64 cycle_time_extension;
	s64 base_time;
};

struct taprio_sched {
	struct Qdisc **qdiscs;
	struct Qdisc *root;
	int clockid;
	atomic64_t picos_per_byte; /* Using picoseconds because for 10Gbps+
				    * speeds it's sub-nanoseconds per byte
				    */

	/* Protects the update side of the RCU protected current_entry */
	spinlock_t current_entry_lock;
	struct sched_entry __rcu *current_entry;
	struct sched_gate_list __rcu *oper_sched;
	struct sched_gate_list __rcu *admin_sched;
	ktime_t (*get_time)(void);
	struct hrtimer advance_timer;
	struct list_head taprio_list;
};

static ktime_t sched_base_time(const struct sched_gate_list *sched)
{
	if (!sched)
		return KTIME_MAX;

	return ns_to_ktime(sched->base_time);
}

static void taprio_free_sched_cb(struct rcu_head *head)
{
	struct sched_gate_list *sched = container_of(head, struct sched_gate_list, rcu);
	struct sched_entry *entry, *n;

	if (!sched)
		return;

	list_for_each_entry_safe(entry, n, &sched->entries, list) {
		list_del(&entry->list);
		kfree(entry);
	}

	kfree(sched);
}

static void switch_schedules(struct taprio_sched *q,
			     struct sched_gate_list **admin,
			     struct sched_gate_list **oper)
{
	rcu_assign_pointer(q->oper_sched, *admin);
	rcu_assign_pointer(q->admin_sched, NULL);

	if (*oper)
		call_rcu(&(*oper)->rcu, taprio_free_sched_cb);

	*oper = *admin;
	*admin = NULL;
}

static ktime_t