path: root/drivers/gpu/drm/xe/xe_exec_queue.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include "xe_exec_queue.h"

#include <linux/nospec.h>

#include <drm/drm_device.h>
#include <drm/drm_file.h>
#include <drm/xe_drm.h>

#include "xe_device.h"
#include "xe_gt.h"
#include "xe_hw_engine_class_sysfs.h"
#include "xe_hw_fence.h"
#include "xe_lrc.h"
#include "xe_macros.h"
#include "xe_migrate.h"
#include "xe_pm.h"
#include "xe_ring_ops_types.h"
#include "xe_trace.h"
#include "xe_vm.h"

enum xe_exec_queue_sched_prop {
	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
	XE_EXEC_QUEUE_TIMESLICE = 1,
	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,
	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,
};

static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe,
						    struct xe_vm *vm,
						    u32 logical_mask,
						    u16 width, struct xe_hw_engine *hwe,
						    u32 flags)
{
	struct xe_exec_queue *q;
	struct xe_gt *gt = hwe->gt;
	int err;
	int i;

	/* only kernel queues can be permanent */
	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));

	q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL);
	if (!q)
		return ERR_PTR(-ENOMEM);

	kref_init(&q->refcount);
	q->flags = flags;
	q->hwe = hwe;
	q->gt = gt;
	if (vm)
		q->vm = xe_vm_get(vm);
	q->class = hwe->class;
	q->width = width;
	q->logical_mask = logical_mask;
	q->fence_irq = &gt->fence_irq[hwe->class];
	q->ring_ops = gt->ring_ops[hwe->class];
	q->ops = gt->exec_queue_ops;
	INIT_LIST_HEAD(&q->compute.link);
	INIT_LIST_HEAD(&q->multi_gt_link);

	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
	q->sched_props.preempt_timeout_us =
				hwe->eclass->sched_props.preempt_timeout_us;
	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&
	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)
		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;
	else
		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;

	if (xe_exec_queue_is_parallel(q)) {
		q->parallel.composite_fence_ctx = dma_fence_context_alloc(1);
		q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO;
	}
	if (q->flags & EXEC_QUEUE_FLAG_VM) {
		q->bind.fence_ctx = dma_fence_context_alloc(1);
		q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO;
	}

	for (i = 0; i < width; ++i) {
		err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K);
		if (err)
			goto err_lrc;
	}

	err = q->ops->init(q);
	if (err)
		goto err_lrc;

	/*
	 * Normally the user vm holds an rpm ref to keep the device
	 * awake, and the context holds a ref for the vm, however for
	 * some engines we use the kernels migrate vm underneath which offers no
	 * such rpm ref, or we lack a vm. Make sure we keep a ref here, so we
	 * can perform GuC CT actions when needed. Caller is expected to have
	 * already grabbed the rpm ref outside any sensitive locks.
	 */
	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !vm))
		drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe));

	return q;

err_lrc:
	for (i = i - 1; i >= 0; --i)
		xe_lrc_finish(q->lrc + i);
	kfree(q);
	return ERR_PTR(err);
}

struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
					   u32 logical_mask, u16 width,
					   struct xe_hw_engine *hwe, u32 flags)
{
	struct xe_exec_queue *q;
	int err;

	if (vm) {
		err = xe_vm_lock(vm, true);
		if (err)
			return ERR_PTR(err);
	}
	q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags);
	if (vm)
		xe_vm_unlock(vm);

	return q;
}

struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct