path: root/drivers/accel/habanalabs/common/hw_queue.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2019 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#include "habanalabs.h"

#include <linux/slab.h>

/*
 * hl_queue_add_ptr - add to pi or ci and checks if it wraps around
 *
 * @ptr: the current pi/ci value
 * @val: the amount to add
 *
 * Add val to ptr. It can go until twice the queue length.
 */
inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
{
	ptr += val;
	ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
	return ptr;
}
static inline int queue_ci_get(atomic_t *ci, u32 queue_len)
{
	return atomic_read(ci) & ((queue_len << 1) - 1);
}

static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
{
	int delta = (q->pi - queue_ci_get(&q->ci, queue_len));

	if (delta >= 0)
		return (queue_len - delta);
	else
		return (abs(delta) - queue_len);
}

void hl_hw_queue_update_ci(struct hl_cs *cs)
{
	struct hl_device *hdev = cs->ctx->hdev;
	struct hl_hw_queue *q;
	int i;

	if (hdev->disabled)
		return;

	q = &hdev->kernel_queues[0];

	/* There are no internal queues if H/W queues are being used */
	if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)
		return;

	/* We must increment CI for every queue that will never get a
	 * completion, there are 2 scenarios this can happen:
	 * 1. All queues of a non completion CS will never get a completion.
	 * 2. Internal queues never gets completion.
	 */
	for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {
		if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT)
			atomic_add(cs->jobs_in_queue_cnt[i], &q->ci);
	}
}

/*
 * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
 *                                H/W queue.
 * @hdev: pointer to habanalabs device structure
 * @q: pointer to habanalabs queue structure
 * @ctl: BD's control word
 * @len: BD's length
 * @ptr: BD's pointer
 *
 * This function assumes there is enough space on the queue to submit a new
 * BD to it. It initializes the next BD and calls the device specific
 * function to set the pi (and doorbell)
 *
 * This function must be called when the scheduler mutex is taken
 *
 */
void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
		u32 ctl, u32 len, u64 ptr)
{
	struct hl_bd *bd;
	u64 addr;
	int i;

	bd = q->kernel_address;
	bd += hl_pi_2_offset(q->pi);
	bd->ctl = cpu_to_le32(ctl);
	bd->len = cpu_to_le32(len);
	bd->ptr = cpu_to_le64(ptr);

	if (q->dram_bd)
		for (i = 0 ; i < 2 ; i++) {
			addr = q->pq_dram_address +
			((hl_pi_2_offset(q->pi) * sizeof(struct hl_bd))	+ (i * sizeof(u64)));
			hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM,	addr,
						(u64 *)(bd) + i, DEBUGFS_WRITE64);
		}

	q->pi = hl_queue_inc_ptr(q->