path: root/drivers/vdpa/pds/vdpa_dev.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2023 Advanced Micro Devices, Inc */

#include <linux/pci.h>
#include <linux/vdpa.h>
#include <uapi/linux/vdpa.h>
#include <linux/virtio_pci_modern.h>

#include <linux/pds/pds_common.h>
#include <linux/pds/pds_core_if.h>
#include <linux/pds/pds_adminq.h>
#include <linux/pds/pds_auxbus.h>

#include "vdpa_dev.h"
#include "aux_drv.h"
#include "cmds.h"
#include "debugfs.h"

static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev);

static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev)
{
	return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
}

static int pds_vdpa_notify_handler(struct notifier_block *nb,
				   unsigned long ecode,
				   void *data)
{
	struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb);
	struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;

	dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);

	if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
		if (pdsv->config_cb.callback)
			pdsv->config_cb.callback(pdsv->config_cb.private);
	}

	return 0;
}

static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
{
	struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
	struct notifier_block *nb = &pdsv->nb;
	int err;

	if (!nb->notifier_call) {
		nb->notifier_call = pds_vdpa_notify_handler;
		err = pdsc_register_notify(nb);
		if (err) {
			nb->notifier_call = NULL;
			dev_err(dev, "failed to register pds event handler: %pe\n",
				ERR_PTR(err));
			return -EINVAL;
		}
		dev_dbg(dev, "pds event handler registered\n");
	}

	return 0;
}

static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
{
	if (pdsv->nb.notifier_call) {
		pdsc_unregister_notify(&pdsv->nb);
		pdsv->nb.notifier_call = NULL;
	}
}

static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
				   u64 desc_addr, u64 driver_addr, u64 device_addr)
{
	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);

	pdsv->vqs[qid].desc_addr = desc_addr;
	pdsv->vqs[qid].avail_addr = driver_addr;
	pdsv->vqs[qid].used_addr = device_addr;

	return 0;
}

static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
{
	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);

	pdsv->vqs[qid].q_len = num;
}

static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
{
	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);

	iowrite16(qid, pdsv->vqs[qid].notify);
}

static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
			       struct vdpa_callback *cb)
{
	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);

	pdsv->vqs[qid].event_cb = *cb;
}

static irqreturn_t pds_vdpa_isr(int irq, void *data)
{
	struct pds_vdpa_vq_info *vq;

	vq = data;
	if (vq->event_cb.callback)
		vq->event_cb.callback(vq->event_cb.private);

	return IRQ_HANDLED;
}

static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid)
{
	if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR)
		return;

	free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]);
	pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
}

static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
{
	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
	struct device *dev = &pdsv->vdpa_dev.dev;
	u64 driver_features;
	u16 invert_idx = 0;
	int err;

	dev_dbg(dev, "%s: qid %d ready %d => %d\n",
		__func__, qid, pdsv->vqs[qid].ready, ready);
	if (ready == pdsv->vqs[qid].ready)
		return;

	driver_features = pds_vdpa_get_driver_features(vdpa_dev);
	if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
		invert_idx = PDS_VDPA_PACKED_INVERT_IDX;

	if (ready) {
		/* Pass vq setup info to DSC using adminq to gather up and
		 * send all info at once so FW can do its full set up in
		 * one easy operation
		 */
		err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
		if (err) {
			dev_err(dev, "Failed to init vq %d: %pe\n