path: root/drivers/dma/idxd/init.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/fs.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/idr.h>
#include <linux/iommu.h>
#include <uapi/linux/idxd.h>
#include <linux/dmaengine.h>
#include "../dmaengine.h"
#include "registers.h"
#include "idxd.h"
#include "perfmon.h"

MODULE_VERSION(IDXD_DRIVER_VERSION);
MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Intel Corporation");
MODULE_IMPORT_NS("IDXD");

static bool sva = true;
module_param(sva, bool, 0644);
MODULE_PARM_DESC(sva, "Toggle SVA support on/off");

bool tc_override;
module_param(tc_override, bool, 0644);
MODULE_PARM_DESC(tc_override, "Override traffic class defaults");

#define DRV_NAME "idxd"

bool support_enqcmd;
DEFINE_IDA(idxd_ida);

static struct idxd_driver_data idxd_driver_data[] = {
	[IDXD_TYPE_DSA] = {
		.name_prefix = "dsa",
		.type = IDXD_TYPE_DSA,
		.compl_size = sizeof(struct dsa_completion_record),
		.align = 32,
		.dev_type = &dsa_device_type,
		.evl_cr_off = offsetof(struct dsa_evl_entry, cr),
		.user_submission_safe = false, /* See INTEL-SA-01084 security advisory */
		.cr_status_off = offsetof(struct dsa_completion_record, status),
		.cr_result_off = offsetof(struct dsa_completion_record, result),
	},
	[IDXD_TYPE_IAX] = {
		.name_prefix = "iax",
		.type = IDXD_TYPE_IAX,
		.compl_size = sizeof(struct iax_completion_record),
		.align = 64,
		.dev_type = &iax_device_type,
		.evl_cr_off = offsetof(struct iax_evl_entry, cr),
		.user_submission_safe = false, /* See INTEL-SA-01084 security advisory */
		.cr_status_off = offsetof(struct iax_completion_record, status),
		.cr_result_off = offsetof(struct iax_completion_record, error_code),
		.load_device_defaults = idxd_load_iaa_device_defaults,
	},
};

static struct pci_device_id idxd_pci_tbl[] = {
	/* DSA ver 1.0 platforms */
	{ PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
	/* DSA on GNR-D platforms */
	{ PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) },
	/* DSA on DMR platforms */
	{ PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) },

	/* IAX ver 1.0 platforms */
	{ PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
	/* IAA on DMR platforms */
	{ PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) },
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);

static int idxd_setup_interrupts(struct idxd_device *idxd)
{
	struct pci_dev *pdev = idxd->pdev;
	struct device *dev = &pdev->dev;
	struct idxd_irq_entry *ie;
	int i, msixcnt;
	int rc = 0;

	msixcnt = pci_msix_vec_count(pdev);
	if (msixcnt < 0) {
		dev_err(dev, "Not MSI-X interrupt capable.\n");
		return -ENOSPC;
	}
	idxd->irq_cnt = msixcnt;

	rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
	if (rc != msixcnt) {
		dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
		return -ENOSPC;
	}
	dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);


	ie = idxd_get_ie(idxd, 0);
	ie->vector = pci_irq_vector(pdev, 0);
	rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie);
	if (rc < 0) {
		dev_err(dev, "Failed to allocate misc interrupt.\n");
		goto err_misc_irq;
	}
	dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector);

	for (i = 0; i < idxd->max_wqs; i++) {
		int msix_idx = i + 1;

		ie = idxd_get_ie(idxd, msix_idx);
		ie->id = msix_idx;
		ie->int_handle = INVALID_INT_HANDLE;
		ie->pasid = IOMMU_PASID_INVALID;

		spin_lock_init(&ie