path: root/drivers/iommu/bcm2712-iommu.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * IOMMU driver for Broadcom BCM2712
 *
 * Copyright (c) 2023-2025 Raspberry Pi Ltd.
 * Copyright (c) 2026 Daniel Drake
 *
 * Each BCM2712 IOMMU has multiple devices hardwired into it, whose
 * DMA transactions all route through the IOMMU. There is no stream ID tagging
 * or any other kind of segmentation to differentiate between requests from
 * different devices. It is also not possible to toggle a specific device
 * between iommu-mapped and bypass modes.
 *
 * The hardware supports the simultaneous configuration of a bypass window
 * (where memory accesses operate in identity mode) and a translation aperture
 * (supporting page mapping). While combining these two would allow for
 * GART-like operation, this driver's paging domain implementation configures
 * only the translation aperture (starting at IOVA 0) offering a degree of
 * memory protection.
 *
 * The page table format is a two-level format handled by generic_pt/bcm2712.
 */

#include <linux/bitfield.h>
#include <linux/cleanup.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/sizes.h>
#include <linux/generic_pt/iommu.h>

#include "bcm2712-iommu-cache.h"
#include "dma-iommu.h"
#include "iommu-pages.h"

/* BCM2712 IOMMU is organized around 4Kbyte pages */
#define IOMMU_PAGE_SHIFT       12
#define IOMMU_PAGE_SIZE        (1ul << IOMMU_PAGE_SHIFT)
/* A PTE is 4 bytes */
#define PTE_SIZE_SHIFT         2
/* L1/L2 table sizing (IOMMU hardware pages): 1024 entries per page */
#define PTES_PER_IOPG_SHIFT    (IOMMU_PAGE_SHIFT - PTE_SIZE_SHIFT)

#define MMMU_CTRL_OFFSET                       0x00
#define MMMU_CTRL_CAP_EXCEEDED                 BIT(27)
#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN        BIT(26)
#define MMMU_CTRL_CAP_EXCEEDED_INT_EN          BIT(25)
#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN    BIT(24)
#define MMMU_CTRL_PT_INVALID                   BIT(20)
#define MMMU_CTRL_PT_INVALID_ABORT_EN          BIT(19)
#define MMMU_CTRL_PT_INVALID_INT_EN            BIT(18)
#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN      BIT(17)
#define MMMU_CTRL_PT_INVALID_EN                BIT(16)
#define MMMU_CTRL_WRITE_VIOLATION              BIT(12)
#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN     BIT(11)
#define MMMU_CTRL_WRITE_VIOLATION_INT_EN       BIT(10)
#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9)
#define MMMU_CTRL_BYPASS                       BIT(8)
#define MMMU_CTRL_TLB_CLEARING                 BIT(7)
#define MMMU_CTRL_STATS_CLEAR                  BIT(3)
#define MMMU_CTRL_TLB_CLEAR                    BIT(2)
#define MMMU_CTRL_STATS_ENABLE                 BIT(1)
#define MMMU_CTRL_ENABLE                       BIT(0)

#define MMMU_CTRL_OPERATING_FLAGS (\
	MMMU_CTRL_CAP_EXCEEDED_ABORT_EN    | \
	MMMU_CTRL_PT_INVALID_ABORT_EN      | \
	MMMU_CTRL_PT_INVALID_EN            | \
	MMMU_CTRL_WRITE_VIOLATION_ABORT_EN | \
	MMMU_CTRL_STATS_ENABLE             | \
	MMMU_CTRL_ENABLE)

#define MMMU_PT_PA_BASE_OFFSET                 0x04

#define MMMU_ADDR_CAP_OFFSET                   0x14
#define MMMU_ADDR_CAP_ENABLE                   BIT(31)
#define ADDR_CAP_SHIFT                         ilog2(SZ_256M)

#define MMMU_SHOOT_DOWN_OFFSET                 0x18
#define MMMU_SHOOT_DOWN_SHOOTING               BIT(31)
#define MMMU_SHOOT_DOWN_SHOOT                  BIT(30)

#define MMMU_BYPASS_START_OFFSET               0x1c
#define MMMU_BYPASS_START_ENABLE               BIT(31)

#define MMMU_BYPASS_END_OFFSET                 0x20
#define MMMU_BYPASS_END_ENABLE                 BIT(31)

#define MMMU_MISC_OFFSET                       0x24
#define MMMU_MISC_SINGLE_TABLE                 BIT(31)

#define MMMU_ILLEGAL_ADR_OFFSET                0x30
#define MMMU_ILLEGAL_ADR_ENABLE                BIT(31)

#define MMMU_DEBUG_INFO_OFFSET                 0x38
#define MMMU_DEBUG_INFO_VERSION_MASK           0x0000000Fu
#define MMMU_DEBUG_INFO_VA_WIDTH_MASK          0x000000F0u
#define MMMU_DEBUG_INFO_PA_WIDTH_MASK          0x00000F00u
#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK     0x000FF000u
#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK   0x0FF00000u
#define MMMU_DEBUG_INFO_BYPASS_4M              BIT(28)
#define MMMU_DEBUG_INFO_BYPASS                 BIT(29)

struct bcm2712_iommu {
	struct device *dev;
	struct iommu_device iommu;
	struct bcm2712_iommu_domain *domain;
	struct bcm2712_iommu_cache *cache;
	void __iomem *reg_base;
	spinlock_t hw_lock;
	size_t bigpage_size;
	size_t superpage_size;
};

struct bcm2712_iommu_domain {
	union {
		struct iommu_domain base;
		struct pt_iommu_bcm2712 pt;
	};
	struct bcm2712_iommu *mmu;
	void *default_page;
};

static inline void bcm2712_iommu_writel(struct bcm2712_iommu *mmu,
					unsigned int offset, u32 val)
{
	writel(val, mmu->reg_base + offset);
}

static inline u32 bcm2712_iommu_readl(struct bcm2712_iommu *mmu,
				      unsigned int offset)
{
	return readl(mmu->reg_base + offset);
}

static struct bcm2712_iommu_domain *
to_bcm2712_domain(struct iommu_domain *domain)
{
	return container_of(domain, struct bcm2712_iommu_domain, base);
}

static void bcm2712_iommu_init(struct bcm2712_iommu *mmu)
{
	unsigned int bigpage_width, superpage_width;
	u32 u = bcm2712_iommu_readl(mmu, MMMU_DEBUG_INFO_OFFSET);
	u32 pa_width = FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u);

	dev_dbg(mmu->dev, "DEBUG_INFO = 0x%08x\n", u);
	WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 ||
		FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 ||
		pa_width < 6 || !(u & MMMU_DEBUG_INFO_BYPASS));

	dma_set_mask_and_coherent(mmu->dev, DMA_BIT_MASK(pa_width + 30u));

	bigpage_width = FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u);
	if (bigpage_width)
		mmu->bigpage_size = IOMMU_PAGE_SIZE << bigpage_width;

	superpage_width = FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u);
	if (superpage_width)
		mmu->superpage_size = IOMMU_PAGE_SIZE << superpage_width;

	/* Disable MMU and clear sticky flags */
	bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET,
			     MMMU_CTRL_CAP_EXCEEDED | MMMU_CTRL_PT_INVALID |
			     MMMU_CTRL_WRITE_VIOLATION |
			     MMMU_CTRL_STATS_CLEAR);

	/* Put MMU into 2-level mode */
	bcm2712_iommu_writel(mmu, MMMU_MISC_OFFSET,
			     bcm2712_iommu_readl(mmu, MMMU_MISC_OFFSET) &
				     ~MMMU_MISC_SINGLE_TABLE);
}

static int bcm2712_iommu_identity_attach(struct iommu_domain *identity_domain,
					 struct device *dev,
					 struct iommu_domain *old)
{
	struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);

	guard(spinlock_irqsave)(&mmu->hw_lock);
	bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
	mmu->domain = NULL;

	return 0;
}

static struct iommu_domain bcm2712_identity_domain = {
	.type = IOMMU_DOMAIN_IDENTITY,
	.ops = &(const struct iommu_domain_ops) {
		.attach_dev = bcm2712_iommu_identity_attach,
	},
};

static int bcm2712_iommu_clear_and_enable(struct bcm2712_iommu *mmu)
	__must_hold(&mmu->hw_lock)
{
	u32 val;

	bcm2712_iommu_cache_flush(mmu->cache);
	bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET,
			     MMMU_CTRL_OPERATING_FLAGS | MMMU_CTRL_TLB_CLEAR);
	return readl_poll_timeout_atomic(mmu->reg_base + MMMU_CTRL_OFFSET, val,
					 !(val & MMMU_CTRL_TLB_CLEARING), 0,
					 50);
}


static int bcm2712_iommu_attach_dev(struct iommu_domain *domain,
				    struct device *dev,
				    struct iommu_domain *old)
{
	struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
	struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
	struct pt_iommu_bcm2712_hw_info info;
	u32 default_page_pfn;
	int ret = 0;

	/* all attached devices must belong to the same IOMMU instance */
	if (mydomain->mmu != mmu)
		return -EINVAL;

	scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
		if (mmu->domain == mydomain)
			break;

		/* Configure translation aperture */
		bcm2712_iommu_writel(mmu, MMMU_ADDR_CAP_OFFSET,
				     MMMU_ADDR_CAP_ENABLE |
					     (domain->geometry.aperture_end >>
					      ADDR_CAP_SHIFT));

		/*
		 * When the IOMMU handles a request, it adds the
		 * PT_PA_BASE_OFFSET to (IOVA>>32) to calculate the PFN of the
		 * corresponding L1 directory page.
		 * IOVA bits [31:22] are then used to fetch the L1 descriptor
		 * within (which in turn points to the L2 table).
		 * This clever logic would allow for a L1 table larger than 4kb
		 * (and hence a larger aperture).
		 */
		pt_iommu_bcm2712_hw_info(&mydomain->pt, &info);
		bcm2712_iommu_writel(mmu, MMMU_PT_PA_BASE_OFFSET,
				     info.pt_base >> IOMMU_PAGE_SHIFT);

		/* Disable bypass window */
		bcm2712_iommu_writel(mmu, MMMU_BYPASS_START_OFFSET, 0);
		bcm2712_iommu_writel(mmu, MMMU_BYPASS_END_OFFSET, 0);

		/*
		 * A default (error) page is used to catch illegal reads/writes.
		 */
		default_page_pfn = virt_to_phys(mydomain->default_page) >>
				   IOMMU_PAGE_SHIFT;
		bcm2712_iommu_writel(mmu, MMMU_ILLEGAL_ADR_OFFSET,
				     MMMU_ILLEGAL_ADR_ENABLE |
					     default_page_pfn);

		ret = bcm2712_iommu_clear_and_enable(mmu);
		if (ret) {
			bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
			mmu->domain = NULL;
			break;
		}

		mmu->domain = mydomain;
	}

	if (ret)
		dev_err_ratelimited(mmu->dev,
				    "TLB clear timed out during attach\n");

	return ret;
}

static int bcm2712_iommu_shootdown_range(struct bcm2712_iommu *mmu,
					 unsigned long iova, size_t size)
	__must_hold(&mmu->hw_lock)
{
	unsigned long start_pfn = iova >> IOMMU_PAGE_SHIFT;
	unsigned long end_pfn = (iova + size - 1) >> IOMMU_PAGE_SHIFT;
	unsigned long pfn;
	u32 val;
	int ret;