[PATCH 2/7] iommu: apple: Implement DMA mapping operations for Apple DART
Mark Kettenis
kettenis at openbsd.org
Tue Jan 17 23:03:59 CET 2023
Implement translation table support for all the variations of
Apple's DART IOMMU that can be found on Apple's M1 and M2 SoCs.
Signed-off-by: Mark Kettenis <kettenis at openbsd.org>
---
drivers/iommu/apple_dart.c | 311 +++++++++++++++++++++++++++++++++----
1 file changed, 277 insertions(+), 34 deletions(-)
diff --git a/drivers/iommu/apple_dart.c b/drivers/iommu/apple_dart.c
index 2faacb8f3b..6ecd84303b 100644
--- a/drivers/iommu/apple_dart.c
+++ b/drivers/iommu/apple_dart.c
@@ -6,57 +6,296 @@
#include <common.h>
#include <cpu_func.h>
#include <dm.h>
+#include <iommu.h>
+#include <lmb.h>
+#include <memalign.h>
#include <asm/io.h>
#define DART_PARAMS2 0x0004
#define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
-#define DART_TLB_OP 0x0020
-#define DART_TLB_OP_OPMASK (0xfff << 20)
-#define DART_TLB_OP_FLUSH (0x001 << 20)
-#define DART_TLB_OP_BUSY BIT(2)
-#define DART_TLB_OP_SIDMASK 0x0034
-#define DART_ERROR_STATUS 0x0040
-#define DART_TCR(sid) (0x0100 + 4 * (sid))
-#define DART_TCR_TRANSLATE_ENABLE BIT(7)
-#define DART_TCR_BYPASS_DART BIT(8)
-#define DART_TCR_BYPASS_DAPF BIT(12)
-#define DART_TTBR(sid, idx) (0x0200 + 16 * (sid) + 4 * (idx))
-#define DART_TTBR_VALID BIT(31)
-#define DART_TTBR_SHIFT 12
-#define DART_T8110_TCR(sid) (0x1000 + 4 * (sid))
+#define DART_T8020_TLB_CMD 0x0020
+#define DART_T8020_TLB_CMD_FLUSH BIT(20)
+#define DART_T8020_TLB_CMD_BUSY BIT(2)
+#define DART_T8020_TLB_SIDMASK 0x0034
+#define DART_T8020_ERROR 0x0040
+#define DART_T8020_ERROR_ADDR_LO 0x0050
+#define DART_T8020_ERROR_ADDR_HI 0x0054
+#define DART_T8020_CONFIG 0x0060
+#define DART_T8020_CONFIG_LOCK BIT(15)
+#define DART_T8020_SID_ENABLE 0x00fc
+#define DART_T8020_TCR_BASE 0x0100
+#define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
+#define DART_T8020_TCR_BYPASS_DART BIT(8)
+#define DART_T8020_TCR_BYPASS_DAPF BIT(12)
+#define DART_T8020_TTBR_BASE 0x0200
+#define DART_T8020_TTBR_VALID BIT(31)
+
+#define DART_T8110_PARAMS4 0x000c
+#define DART_T8110_PARAMS4_NSID_MASK (0x1ff << 0)
+#define DART_T8110_TLB_CMD 0x0080
+#define DART_T8110_TLB_CMD_BUSY BIT(31)
+#define DART_T8110_TLB_CMD_FLUSH_ALL BIT(8)
+#define DART_T8110_ERROR 0x0100
+#define DART_T8110_ERROR_MASK 0x0104
+#define DART_T8110_ERROR_ADDR_LO 0x0170
+#define DART_T8110_ERROR_ADDR_HI 0x0174
+#define DART_T8110_PROTECT 0x0200
+#define DART_T8110_PROTECT_TTBR_TCR BIT(0)
+#define DART_T8110_SID_ENABLE_BASE 0x0c00
+#define DART_T8110_TCR_BASE 0x1000
#define DART_T8110_TCR_BYPASS_DAPF BIT(2)
#define DART_T8110_TCR_BYPASS_DART BIT(1)
#define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
-#define DART_T8110_TTBR(sid) (0x1400 + 4 * (sid))
+#define DART_T8110_TTBR_BASE 0x1400
+#define DART_T8110_TTBR_VALID BIT(0)
+
+#define DART_SID_ENABLE(priv, idx) \
+ ((priv)->sid_enable_base + 4 * (idx))
+#define DART_TCR(priv, sid) ((priv)->tcr_base + 4 * (sid))
+#define DART_TTBR(priv, sid, idx) \
+ ((priv)->ttbr_base + 4 * (priv)->nttbr * (sid) + 4 * (idx))
+#define DART_TTBR_SHIFT 12
+
+#define DART_ALL_STREAMS(priv) ((1U << (priv)->nsid) - 1)
+
+#define DART_PAGE_SIZE SZ_16K
+#define DART_PAGE_MASK (DART_PAGE_SIZE - 1)
+
+#define DART_L1_TABLE 0x3
+#define DART_L2_INVAL 0
+#define DART_L2_VALID BIT(0)
+#define DART_L2_FULL_PAGE BIT(1)
+#define DART_L2_START(addr) ((((addr) & DART_PAGE_MASK) >> 2) << 52)
+#define DART_L2_END(addr) ((((addr) & DART_PAGE_MASK) >> 2) << 40)
+
+struct apple_dart_priv {
+ void *base;
+ struct lmb lmb;
+ u64 *l1, *l2;
+ int bypass, shift;
+
+ dma_addr_t dvabase;
+ dma_addr_t dvaend;
+
+ int nsid;
+ int nttbr;
+ int sid_enable_base;
+ int tcr_base;
+ u32 tcr_translate_enable;
+ u32 tcr_bypass;
+ int ttbr_base;
+ u32 ttbr_valid;
+ void (*flush_tlb)(struct apple_dart_priv *priv);
+};
+
+static void apple_dart_t8020_flush_tlb(struct apple_dart_priv *priv)
+{
+ dsb();
+
+ writel(DART_ALL_STREAMS(priv), priv->base + DART_T8020_TLB_SIDMASK);
+ writel(DART_T8020_TLB_CMD_FLUSH, priv->base + DART_T8020_TLB_CMD);
+ while (readl(priv->base + DART_T8020_TLB_CMD) &
+ DART_T8020_TLB_CMD_BUSY)
+ continue;
+}
+
+static void apple_dart_t8110_flush_tlb(struct apple_dart_priv *priv)
+{
+ dsb();
+
+ writel(DART_T8110_TLB_CMD_FLUSH_ALL,
+ priv->base + DART_T8110_TLB_CMD_FLUSH_ALL);
+ while (readl(priv->base + DART_T8110_TLB_CMD) &
+ DART_T8110_TLB_CMD_BUSY)
+ continue;
+}
+
+static dma_addr_t apple_dart_map(struct udevice *dev, void *addr, size_t size)
+{
+ struct apple_dart_priv *priv = dev_get_priv(dev);
+ phys_addr_t paddr, dva;
+ phys_size_t psize, off;
+ int i, idx;
+
+ if (priv->bypass)
+ return (phys_addr_t)addr;
+
+ paddr = ALIGN_DOWN((phys_addr_t)addr, DART_PAGE_SIZE);
+ off = (phys_addr_t)addr - paddr;
+ psize = ALIGN(size + off, DART_PAGE_SIZE);
+
+ dva = lmb_alloc(&priv->lmb, psize, DART_PAGE_SIZE);
+
+ idx = dva / DART_PAGE_SIZE;
+ for (i = 0; i < psize / DART_PAGE_SIZE; i++) {
+ priv->l2[idx + i] = (paddr >> priv->shift) | DART_L2_VALID |
+ DART_L2_START(0LL) | DART_L2_END(~0LL);
+ paddr += DART_PAGE_SIZE;
+ }
+ flush_dcache_range((unsigned long)&priv->l2[idx],
+ (unsigned long)&priv->l2[idx + i]);
+ priv->flush_tlb(priv);
+
+ return dva + off;
+}
+
+static void apple_dart_unmap(struct udevice *dev, dma_addr_t addr, size_t size)
+{
+ struct apple_dart_priv *priv = dev_get_priv(dev);
+ phys_addr_t dva;
+ phys_size_t psize;
+ int i, idx;
+
+ if (priv->bypass)
+ return;
+
+ dva = ALIGN_DOWN(addr, DART_PAGE_SIZE);
+ psize = size + (addr - dva);
+ psize = ALIGN(psize, DART_PAGE_SIZE);
+
+ idx = dva / DART_PAGE_SIZE;
+ for (i = 0; i < psize / DART_PAGE_SIZE; i++)
+ priv->l2[idx + i] = DART_L2_INVAL;
+ flush_dcache_range((unsigned long)&priv->l2[idx],
+ (unsigned long)&priv->l2[idx + i]);
+ priv->flush_tlb(priv);
+
+ lmb_free(&priv->lmb, dva, psize);
+}
+
+static struct iommu_ops apple_dart_ops = {
+ .map = apple_dart_map,
+ .unmap = apple_dart_unmap,
+};
static int apple_dart_probe(struct udevice *dev)
{
- void *base;
+ struct apple_dart_priv *priv = dev_get_priv(dev);
+ dma_addr_t addr;
+ phys_addr_t l2;
+ int ntte, nl1, nl2;
int sid, i;
+ u32 params2, params4;
- base = dev_read_addr_ptr(dev);
- if (!base)
+ priv->base = dev_read_addr_ptr(dev);
+ if (!priv->base)
return -EINVAL;
- u32 params2 = readl(base + DART_PARAMS2);
- if (!(params2 & DART_PARAMS2_BYPASS_SUPPORT))
- return 0;
+ if (device_is_compatible(dev, "apple,t8110-dart")) {
+ params4 = readl(priv->base + DART_T8110_PARAMS4);
+ priv->nsid = params4 & DART_T8110_PARAMS4_NSID_MASK;
+ priv->nttbr = 1;
+ priv->sid_enable_base = DART_T8110_SID_ENABLE_BASE;
+ priv->tcr_base = DART_T8110_TCR_BASE;
+ priv->tcr_translate_enable = DART_T8110_TCR_TRANSLATE_ENABLE;
+ priv->tcr_bypass =
+ DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART;
+ priv->ttbr_base = DART_T8110_TTBR_BASE;
+ priv->ttbr_valid = DART_T8110_TTBR_VALID;
+ priv->flush_tlb = apple_dart_t8110_flush_tlb;
+ } else {
+ priv->nsid = 16;
+ priv->nttbr = 4;
+ priv->sid_enable_base = DART_T8020_SID_ENABLE;
+ priv->tcr_base = DART_T8020_TCR_BASE;
+ priv->tcr_translate_enable = DART_T8020_TCR_TRANSLATE_ENABLE;
+ priv->tcr_bypass =
+ DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART;
+ priv->ttbr_base = DART_T8020_TTBR_BASE;
+ priv->ttbr_valid = DART_T8020_TTBR_VALID;
+ priv->flush_tlb = apple_dart_t8020_flush_tlb;
+ }
+
+ if (device_is_compatible(dev, "apple,t6000-dart") ||
+ device_is_compatible(dev, "apple,t8110-dart"))
+ priv->shift = 4;
- if (device_is_compatible(dev, "apple,t8112-dart")) {
- for (sid = 0; sid < 256; sid++) {
- writel(DART_T8110_TCR_BYPASS_DART | DART_T8110_TCR_BYPASS_DAPF,
- base + DART_T8110_TCR(sid));
- writel(0, base + DART_T8110_TTBR(sid));
+ priv->dvabase = DART_PAGE_SIZE;
+ priv->dvaend = SZ_4G - DART_PAGE_SIZE;
+
+ lmb_init(&priv->lmb);
+ lmb_add(&priv->lmb, priv->dvabase, priv->dvaend - priv->dvabase);
+
+ /* Disable translations. */
+ for (sid = 0; sid < priv->nsid; sid++)
+ writel(0, priv->base + DART_TCR(priv, sid));
+
+ /* Remove page tables. */
+ for (sid = 0; sid < priv->nsid; sid++) {
+ for (i = 0; i < priv->nttbr; i++)
+ writel(0, priv->base + DART_TTBR(priv, sid, i));
+ }
+ priv->flush_tlb(priv);
+
+ params2 = readl(priv->base + DART_PARAMS2);
+ if (params2 & DART_PARAMS2_BYPASS_SUPPORT) {
+ for (sid = 0; sid < priv->nsid; sid++) {
+ writel(priv->tcr_bypass,
+ priv->base + DART_TCR(priv, sid));
}
- } else {
- for (sid = 0; sid < 16; sid++) {
- writel(DART_TCR_BYPASS_DART | DART_TCR_BYPASS_DAPF,
- base + DART_TCR(sid));
- for (i = 0; i < 4; i++)
- writel(0, base + DART_TTBR(sid, i));
+ priv->bypass = 1;
+ return 0;
+ }
+
+ ntte = DIV_ROUND_UP(priv->dvaend, DART_PAGE_SIZE);
+ nl2 = DIV_ROUND_UP(ntte, DART_PAGE_SIZE / sizeof(u64));
+ nl1 = DIV_ROUND_UP(nl2, DART_PAGE_SIZE / sizeof(u64));
+
+ priv->l2 = memalign(DART_PAGE_SIZE, nl2 * DART_PAGE_SIZE);
+ memset(priv->l2, 0, nl2 * DART_PAGE_SIZE);
+ flush_dcache_range((unsigned long)priv->l2,
+ (unsigned long)priv->l2 + nl2 * DART_PAGE_SIZE);
+
+ priv->l1 = memalign(DART_PAGE_SIZE, nl1 * DART_PAGE_SIZE);
+ memset(priv->l1, 0, nl1 * DART_PAGE_SIZE);
+ l2 = (phys_addr_t)priv->l2;
+ for (i = 0; i < nl2; i++) {
+ priv->l1[i] = (l2 >> priv->shift) | DART_L1_TABLE;
+ l2 += DART_PAGE_SIZE;
+ }
+ flush_dcache_range((unsigned long)priv->l1,
+ (unsigned long)priv->l1 + nl1 * DART_PAGE_SIZE);
+
+ /* Install page tables. */
+ for (sid = 0; sid < priv->nsid; sid++) {
+ addr = (phys_addr_t)priv->l1;
+ for (i = 0; i < nl1; i++) {
+ writel(addr >> DART_TTBR_SHIFT | priv->ttbr_valid,
+ priv->base + DART_TTBR(priv, sid, i));
+ addr += DART_PAGE_SIZE;
}
}
+ priv->flush_tlb(priv);
+
+ /* Enable all streams. */
+ for (i = 0; i < priv->nsid / 32; i++)
+ writel(~0, priv->base + DART_SID_ENABLE(priv, i));
+
+ /* Enable translations. */
+ for (sid = 0; sid < priv->nsid; sid++) {
+ writel(priv->tcr_translate_enable,
+ priv->base + DART_TCR(priv, sid));
+ }
+
+ return 0;
+}
+
+static int apple_dart_remove(struct udevice *dev)
+{
+ struct apple_dart_priv *priv = dev_get_priv(dev);
+ int sid, i;
+
+ /* Disable translations. */
+ for (sid = 0; sid < priv->nsid; sid++)
+ writel(0, priv->base + DART_TCR(priv, sid));
+
+ /* Remove page tables. */
+ for (sid = 0; sid < priv->nsid; sid++) {
+ for (i = 0; i < priv->nttbr; i++)
+ writel(0, priv->base + DART_TTBR(priv, sid, i));
+ }
+ priv->flush_tlb(priv);
return 0;
}
@@ -64,7 +303,7 @@ static int apple_dart_probe(struct udevice *dev)
static const struct udevice_id apple_dart_ids[] = {
{ .compatible = "apple,t8103-dart" },
{ .compatible = "apple,t6000-dart" },
- { .compatible = "apple,t8112-dart" },
+ { .compatible = "apple,t8110-dart" },
{ /* sentinel */ }
};
@@ -72,5 +311,9 @@ U_BOOT_DRIVER(apple_dart) = {
.name = "apple_dart",
.id = UCLASS_IOMMU,
.of_match = apple_dart_ids,
- .probe = apple_dart_probe
+ .priv_auto = sizeof(struct apple_dart_priv),
+ .ops = &apple_dart_ops,
+ .probe = apple_dart_probe,
+ .remove = apple_dart_remove,
+ .flags = DM_FLAG_OS_PREPARE
};
--
2.39.0
More information about the U-Boot
mailing list