soc/intel/xeon_sp/cpx: add VT-d support

Intel CPX-SP FSP added support for VT-d through adding UPD
parameter X2apic. Based on devicetree.cb setting, enable
VT-d programming through FSP-M.

When VT-d is enabled, add DMAR ACPI table.

Signed-off-by: Jonathan Zhang <jonzhang@fb.com>
Change-Id: Ic66374af6e53fb847c1bdc324eb3f4e01c334a94
Reviewed-on: https://review.coreboot.org/c/coreboot/+/44280
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Angel Pons <th3fanbus@gmail.com>
Reviewed-by: Maxim Polyakov <max.senia.poliak@gmail.com>

3 files changed, 324 insertions, 0 deletions

diff --git a/src/soc/intel/xeon_sp/cpx/acpi.c b/src/soc/intel/xeon_sp/cpx/acpi.c
index 62569e68b7..4c7336396c 100644
--- a/src/soc/intel/xeon_sp/cpx/acpi.c
+++ b/src/soc/intel/xeon_sp/cpx/acpi.c
@@ -448,12 +448,315 @@ static unsigned long acpi_fill_slit(unsigned long current)
 	return current;
 }
 
+/*
+ * Ports    Stack   Stack(HOB)  IioConfigIou
+ * ==========================================
+ * 0        CSTACK  stack 0     IOU0
+ * 1A..1D   PSTACK0	stack 1     IOU1
+ * 2A..2D   PSTACK1	stack 2     IOU2
+ * 3A..3D   PSTACK2	stack 4     IOU3
+ */
+static int get_stack_for_port(int p)
+{
+	if (p == 0)
+		return CSTACK;
+	else if (p >= PORT_1A && p <= PORT_1D)
+		return PSTACK0;
+	else if (p >= PORT_2A && p <= PORT_2D)
+		return PSTACK1;
+	else //if (p >= PORT_3A && p <= PORT_3D)
+		return PSTACK2;
+}
+
+static unsigned long acpi_create_drhd(unsigned long current, int socket, int stack)
+{
+	int IoApicID[] = {
+		// socket 0
+		PC00_IOAPIC_ID, PC01_IOAPIC_ID, PC02_IOAPIC_ID, PC03_IOAPIC_ID,
+		PC04_IOAPIC_ID, PC05_IOAPIC_ID,
+		// socket 1
+		PC06_IOAPIC_ID, PC07_IOAPIC_ID, PC08_IOAPIC_ID, PC09_IOAPIC_ID,
+		PC10_IOAPIC_ID, PC11_IOAPIC_ID,
+	};
+
+	uint32_t enum_id;
+	unsigned long tmp = current;
+
+	size_t hob_size;
+	const uint8_t fsp_hob_iio_universal_data_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID;
+	const IIO_UDS *hob = fsp_find_extension_hob_by_guid(
+		fsp_hob_iio_universal_data_guid, &hob_size);
+	assert(hob != NULL && hob_size != 0);
+
+	uint32_t bus = hob->PlatformData.IIO_resource[socket].StackRes[stack].BusBase;
+	uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment;
+	uint32_t reg_base =
+		hob->PlatformData.IIO_resource[socket].StackRes[stack].VtdBarAddress;
+	printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, pcie_seg: 0x%x, reg_base: 0x%x\n",
+		__func__, socket, stack, bus, pcie_seg, reg_base);
+
+	// Add DRHD Hardware Unit
+	if (socket == 0 && stack == CSTACK) {
+		printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, "
+			"Register Base Address: 0x%x\n",
+			DRHD_INCLUDE_PCI_ALL, pcie_seg, reg_base);
+		current += acpi_create_dmar_drhd(current, DRHD_INCLUDE_PCI_ALL,
+			pcie_seg, reg_base);
+	} else {
+		printk(BIOS_DEBUG, "[Hardware Unit Definition] Flags: 0x%x, PCI Segment Number: 0x%x, "
+			"Register Base Address: 0x%x\n", 0, pcie_seg, reg_base);
+		current += acpi_create_dmar_drhd(current, 0, pcie_seg, reg_base);
+	}
+
+	// Add PCH IOAPIC
+	if (socket == 0 && stack == CSTACK) {
+		printk(BIOS_DEBUG, "    [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, "
+			"PCI Path: 0x%x, 0x%x\n",
+			PCH_IOAPIC_ID, PCH_IOAPIC_BUS_NUMBER,
+			PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM);
+		current += acpi_create_dmar_ds_ioapic(current, PCH_IOAPIC_ID,
+			PCH_IOAPIC_BUS_NUMBER, PCH_IOAPIC_DEV_NUM, PCH_IOAPIC_FUNC_NUM);
+	}
+
+	// Add IOAPIC entry
+	enum_id = IoApicID[(socket*MAX_IIO_STACK)+stack];
+	printk(BIOS_DEBUG, "    [IOAPIC Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, "
+		"PCI Path: 0x%x, 0x%x\n", enum_id, bus, APIC_DEV_NUM, APIC_FUNC_NUM);
+	current += acpi_create_dmar_ds_ioapic(current, enum_id, bus,
+		APIC_DEV_NUM, APIC_FUNC_NUM);
+
+	// Add CBDMA devices for CSTACK
+	if (socket != 0 && stack == CSTACK) {
+		for (int cbdma_func_id = 0; cbdma_func_id < 8; ++cbdma_func_id) {
+			printk(BIOS_DEBUG, "    [PCI Endpoint Device] Enumeration ID: 0x%x, "
+				"PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n",
+				0, bus, CBDMA_DEV_NUM, cbdma_func_id);
+			current += acpi_create_dmar_ds_pci(current,
+				bus, CBDMA_DEV_NUM, cbdma_func_id);
+		}
+	}
+
+	// Add PCIe Ports
+	if (socket != 0 || stack != CSTACK) {
+		IIO_RESOURCE_INSTANCE iio_resource =
+			hob->PlatformData.IIO_resource[socket];
+		for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) {
+			if (get_stack_for_port(p) != stack)
+				continue;
+
+			uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device;
+			uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function;
+
+			uint32_t id = pci_mmio_read_config32(PCI_DEV(bus, dev, func),
+				PCI_VENDOR_ID);
+			if (id == 0xffffffff)
+				continue;
+
+			printk(BIOS_DEBUG, "    [PCI Bridge Device] Enumeration ID: 0x%x, "
+				"PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n",
+				0, bus, dev, func);
+			current += acpi_create_dmar_ds_pci_br(current,
+				bus, dev, func);
+		}
+
+		// Add VMD
+		if (hob->PlatformData.VMDStackEnable[socket][stack] &&
+			stack >= PSTACK0 && stack <= PSTACK2) {
+			printk(BIOS_DEBUG, "    [PCI Endpoint Device] Enumeration ID: 0x%x, "
+				"PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n",
+				 0, bus, VMD_DEV_NUM, VMD_FUNC_NUM);
+			current += acpi_create_dmar_ds_pci(current,
+				bus, VMD_DEV_NUM, VMD_FUNC_NUM);
+		}
+	}
+
+	// Add HPET
+	if (socket == 0 && stack == CSTACK) {
+		uint16_t hpet_capid = read16((void *)HPET_BASE_ADDRESS);
+		uint16_t num_hpets = (hpet_capid >> 0x08) & 0x1F;  // Bits [8:12] has hpet count
+		printk(BIOS_SPEW, "%s hpet_capid: 0x%x, num_hpets: 0x%x\n",
+			__func__, hpet_capid, num_hpets);
+		//BIT 15
+		if (num_hpets && (num_hpets != 0x1f) &&
+			(read32((void *)(HPET_BASE_ADDRESS + 0x100)) & (0x00008000))) {
+			printk(BIOS_DEBUG, "    [Message-capable HPET Device] Enumeration ID: 0x%x, "
+				"PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n",
+				0, HPET_BUS_NUM, HPET_DEV_NUM, HPET0_FUNC_NUM);
+			current += acpi_create_dmar_ds_msi_hpet(current, 0, HPET_BUS_NUM,
+				HPET_DEV_NUM, HPET0_FUNC_NUM);
+		}
+	}
+
+	acpi_dmar_drhd_fixup(tmp, current);
+
+	return current;
+}
+
+static unsigned long acpi_create_atsr(unsigned long current)
+{
+	size_t hob_size;
+	const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID;
+	const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size);
+	assert(hob != NULL && hob_size != 0);
+
+	for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) {
+		uint32_t pcie_seg = hob->PlatformData.CpuQpiInfo[socket].PcieSegment;
+		unsigned long tmp = current;
+		bool first = true;
+		IIO_RESOURCE_INSTANCE iio_resource =
+			hob->PlatformData.IIO_resource[socket];
+
+		for (int stack = 0; stack <= PSTACK2; ++stack) {
+			uint32_t bus = iio_resource.StackRes[stack].BusBase;
+			uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress;
+			if (!vtd_base)
+				continue;
+			uint64_t vtd_mmio_cap = read64((void *)(vtd_base + VTD_EXT_CAP_LOW));
+			printk(BIOS_SPEW, "%s socket: %d, stack: %d, bus: 0x%x, vtd_base: 0x%x, "
+				"vtd_mmio_cap: 0x%llx\n",
+				__func__, socket, stack, bus, vtd_base, vtd_mmio_cap);
+
+			// ATSR is applicable only for platform supporting device IOTLBs
+			// through the VT-d extended capability register
+			assert(vtd_mmio_cap != 0xffffffffffffffff);
+			if ((vtd_mmio_cap & 0x4) == 0) // BIT 2
+				continue;
+
+			for (int p = 0; p < NUMBER_PORTS_PER_SOCKET; ++p) {
+				if (socket == 0 && p == 0)
+					continue;
+				if (get_stack_for_port(p) != stack)
+					continue;
+
+				uint32_t dev = iio_resource.PcieInfo.PortInfo[p].Device;
+				uint32_t func = iio_resource.PcieInfo.PortInfo[p].Function;
+
+				u32 id = pci_mmio_read_config32(PCI_DEV(bus, dev, func),
+					PCI_VENDOR_ID);
+				if (id == 0xffffffff)
+					continue;
+
+				if (first) {
+					printk(BIOS_DEBUG, "[Root Port ATS Capability] Flags: 0x%x, "
+						"PCI Segment Number: 0x%x\n",
+						0, pcie_seg);
+					current += acpi_create_dmar_atsr(current, 0, pcie_seg);
+					first = 0;
+				}
+
+				printk(BIOS_DEBUG, "    [PCI Bridge Device] Enumeration ID: 0x%x, "
+					"PCI Bus Number: 0x%x, PCI Path: 0x%x, 0x%x\n",
+					0, bus, dev, func);
+				current += acpi_create_dmar_ds_pci_br(current, bus, dev, func);
+			}
+		}
+		if (tmp != current)
+			acpi_dmar_atsr_fixup(tmp, current);
+	}
+
+	return current;
+}
+
+static unsigned long acpi_create_rmrr(unsigned long current)
+{
+	uint32_t size = ALIGN_UP(MEM_BLK_COUNT * sizeof(MEM_BLK), 0x1000);
+
+	uint32_t *ptr;
+
+	// reserve memory
+	ptr = cbmem_find(CBMEM_ID_STORAGE_DATA);
+	if (!ptr) {
+		ptr = cbmem_add(CBMEM_ID_STORAGE_DATA, size);
+		assert(ptr != NULL);
+		memset(ptr, 0, size);
+	}
+
+	unsigned long tmp = current;
+	printk(BIOS_DEBUG, "[Reserved Memory Region] PCI Segment Number: 0x%x, Base Address: 0x%x, "
+		"End Address (limit): 0x%x\n",
+		0, (uint32_t) ptr, (uint32_t) ((uint32_t) ptr + size - 1));
+	current += acpi_create_dmar_rmrr(current, 0, (uint32_t) ptr,
+		(uint32_t) ((uint32_t) ptr + size - 1));
+
+	printk(BIOS_DEBUG, "    [PCI Endpoint Device] Enumeration ID: 0x%x, PCI Bus Number: 0x%x, "
+		"PCI Path: 0x%x, 0x%x\n",
+		 0, XHCI_BUS_NUMBER, PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM);
+	current += acpi_create_dmar_ds_pci(current, XHCI_BUS_NUMBER,
+		PCH_DEV_SLOT_XHCI, XHCI_FUNC_NUM);
+
+	acpi_dmar_rmrr_fixup(tmp, current);
+
+	return current;
+}
+
+static unsigned long acpi_create_rhsa(unsigned long current)
+{
+	size_t hob_size;
+	const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID;
+	const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size);
+	assert(hob != NULL && hob_size != 0);
+
+	for (int socket = 0; socket < hob->PlatformData.numofIIO; ++socket) {
+		IIO_RESOURCE_INSTANCE iio_resource =
+			hob->PlatformData.IIO_resource[socket];
+		for (int stack = 0; stack <= PSTACK2; ++stack) {
+			uint32_t vtd_base = iio_resource.StackRes[stack].VtdBarAddress;
+			if (!vtd_base)
+				continue;
+
+			printk(BIOS_DEBUG, "[Remapping Hardware Static Affinity] Base Address: 0x%x, "
+				"Proximity Domain: 0x%x\n", vtd_base, socket);
+			current += acpi_create_dmar_rhsa(current, vtd_base, socket);
+		}
+	}
+
+	return current;
+}
+
+static unsigned long acpi_fill_dmar(unsigned long current)
+{
+	size_t hob_size;
+	const uint8_t uds_guid[16] = FSP_HOB_IIO_UNIVERSAL_DATA_GUID;
+	const IIO_UDS *hob = fsp_find_extension_hob_by_guid(uds_guid, &hob_size);
+	assert(hob != NULL && hob_size != 0);
+
+	// DRHD
+	for (int iio = 1; iio <= hob->PlatformData.numofIIO; ++iio) {
+		int socket = iio;
+		if (socket == hob->PlatformData.numofIIO) // socket 0 should be last DRHD entry
+			socket = 0;
+
+		if (socket == 0) {
+			for (int stack = 1; stack <= PSTACK2; ++stack)
+				current = acpi_create_drhd(current, socket, stack);
+			current = acpi_create_drhd(current, socket, CSTACK);
+		} else {
+			for (int stack = 0; stack <= PSTACK2; ++stack)
+				current = acpi_create_drhd(current, socket, stack);
+		}
+	}
+
+	// RMRR
+	current = acpi_create_rmrr(current);
+
+	// ATSR - causes hang
+	current = acpi_create_atsr(current);
+
+	// RHSA
+	current = acpi_create_rhsa(current);
+
+	return current;
+}
+
 unsigned long northbridge_write_acpi_tables(const struct device *device,
 					    unsigned long current,
 					    struct acpi_rsdp *rsdp)
 {
 	acpi_srat_t *srat;
 	acpi_slit_t *slit;
+	acpi_dmar_t *dmar;
+
+	const struct soc_intel_xeon_sp_cpx_config *const config = config_of(device);
 
 	/* SRAT */
 	current = ALIGN(current, 8);
@@ -471,5 +774,18 @@ unsigned long northbridge_write_acpi_tables(const struct device *device,
 	current += slit->header.length;
 	acpi_add_table(rsdp, slit);
 
+	/* DMAR */
+	if (config->vtd_support) {
+		current = ALIGN(current, 8);
+		dmar = (acpi_dmar_t *)current;
+		printk(BIOS_DEBUG, "ACPI:    * DMAR\n");
+		printk(BIOS_DEBUG, "[DMA Remapping table] Flags: 0x%x\n",
+			(DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT));
+		acpi_create_dmar(dmar, (DMAR_INTR_REMAP | DMAR_X2APIC_OPT_OUT), acpi_fill_dmar);
+		current += dmar->header.length;
+		current = acpi_align_current(current);
+		acpi_add_table(rsdp, dmar);
+	}
+
 	return current;
 }
diff --git a/src/soc/intel/xeon_sp/cpx/chip.h b/src/soc/intel/xeon_sp/cpx/chip.h
index aa605a4aad..434b343bb2 100644
--- a/src/soc/intel/xeon_sp/cpx/chip.h
+++ b/src/soc/intel/xeon_sp/cpx/chip.h
@@ -84,6 +84,9 @@ struct soc_intel_xeon_sp_cpx_config {
 
 	uint32_t pstate_req_ratio;
 
+	uint8_t vtd_support;
+	uint8_t x2apic;
+
 	/* Generic IO decode ranges */
 	uint32_t gen1_dec;
 	uint32_t gen2_dec;
diff --git a/src/soc/intel/xeon_sp/cpx/romstage.c b/src/soc/intel/xeon_sp/cpx/romstage.c
index 6c65994412..a198c993f3 100644
--- a/src/soc/intel/xeon_sp/cpx/romstage.c
+++ b/src/soc/intel/xeon_sp/cpx/romstage.c
@@ -15,6 +15,7 @@ void platform_fsp_memory_init_params_cb(FSPM_UPD *mupd, uint32_t version)
 {
 	FSPM_CONFIG *m_cfg = &mupd->FspmConfig;
 	const struct device *dev;
+	const config_t *config = config_of_soc();
 
 	/* ErrorLevel - 0 (disable) to 8 (verbose) */
 	m_cfg->DebugPrintLevel = 8;
@@ -68,5 +69,9 @@ void platform_fsp_memory_init_params_cb(FSPM_UPD *mupd, uint32_t version)
 	dev = pcidev_path_on_root(PCH_DEVFN_THERMAL);
 	m_cfg->ThermalDeviceEnable = dev && dev->enabled;
 
+	/* Enable VT-d according to DTB */
+	m_cfg->VtdSupport = config->vtd_support;
+	m_cfg->X2apic = config->x2apic;
+
 	mainboard_memory_init_params(mupd);
 }