device: Add support for multiple PCI segment groups

Add initial support for multiple PCI segment groups. Instead of
modifying secondary in the bus struct introduce a new segment_group
struct element and keep existing common code.

Since all platforms currently only use 1 segment this is not a
functional change. On platforms that support more than 1 segment the
segment has to be set when creating the PCI domain.

Signed-off-by: Patrick Rudolph <patrick.rudolph@9elements.com>
Signed-off-by: Felix Held <felix-coreboot@felixheld.de>
Change-Id: Ied3313c41896362dd989ee2ab1b1bcdced840aa8
Reviewed-on: https://review.coreboot.org/c/coreboot/+/79927
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Nico Huber <nico.h@gmx.de>
Reviewed-by: Martin Roth <martin.roth@amd.corp-partner.google.com>

src/acpi/acpi.c

@@ -150,9 +150,14 @@ static void acpi_create_madt(acpi_header_t *header, void *unused)
 
 static unsigned long acpi_fill_mcfg(unsigned long current)
 {
-	current += acpi_create_mcfg_mmconfig((acpi_mcfg_mmconfig_t *)current,
-			CONFIG_ECAM_MMCONF_BASE_ADDRESS, 0, 0,
-			CONFIG_ECAM_MMCONF_BUS_NUMBER - 1);
+	for (int i = 0; i < PCI_SEGMENT_GROUP_COUNT; i++) {
+		current += acpi_create_mcfg_mmconfig((acpi_mcfg_mmconfig_t *)current,
+			CONFIG_ECAM_MMCONF_BASE_ADDRESS + i * PCI_PER_SEGMENT_GROUP_ECAM_SIZE,
+			i,
+			0,
+			PCI_BUSES_PER_SEGMENT_GROUP - 1);
+	}
+
 	return current;
 }
 
@@ -695,6 +700,7 @@ void acpi_create_ipmi(const struct device *device,
 
 	if (device->path.type == DEVICE_PATH_PCI) {
 		spmi->pci_device_flag = ACPI_IPMI_PCI_DEVICE_FLAG;
+		spmi->pci_segment_group = device->bus->segment_group;
 		spmi->pci_bus = device->bus->secondary;
 		spmi->pci_device = device->path.pci.devfn >> 3;
 		spmi->pci_function = device->path.pci.devfn & 0x7;

src/arch/x86/mpspec.c

@@ -8,11 +8,16 @@
 #include <cpu/x86/lapic.h>
 #include <device/device.h>
 #include <device/path.h>
+#include <device/pci_def.h>
 #include <device/pci_ids.h>
 #include <identity.h>
 #include <stdint.h>
 #include <string.h>
 
+#if CONFIG(ECAM_MMCONF_SUPPORT) && PCI_SEGMENT_GROUP_COUNT > 1
+#error "MPTable doesn't support systems with multiple PCI segment groups"
+#endif
+
 /* Initialize the specified "mc" struct with initial values. */
 void mptable_init(struct mp_config_table *mc)
 {

src/device/Kconfig

@@ -589,6 +589,13 @@ config ECAM_MMCONF_BASE_ADDRESS
 config ECAM_MMCONF_BUS_NUMBER
 	int
 	depends on ECAM_MMCONF_SUPPORT
+	help
+	  Total number of PCI buses in the system across all segment groups.
+	  The number needs to be a power of 2. For values <= 256,
+	  PCI_BUSES_PER_SEGMENT_GROUP is CONFIG_ECAM_MMCONF_BUS_NUMBER and
+	  PCI_SEGMENT_GROUP_COUNT is 1. For values > 256,
+	  PCI_BUSES_PER_SEGMENT_GROUP is 256 and PCI_SEGMENT_GROUP_COUNT is
+	  CONFIG_ECAM_MMCONF_BUS_NUMBER / 256.
 
 config ECAM_MMCONF_LENGTH
 	hex
@@ -597,6 +604,8 @@ config ECAM_MMCONF_LENGTH
 	default 0x04000000 if ECAM_MMCONF_BUS_NUMBER = 64
 	default 0x08000000 if ECAM_MMCONF_BUS_NUMBER = 128
 	default 0x10000000 if ECAM_MMCONF_BUS_NUMBER = 256
+	default 0x20000000 if ECAM_MMCONF_BUS_NUMBER = 512
+	default 0x80000000 if ECAM_MMCONF_BUS_NUMBER = 1024
 	default 0x0
 
 config PCI_ALLOW_BUS_MASTER

src/device/device.c

@@ -152,8 +152,8 @@ static void read_resources(struct bus *bus)
 {
 	struct device *curdev;
 
-	printk(BIOS_SPEW, "%s %s bus %d link: %d\n", dev_path(bus->dev),
-	       __func__, bus->secondary, bus->link_num);
+	printk(BIOS_SPEW, "%s %s segment group %d bus %d link: %d\n", dev_path(bus->dev),
+	       __func__, bus->segment_group, bus->secondary, bus->link_num);
 
 	/* Walk through all devices and find which resources they need. */
 	for (curdev = bus->children; curdev; curdev = curdev->sibling) {
@@ -176,8 +176,8 @@ static void read_resources(struct bus *bus)
 			read_resources(link);
 	}
 	post_log_clear();
-	printk(BIOS_SPEW, "%s %s bus %d link: %d done\n",
-	       dev_path(bus->dev), __func__, bus->secondary, bus->link_num);
+	printk(BIOS_SPEW, "%s %s segment group %d bus %d link: %d done\n",
+	       dev_path(bus->dev), __func__, bus->segment_group, bus->secondary, bus->link_num);
 }
 
 struct device *vga_pri = NULL;
@@ -266,8 +266,8 @@ void assign_resources(struct bus *bus)
 {
 	struct device *curdev;
 
-	printk(BIOS_SPEW, "%s %s, bus %d link: %d\n",
-	       dev_path(bus->dev), __func__, bus->secondary, bus->link_num);
+	printk(BIOS_SPEW, "%s %s, segment group %d bus %d link: %d\n",
+	       dev_path(bus->dev), __func__, bus->segment_group, bus->secondary, bus->link_num);
 
 	for (curdev = bus->children; curdev; curdev = curdev->sibling) {
 		if (!curdev->enabled || !curdev->resource_list)
@@ -282,8 +282,8 @@ void assign_resources(struct bus *bus)
 		curdev->ops->set_resources(curdev);
 	}
 	post_log_clear();
-	printk(BIOS_SPEW, "%s %s, bus %d link: %d done\n",
-	       dev_path(bus->dev), __func__, bus->secondary, bus->link_num);
+	printk(BIOS_SPEW, "%s %s, segment group %d bus %d link: %d done\n",
+	       dev_path(bus->dev), __func__, bus->segment_group, bus->secondary, bus->link_num);
 }
 
 /**

src/device/device_const.c

@@ -35,6 +35,7 @@ static DEVTREE_CONST struct device *dev_find_slot(unsigned int bus,
 	for (dev = all_devices; dev; dev = dev->next) {
 		if ((dev->path.type == DEVICE_PATH_PCI) &&
 		    (dev->bus->secondary == bus) &&
+		    (dev->bus->segment_group == 0) &&
 		    (dev->path.pci.devfn == devfn)) {
 			result = dev;
 			break;
@@ -233,7 +234,7 @@ DEVTREE_CONST struct device *pcidev_path_on_bus(unsigned int bus, pci_devfn_t de
 			dev = dev->next;
 			continue;
 		}
-		if (dev->bus->secondary == bus)
+		if (dev->bus->secondary == bus && dev->bus->segment_group == 0)
 			return pcidev_path_behind(dev->bus, devfn);
 		dev = dev->next;
 	}

src/device/device_util.c

@@ -97,7 +97,7 @@ u32 dev_path_encode(const struct device *dev)
 	case DEVICE_PATH_ROOT:
 		break;
 	case DEVICE_PATH_PCI:
-		ret |= dev->bus->secondary << 8 | dev->path.pci.devfn;
+		ret |= dev->bus->segment_group << 16 | dev->bus->secondary << 8 | dev->path.pci.devfn;
 		break;
 	case DEVICE_PATH_PNP:
 		ret |= dev->path.pnp.port << 8 | dev->path.pnp.device;
@@ -168,7 +168,8 @@ const char *dev_path(const struct device *dev)
 			break;
 		case DEVICE_PATH_PCI:
 			snprintf(buffer, sizeof(buffer),
-				 "PCI: %02x:%02x.%01x",
+				 "PCI: %02x:%02x:%02x.%01x",
+				 dev->bus->segment_group,
 				 dev->bus->secondary,
 				 PCI_SLOT(dev->path.pci.devfn),
 				 PCI_FUNC(dev->path.pci.devfn));
@@ -525,7 +526,8 @@ void report_resource_stored(struct device *dev, const struct resource *resource,
 
 	if (dev->link_list && (resource->flags & IORESOURCE_PCI_BRIDGE)) {
 		snprintf(buf, sizeof(buf),
-			 "bus %02x ", dev->link_list->secondary);
+			 "seg %02x bus %02x ", dev->link_list->segment_group,
+			 dev->link_list->secondary);
 	}
 	printk(BIOS_DEBUG, "%s %02lx <- [0x%016llx - 0x%016llx] size 0x%08llx "
 	       "gran 0x%02x %s%s%s\n", dev_path(dev), resource->index,
@@ -982,5 +984,5 @@ bool is_enabled_pci(const struct device *pci)
 
 bool is_pci_dev_on_bus(const struct device *pci, unsigned int bus)
 {
-	return is_pci(pci) && pci->bus->secondary == bus;
+	return is_pci(pci) && pci->bus->segment_group == 0 && pci->bus->secondary == bus;
 }

src/device/pci_device.c

@@ -846,6 +846,12 @@ static int should_run_oprom(struct device *dev, struct rom_header *rom)
 {
 	static int should_run = -1;
 
+	if (dev->bus->segment_group) {
+		printk(BIOS_ERR, "Only option ROMs of devices in first PCI segment group can "
+				 "be run.\n");
+		return 0;
+	}
+
 	if (CONFIG(VENDORCODE_ELTAN_VBOOT))
 		if (rom != NULL)
 			if (!verified_boot_should_run_oprom(rom))
@@ -1314,7 +1320,8 @@ struct device *pci_probe_dev(struct device *dev, struct bus *bus,
  */
 unsigned int pci_match_simple_dev(struct device *dev, pci_devfn_t sdev)
 {
-	return dev->bus->secondary == PCI_DEV2SEGBUS(sdev) &&
+	return dev->bus->secondary == PCI_DEV2BUS(sdev) &&
+			dev->bus->segment_group == PCI_DEV2SEG(sdev) &&
 			dev->path.pci.devfn == PCI_DEV2DEVFN(sdev);
 }
 
@@ -1428,7 +1435,8 @@ void pci_scan_bus(struct bus *bus, unsigned int min_devfn,
 	struct device *dev, **prev;
 	int once = 0;
 
-	printk(BIOS_DEBUG, "PCI: %s for bus %02x\n", __func__, bus->secondary);
+	printk(BIOS_DEBUG, "PCI: %s for segment group %02x bus %02x\n", __func__,
+	       bus->segment_group, bus->secondary);
 
 	/* Maximum sane devfn is 0xFF. */
 	if (max_devfn > 0xff) {
@@ -1549,7 +1557,8 @@ static void pci_bridge_route(struct bus *link, scan_state state)
 		link->subordinate = link->secondary + dev->hotplug_buses;
 		link->max_subordinate = parent->max_subordinate
 						? parent->max_subordinate
-						: (CONFIG_ECAM_MMCONF_BUS_NUMBER - 1);
+						: (PCI_BUSES_PER_SEGMENT_GROUP - 1);
+		link->segment_group = parent->segment_group;
 	}
 
 	if (link->secondary > link->max_subordinate)

src/device/pci_rom.c

@@ -230,6 +230,10 @@ ati_rom_acpi_fill_vfct(const struct device *device, acpi_vfct_t *vfct_struct,
 		printk(BIOS_ERR, "%s failed\n", __func__);
 		return current;
 	}
+	if (device->bus->segment_group) {
+		printk(BIOS_ERR, "VFCT only supports GPU in first PCI segment group.\n");
+		return current;
+	}
 
 	printk(BIOS_DEBUG, "           Copying %sVBIOS image from %p\n",
 			rom == (struct rom_header *)

src/device/pcix_device.c

@@ -110,8 +110,8 @@ void pcix_scan_bridge(struct device *dev)
 		pcix_tune_bus(dev->link_list);
 
 	/* Print the PCI-X bus speed. */
-	printk(BIOS_DEBUG, "PCI: %02x: %s\n", dev->link_list->secondary,
-	       pcix_speed(sstatus));
+	printk(BIOS_DEBUG, "PCI: %02x:%02x: %s\n", dev->link_list->segment_group,
+	       dev->link_list->secondary, pcix_speed(sstatus));
 }
 
 /** Default device operations for PCI-X bridges */

src/include/device/device.h

@@ -86,6 +86,7 @@ struct bus {
 	uint16_t	secondary;		/* secondary bus number */
 	uint16_t	subordinate;		/* subordinate bus number */
 	uint16_t	max_subordinate;	/* max subordinate bus number */
+	uint8_t		segment_group;		/* PCI segment group */
 
 	unsigned int	reset_needed : 1;
 	unsigned int	no_vga16 : 1;		/* No support for 16-bit VGA decoding */

src/include/device/pci_def.h

@@ -591,9 +591,22 @@
 #define PCI_SLOT(devfn)		(((devfn) >> 3) & 0x1f)
 #define PCI_FUNC(devfn)		((devfn) & 0x07)
 
+/*
+ * CONFIG_ECAM_MMCONF_BUS_NUMBER is a power of 2. For values <= 256,
+ * PCI_BUSES_PER_SEGMENT_GROUP is CONFIG_ECAM_MMCONF_BUS_NUMBER and PCI_SEGMENT_GROUP_COUNT
+ * is 1. For values > 256, PCI_BUSES_PER_SEGMENT_GROUP is 256 and PCI_SEGMENT_GROUP_COUNT is
+ * CONFIG_ECAM_MMCONF_BUS_NUMBER / 256.
+*/
+#define PCI_BUS_NUMBER_MASK		0xff
+#define PCI_SEGMENT_GROUP_COUNT		(((CONFIG_ECAM_MMCONF_BUS_NUMBER - 1) >> 8) + 1)
+#define PCI_BUSES_PER_SEGMENT_GROUP	(((CONFIG_ECAM_MMCONF_BUS_NUMBER - 1) & PCI_BUS_NUMBER_MASK) + 1)
+#define PCI_PER_SEGMENT_GROUP_ECAM_SIZE	(256 * MiB)
+
 /* Translation from PCI_DEV() to devicetree bus and path.pci.devfn. */
 #define PCI_DEV2DEVFN(sdev)		(((sdev)>>12) & 0xff)
 #define PCI_DEV2SEGBUS(sdev)	(((sdev)>>20) & 0xfff)
+#define PCI_DEV2BUS(sdev)	(((sdev)>>20) & PCI_BUS_NUMBER_MASK)
+#define PCI_DEV2SEG(sdev)	(((sdev)>>28) & 0xf)
 
 /* Fields from within the device's class value. */
 #define PCI_CLASS_GET_DEVICE(c) (c >> 8)

src/include/device/pci_ops.h

@@ -12,7 +12,8 @@ void __noreturn pcidev_die(void);
 
 static __always_inline pci_devfn_t pcidev_bdf(const struct device *dev)
 {
-	return (dev->path.pci.devfn << 12) | (dev->bus->secondary << 20);
+	return (dev->path.pci.devfn << 12) | (dev->bus->secondary << 20) |
+			(dev->bus->segment_group << 28);
 }
 
 static __always_inline pci_devfn_t pcidev_assert(const struct device *dev)

src/lib/smbios.c

@@ -1114,7 +1114,7 @@ static int smbios_generate_type41_from_devtree(struct device *dev, int *handle,
 	return smbios_write_type41(current, handle,
 					name, // name
 					instance_id, // inst
-					0, // segment group
+					dev->bus->segment_group, // segment group
 					dev->bus->secondary, //bus
 					PCI_SLOT(dev->path.pci.devfn), // device
 					PCI_FUNC(dev->path.pci.devfn), // func
@@ -1167,7 +1167,7 @@ static int smbios_generate_type9_from_devtree(struct device *dev, int *handle,
 				  0,
 				  1,
 				  0,
-				  0,
+				  dev->bus->segment_group,
 				  dev->bus->secondary,
 				  dev->path.pci.devfn);
 }

src/northbridge/amd/pi/00730F01/northbridge.c

@@ -296,7 +296,7 @@ static unsigned long acpi_fill_ivrs11(unsigned long current, acpi_ivrs_t *ivrs_a
 	ivhd_11->capability_offset = 0x40;
 	ivhd_11->iommu_base_low = ivrs_agesa->ivhd.iommu_base_low;
 	ivhd_11->iommu_base_high = ivrs_agesa->ivhd.iommu_base_high;
-	ivhd_11->pci_segment_group = 0x0000;
+	ivhd_11->pci_segment_group = nb_dev->bus->segment_group;
 	ivhd_11->iommu_info = ivrs_agesa->ivhd.iommu_info;
 	ivhd_11->iommu_attributes.perf_counters =
 		(IOMMU_MMIO32(ivhd_11->iommu_base_low + 0x4000) >> 7) & 0xf;
@@ -364,7 +364,7 @@ static unsigned long acpi_fill_ivrs(acpi_ivrs_t *ivrs, unsigned long current)
 		ivrs->ivhd.capability_offset = 0x40;
 		ivrs->ivhd.iommu_base_low = ivrs_agesa->ivhd.iommu_base_low;
 		ivrs->ivhd.iommu_base_high = ivrs_agesa->ivhd.iommu_base_high;
-		ivrs->ivhd.pci_segment_group = 0x0000;
+		ivrs->ivhd.pci_segment_group = nb_dev->bus->segment_group;
 		ivrs->ivhd.iommu_info = ivrs_agesa->ivhd.iommu_info;
 		ivrs->ivhd.iommu_feature_info = ivrs_agesa->ivhd.iommu_feature_info;
 		/* Enable EFR if supported */

src/soc/amd/common/block/acpi/ivrs.c

@@ -218,7 +218,7 @@ static unsigned long acpi_fill_ivrs40(unsigned long current, acpi_ivrs_ivhd_t *i
 	ivhd_40->capability_offset = pci_find_capability(iommu_dev, IOMMU_CAP_ID);
 	ivhd_40->iommu_base_low = ivhd->iommu_base_low;
 	ivhd_40->iommu_base_high = ivhd->iommu_base_high;
-	ivhd_40->pci_segment_group = 0x0000;
+	ivhd_40->pci_segment_group = nb_dev->bus->segment_group;
 	ivhd_40->iommu_info = ivhd->iommu_info;
 	/* For type 40h bits 31:28 and 12:0 are reserved */
 	ivhd_40->iommu_attributes = ivhd->iommu_feature_info & 0xfffe000;
@@ -275,7 +275,7 @@ static unsigned long acpi_fill_ivrs11(unsigned long current, acpi_ivrs_ivhd_t *i
 	ivhd_11->capability_offset = pci_find_capability(iommu_dev, IOMMU_CAP_ID);
 	ivhd_11->iommu_base_low = ivhd->iommu_base_low;
 	ivhd_11->iommu_base_high = ivhd->iommu_base_high;
-	ivhd_11->pci_segment_group = 0x0000;
+	ivhd_11->pci_segment_group = nb_dev->bus->segment_group;
 	ivhd_11->iommu_info = ivhd->iommu_info;
 	ivhd11_attr_ptr = (ivhd11_iommu_attr_t *)&ivhd->iommu_feature_info;
 	ivhd_11->iommu_attributes.perf_counters = ivhd11_attr_ptr->perf_counters;
@@ -365,7 +365,7 @@ unsigned long acpi_fill_ivrs(acpi_ivrs_t *ivrs, unsigned long current)
 		ivhd->flags |= ((mmio_x18_value & MMIO_CTRL_HT_TUN_EN) ?
 							IVHD_FLAG_HT_TUN_EN : 0);
 
-		ivhd->pci_segment_group = 0x0000;
+		ivhd->pci_segment_group = nb_dev->bus->segment_group;
 
 		ivhd->iommu_info = pci_read_config16(iommu_dev,
 			ivhd->capability_offset + 0x10) & 0x1F;

src/soc/amd/common/block/data_fabric/domain.c

@@ -9,6 +9,7 @@
 #include <cpu/amd/mtrr.h>
 #include <cpu/cpu.h>
 #include <device/device.h>
+#include <device/pci.h>
 #include <device/pci_ops.h>
 #include <types.h>
 
@@ -21,23 +22,24 @@ void amd_pci_domain_scan_bus(struct device *domain)
 		return;
 	}
 
-	/* TODO: Implement support for more than one PCI segment group in coreboot */
-	if (segment_group) {
-		printk(BIOS_ERR, "coreboot currently only supports one PCI segment group.\n");
+	if (segment_group >= PCI_SEGMENT_GROUP_COUNT) {
+		printk(BIOS_ERR, "Skipping domain %u due to too large segment group %u.\n",
+		       domain->path.domain.domain, segment_group);
 		return;
 	}
 
-	/* TODO: Check if bus >= CONFIG_ECAM_MMCONF_BUS_NUMBER and return in that case */
+	/* TODO: Check if bus >= PCI_BUSES_PER_SEGMENT_GROUP and return in that case */
 
-	/* Make sure to not report more than CONFIG_ECAM_MMCONF_BUS_NUMBER PCI buses */
-	limit = MIN(limit, CONFIG_ECAM_MMCONF_BUS_NUMBER - 1);
+	/* Make sure to not report more than PCI_BUSES_PER_SEGMENT_GROUP PCI buses */
+	limit = MIN(limit, PCI_BUSES_PER_SEGMENT_GROUP - 1);
 
 	/* Set bus first number of PCI root */
 	domain->link_list->secondary = bus;
 	/* subordinate needs to be the same as secondary before pci_host_bridge_scan_bus call. */
 	domain->link_list->subordinate = bus;
 	/* Tell allocator about maximum PCI bus number in domain */
 	domain->link_list->max_subordinate = limit;
+	domain->link_list->segment_group = segment_group;
 
 	pci_host_bridge_scan_bus(domain);
 }
@@ -246,12 +248,13 @@ void amd_pci_domain_fill_ssdt(const struct device *domain)
 	acpigen_write_resourcetemplate_header();
 
 	/* PCI bus number range in domain */
-	printk(BIOS_DEBUG, "%s _CRS: adding busses [%x-%x]\n", acpi_device_name(domain),
-	       domain->link_list->secondary, domain->link_list->max_subordinate);
+	printk(BIOS_DEBUG, "%s _CRS: adding busses [%x-%x] in segment group %x\n",
+	       acpi_device_name(domain), domain->link_list->secondary,
+	       domain->link_list->max_subordinate, domain->link_list->segment_group);
 	acpigen_resource_producer_bus_number(domain->link_list->secondary,
 					     domain->link_list->max_subordinate);
 
-	if (domain->link_list->secondary == 0) {
+	if (domain->link_list->secondary == 0 && domain->link_list->segment_group == 0) {
 		/* ACPI 6.4.2.5 I/O Port Descriptor */
 		acpigen_write_io16(PCI_IO_CONFIG_INDEX, PCI_IO_CONFIG_LAST_PORT, 1,
 				   PCI_IO_CONFIG_PORT_COUNT, 1);
@@ -287,7 +290,7 @@ void amd_pci_domain_fill_ssdt(const struct device *domain)
 
 	acpigen_write_resourcetemplate_footer();
 
-	acpigen_write_SEG(0);
+	acpigen_write_SEG(domain->link_list->segment_group);
 	acpigen_write_BBN(domain->link_list->secondary);
 
 	/* Scope */