pciexp: Add support for allocating PCI express hotplug resources

This change adds support for allocating resources for PCI express hotplug
bridges when PCIEXP_HOTPLUG is selected. By default, this will add 32 PCI
subordinate numbers (buses), 256 MiB of prefetchable memory, 8 MiB of
non-prefetchable memory, and 8 KiB of I/O space to any device with the
PCI_EXP_SLTCAP_HPC bit set in the PCI_EXP_SLTCAP register, which
indicates hot-plugging capability. The resource allocation is configurable,
please see the PCIEXP_HOTPLUG_* variables in src/device/Kconfig.

In order to support the allocation of hotplugged PCI buses, a new field
is added to struct device called hotplug_buses. This is defaulted to
zero, but when set, it adds the hotplug_buses value to the subordinate
value of the PCI bridge. This allows devices to be plugged in and
unplugged after boot.

This code was tested on the System76 Darter Pro (darp6). Before this
change, there are not enough resources allocated to the Thunderbolt
PCI bridge to allow plugging in new devices after boot. This can be
worked around in the Linux kernel by passing a boot param such as:
pci=assign-busses,hpbussize=32,realloc

This change makes it possible to use Thunderbolt hotplugging without
kernel parameters, and attempts to match closely what our motherboard
manufacturer's firmware does by default.

Signed-off-by: Jeremy Soller <jeremy@system76.com>
Change-Id: I500191626584b83e6a8ae38417fd324b5e803afc
Reviewed-on: https://review.coreboot.org/c/coreboot/+/35946
Reviewed-by: Philipp Deppenwiese <zaolin.daisuki@gmail.com>
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>

src/device/Kconfig

@@ -555,6 +555,50 @@ config PCIEXP_L1_SUB_STATE
 	help
 	  Detect and enable ASPM on PCIe links.
 
+config PCIEXP_HOTPLUG
+	prompt "Enable PCIe Hotplug Support"
+	bool
+	default n
+	help
+	  Allocate resources for PCIe hotplug bridges
+
+if PCIEXP_HOTPLUG
+
+config PCIEXP_HOTPLUG_BUSES
+	int "PCI Express Hotplug Buses"
+	default 32
+	help
+	  This is the number of buses allocated for hotplug PCI express
+	  bridges, for use by hotplugged child devices. The default is 32
+	  buses.
+
+config PCIEXP_HOTPLUG_MEM
+	hex "PCI Express Hotplug Memory"
+	default 0x800000
+	help
+	  This is the amount of memory space, in bytes, to allocate to
+	  hotplug PCI express bridges, for use by hotplugged child devices.
+	  This size should be page-aligned. The default is 8 MiB.
+
+config PCIEXP_HOTPLUG_PREFETCH_MEM
+	hex "PCI Express Hotplug Prefetch Memory"
+	default 0x10000000
+	help
+	  This is the amount of pre-fetchable memory space, in bytes, to
+	  allocate to hot-plug PCI express bridges, for use by hotplugged
+	  child devices. This size should be page-aligned. The default is
+	  256 MiB.
+
+config PCIEXP_HOTPLUG_IO
+	hex "PCI Express Hotplug I/O Space"
+	default 0x2000
+	help
+	  This is the amount of I/O space to allocate to hot-plug PCI
+	  express bridges, for use by hotplugged child devices. The default
+	  is 8 KiB.
+
+endif # PCIEXP_HOTPLUG
+
 endif # PCIEXP_PLUGIN_SUPPORT
 
 config EARLY_PCI_BRIDGE

src/device/pci_device.c

@@ -878,6 +878,14 @@ static struct device_operations *get_pci_bridge_ops(struct device *dev)
 		case PCI_EXP_TYPE_DOWNSTREAM:
 			printk(BIOS_DEBUG, "%s subordinate bus PCI Express\n",
 			       dev_path(dev));
+#if CONFIG(PCIEXP_HOTPLUG)
+			u16 sltcap;
+			sltcap = pci_read_config16(dev, pciexpos + PCI_EXP_SLTCAP);
+			if (sltcap & PCI_EXP_SLTCAP_HPC) {
+				printk(BIOS_DEBUG, "%s hot-plug capable\n", dev_path(dev));
+				return &default_pciexp_hotplug_ops_bus;
+			}
+#endif /* CONFIG(PCIEXP_HOTPLUG) */
 			return &default_pciexp_ops_bus;
 		case PCI_EXP_TYPE_PCI_BRIDGE:
 			printk(BIOS_DEBUG, "%s subordinate PCI\n",
@@ -1259,7 +1267,7 @@ static void pci_bridge_route(struct bus *link, scan_state state)
 
 	if (state == PCI_ROUTE_SCAN) {
 		link->secondary = parent->subordinate + 1;
-		link->subordinate = link->secondary;
+		link->subordinate = link->secondary + dev->hotplug_buses;
 	}
 
 	if (state == PCI_ROUTE_CLOSE) {

src/device/pciexp_device.c

@@ -518,3 +518,62 @@ struct device_operations default_pciexp_ops_bus = {
 	.reset_bus        = pci_bus_reset,
 	.ops_pci          = &pciexp_bus_ops_pci,
 };
+
+#if CONFIG(PCIEXP_HOTPLUG)
+
+static void pciexp_hotplug_dummy_read_resources(struct device *dev)
+{
+	struct resource *resource;
+
+	// Add extra memory space
+	resource = new_resource(dev, 0x10);
+	resource->size = CONFIG_PCIEXP_HOTPLUG_MEM;
+	resource->align = 12;
+	resource->gran = 12;
+	resource->limit = 0xffffffff;
+	resource->flags |= IORESOURCE_MEM;
+
+	// Add extra prefetchable memory space
+	resource = new_resource(dev, 0x14);
+	resource->size = CONFIG_PCIEXP_HOTPLUG_PREFETCH_MEM;
+	resource->align = 12;
+	resource->gran = 12;
+	resource->limit = 0xffffffffffffffff;
+	resource->flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
+
+	// Add extra I/O space
+	resource = new_resource(dev, 0x18);
+	resource->size = CONFIG_PCIEXP_HOTPLUG_IO;
+	resource->align = 12;
+	resource->gran = 12;
+	resource->limit = 0xffff;
+	resource->flags |= IORESOURCE_IO;
+}
+
+static struct device_operations pciexp_hotplug_dummy_ops = {
+	.read_resources   = pciexp_hotplug_dummy_read_resources,
+};
+
+void pciexp_hotplug_scan_bridge(struct device *dev)
+{
+	dev->hotplug_buses = CONFIG_PCIEXP_HOTPLUG_BUSES;
+
+	/* Normal PCIe Scan */
+	pciexp_scan_bridge(dev);
+
+	/* Add dummy slot to preserve resources, must happen after bus scan */
+	struct device *dummy;
+	struct device_path dummy_path = { .type = DEVICE_PATH_NONE };
+	dummy = alloc_dev(dev->link_list, &dummy_path);
+	dummy->ops = &pciexp_hotplug_dummy_ops;
+}
+
+struct device_operations default_pciexp_hotplug_ops_bus = {
+	.read_resources   = pci_bus_read_resources,
+	.set_resources    = pci_dev_set_resources,
+	.enable_resources = pci_bus_enable_resources,
+	.scan_bus         = pciexp_hotplug_scan_bridge,
+	.reset_bus        = pci_bus_reset,
+	.ops_pci          = &pciexp_bus_ops_pci,
+};
+#endif /* CONFIG(PCIEXP_HOTPLUG) */

src/include/device/device.h

@@ -121,6 +121,7 @@ struct device {
 	unsigned int    disable_pcie_aspm : 1;
 	unsigned int    hidden : 1;	/* set if we should hide from UI */
 	u8 command;
+	uint16_t hotplug_buses; /* Number of hotplug buses to allocate */
 
 	/* Base registers for this device. I/O, MEM and Expansion ROM */
 	DEVTREE_CONST struct resource *resource_list;

src/include/device/pci_def.h

@@ -435,6 +435,7 @@
 #define  PCI_EXP_LNKSTA_LT	0x800	/* Link Training */
 #define  PCI_EXP_LNKSTA_SLC	0x1000	/* Slot Clock Configuration */
 #define PCI_EXP_SLTCAP		20	/* Slot Capabilities */
+#define  PCI_EXP_SLTCAP_HPC	0x0040	/* Hot-Plug Capable */
 #define PCI_EXP_SLTCTL		24	/* Slot Control */
 #define PCI_EXP_SLTSTA		26	/* Slot Status */
 #define PCI_EXP_RTCTL		28	/* Root Control */

src/include/device/pciexp.h

@@ -26,5 +26,11 @@ void pciexp_scan_bridge(struct device *dev);
 
 extern struct device_operations default_pciexp_ops_bus;
 
+#if CONFIG(PCIEXP_HOTPLUG)
+void pciexp_hotplug_scan_bridge(struct device *dev);
+
+extern struct device_operations default_pciexp_hotplug_ops_bus;
+#endif /* CONFIG(PCIEXP_HOTPLUG) */
+
 unsigned int pciexp_find_extended_cap(struct device *dev, unsigned int cap);
 #endif /* DEVICE_PCIEXP_H */