soc/intel: Implement PCIe RP devicetree update based on LCAP

Most of the current implementations for FSP-based platforms
make (sometimes wrong) assumptions how FSP reorders root ports
and what is specified in the devicetree. We don't have to make
assumptions though, and can read the root-port number from the
PCIe link capapilities (LCAP) instead. This is also what we do
in ASL code for years already.

This new implementation acts solely on information read from
the PCI config space. In a first round, we scan all possible
DEVFNs and store which root port has that DEVFN now. Then, we
walk through the devicetree that still only knows devices that
were originally mentioned in `devicetree.cb`, update device
paths and unlink vanished devices.

To be most compatible, we work with the following constraints:
  o Use only standard PCI config registers.
  o Most notable, don't try to read the registers that
    configure the function numbers. FSP has undocumented
    ways to block access to non-standard registers.
  o Don't make assumptions what function is assigned to
    hidden devices.

The following assumptions were made, though:
  o The absolute root-port numbering as documented in
    datasheets matches what is read from LCAP.
  o This numbering doesn't contain any gaps.
  o Original root-port function numbers below a PCI
    device start at function zero and also don't
    contain any gaps.

Change-Id: Ib17d2b6fd34608603db3936d638bdf5acb46d717
Signed-off-by: Nico Huber <nico.h@gmx.de>
Reviewed-on: https://review.coreboot.org/c/coreboot/+/35985
Reviewed-by: Aaron Durbin <adurbin@chromium.org>
Reviewed-by: Michael Niewöhner
Reviewed-by: Patrick Rudolph <patrick.rudolph@9elements.com>
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>

src/include/device/pci_def.h

@@ -426,6 +426,7 @@
 #define  PCI_EXP_LNKCAP_L0SEL	0x7000	/* L0s Exit Latency */
 #define  PCI_EXP_LNKCAP_L1EL	0x38000	/* L1 Exit Latency */
 #define  PCI_EXP_CLK_PM		0x40000	/* Clock Power Management */
+#define  PCI_EXP_LNKCAP_PORT	0xff000000 /* Port Number */
 #define PCI_EXP_LNKCTL		16	/* Link Control */
 #define  PCI_EXP_LNKCTL_RL	0x20	/* Retrain Link */
 #define  PCI_EXP_LNKCTL_CCC	0x40	/* Common Clock COnfiguration */

src/soc/intel/common/block/include/intelblocks/pcie_rp.h

@@ -0,0 +1,50 @@
+/*
+ * This file is part of the coreboot project.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef SOC_INTEL_COMMON_BLOCK_PCIE_RP_H
+#define SOC_INTEL_COMMON_BLOCK_PCIE_RP_H
+
+#include <stdint.h>
+
+/*
+ * The PCIe Root Ports usually come in groups of up to 8 PCI-device
+ * functions.
+ *
+ * `slot` is the PCI device/slot number of such a group.
+ * `count` is the number of functions within the group. It is assumed that
+ * the first group includes the RPs 1 to the first group's `count` and that
+ * adjacent groups follow without gaps in the numbering.
+ */
+struct pcie_rp_group {
+	unsigned int slot;
+	unsigned int count;
+};
+
+/*
+ * Update PCI paths of the root ports in the devicetree.
+ *
+ * Depending on the board layout and physical presence of downstream
+ * devices, individual root-port functions can be hidden and reordered.
+ * If we have device nodes for root ports in the static `devicetree.cb`,
+ * we need to update their PCI paths, so the nodes still control the
+ * correct root port. Device nodes for disabled root ports will be
+ * unlinked from the bus, to not interfere with PCI enumeration.
+ *
+ * Call this once, after root ports have been reordered, but before PCI
+ * enumeration.
+ *
+ * `groups` points to a list of groups terminated by an entry with `count == 0`.
+ */
+void pcie_rp_update_devicetree(const struct pcie_rp_group *groups);
+
+#endif /* SOC_INTEL_COMMON_BLOCK_PCIE_RP_H */

src/soc/intel/common/block/pcie/Makefile.inc

@@ -1 +1,2 @@
 ramstage-$(CONFIG_SOC_INTEL_COMMON_BLOCK_PCIE) += pcie.c
+ramstage-$(CONFIG_SOC_INTEL_COMMON_BLOCK_PCIE) += pcie_rp.c

src/soc/intel/common/block/pcie/pcie_rp.c

@@ -0,0 +1,177 @@
+/*
+ * This file is part of the coreboot project.
+ *
+ * Copyright (C) 2019 Nico Huber <nico.h@gmx.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <stdint.h>
+
+#include <commonlib/helpers.h>
+#include <console/console.h>
+#include <device/device.h>
+#include <device/pci_def.h>
+#include <device/pci_ops.h>
+#include <device/pci_type.h>
+#include <intelblocks/pcie_rp.h>
+
+static int pcie_rp_original_idx(
+		const struct pcie_rp_group *const group,
+		const unsigned int offset,
+		const pci_devfn_t dev)
+{
+	const uint16_t clist = pci_s_find_capability(dev, PCI_CAP_ID_PCIE);
+	if (clist == 0) {
+		printk(BIOS_WARNING,
+		       "%s: Can't find PCIe capapilities for PCI: 00:%02x.%x, ignoring.\n",
+		       __func__, group->slot, PCI_FUNC(PCI_DEV2DEVFN(dev)));
+		return -1;
+	}
+
+	const uint16_t xcap = pci_s_read_config16(dev, clist + PCI_EXP_FLAGS);
+	if ((xcap & PCI_EXP_FLAGS_TYPE) >> 4 != PCI_EXP_TYPE_ROOT_PORT) {
+		printk(BIOS_WARNING, "%s: Non root-port found at PCI: 00:%02x.%x, ignoring.\n",
+		       __func__, group->slot, PCI_FUNC(PCI_DEV2DEVFN(dev)));
+		return -1;
+	}
+
+	const uint32_t lcap = pci_s_read_config32(dev, clist + PCI_EXP_LNKCAP);
+	/* Read 1-based absolute port number. This reflects the numbering
+	   scheme that Intel uses in their documentation and what we use
+	   as index (0-based, though) in our mapping. */
+	const unsigned int port_num = (lcap & PCI_EXP_LNKCAP_PORT) >> 24;
+
+	/* `port_num` is 1-based, `offset` is 0-based. */
+	if (port_num <= offset || port_num > offset + group->count) {
+		printk(BIOS_WARNING, "%s: Unexpected root-port number '%u'"
+				     " at PCI: 00:%02x.%x, ignoring.\n",
+		       __func__, port_num, group->slot, PCI_FUNC(PCI_DEV2DEVFN(dev)));
+		return -1;
+	}
+
+	return port_num - 1;
+}
+
+/* Scan actual PCI config space to reconstruct current mapping */
+static void pcie_rp_scan_groups(int mapping[], const struct pcie_rp_group *const groups)
+{
+	unsigned int offset = 0;
+	const struct pcie_rp_group *group;
+	for (group = groups; group->count; ++group) {
+		unsigned int fn;
+		for (fn = 0; fn < group->count; ++fn) {
+			const pci_devfn_t dev = PCI_DEV(0, group->slot, fn);
+			const uint16_t did = pci_s_read_config16(dev, PCI_DEVICE_ID);
+			if (did == 0xffff) {
+				if (fn == 0)
+					break;
+				continue;
+			}
+
+			const int rp_idx = pcie_rp_original_idx(group, offset, dev);
+			if (rp_idx < 0)
+				continue;
+			if (mapping[rp_idx] != -1) {
+				printk(BIOS_WARNING, "%s: Root Port #%u reported by PCI: "
+				       "00:%02x.%x already reported by PCI: 00:%02x.%x!\n",
+				       __func__, rp_idx + 1, group->slot, fn,
+				       group->slot, mapping[rp_idx]);
+				continue;
+			}
+
+			printk(BIOS_INFO, "Found PCIe Root Port #%u at PCI: 00:%02x.%x.\n",
+			       rp_idx + 1, group->slot, fn);
+			mapping[rp_idx] = fn;
+		}
+		offset += group->count;
+	}
+}
+
+/* Returns `true` if the device should be unlinked. */
+static bool pcie_rp_update_dev(
+		struct device *const dev,
+		const struct pcie_rp_group *const groups,
+		const int mapping[])
+{
+	if (dev->path.type != DEVICE_PATH_PCI)
+		return false;
+
+	/* Find matching group and offset. */
+	unsigned int offset = 0;
+	const struct pcie_rp_group *group;
+	for (group = groups; group->count; ++group) {
+		if (PCI_SLOT(dev->path.pci.devfn) == group->slot &&
+		    PCI_FUNC(dev->path.pci.devfn) < group->count)
+			break;
+		offset += group->count;
+	}
+	if (!group->count)
+		return false;
+
+	/* Now update based on what we know. */
+	const int rp_idx = offset + PCI_FUNC(dev->path.pci.devfn);
+	const int new_fn = mapping[rp_idx];
+	if (new_fn < 0) {
+		if (dev->enabled) {
+			printk(BIOS_NOTICE, "%s: Couldn't find PCIe Root Port #%u "
+			       "(originally %s) which was enabled in devicetree, removing.\n",
+			       __func__, rp_idx + 1, dev_path(dev));
+		}
+		return true;
+	} else if (PCI_FUNC(dev->path.pci.devfn) != new_fn) {
+		printk(BIOS_INFO,
+		       "Remapping PCIe Root Port #%u from %s to new function number %u.\n",
+		       rp_idx + 1, dev_path(dev), new_fn);
+		dev->path.pci.devfn = PCI_DEVFN(PCI_SLOT(dev->path.pci.devfn), new_fn);
+	}
+	return false;
+}
+
+void pcie_rp_update_devicetree(const struct pcie_rp_group *const groups)
+{
+	/* Maps absolute root-port numbers to function numbers.
+	   Negative if disabled, new function number otherwise. */
+	int mapping[CONFIG_MAX_ROOT_PORTS];
+	unsigned int offset, i;
+
+	struct bus *const root = pci_root_bus();
+	if (!root)
+		return;
+
+	offset = 0;
+	const struct pcie_rp_group *group;
+	for (group = groups; group->count; ++group)
+		offset += group->count;
+
+	if (offset > ARRAY_SIZE(mapping)) {
+		printk(BIOS_ERR, "%s: Error: Group exceeds CONFIG_MAX_ROOT_PORTS.\n", __func__);
+		return;
+	}
+
+	/* Assume everything we don't encounter later is disabled */
+	for (i = 0; i < ARRAY_SIZE(mapping); ++i)
+		mapping[i] = -1;
+
+	pcie_rp_scan_groups(mapping, groups);
+
+	struct device *dev;
+	struct device **link = &root->children;
+	for (dev = *link; dev; dev = *link) {
+		if (pcie_rp_update_dev(dev, groups, mapping)) {
+			/* Unlink vanished device. */
+			*link = dev->sibling;
+			dev->sibling = NULL;
+			continue;
+		}
+
+		link = &dev->sibling;
+	}
+}