cpu/amd/fam10h-fam15h: Update Fam15h APIC config and startup sequence

This fixes Family 15h multiple package support; the previous code
hung in CAR setup and romstage when more than one CPU package was
installed for a variety of loosely related reasons.

TEST: Booted ASUS KGPE-D16 with two Opteron 6328 processors
and several different RDIMM configurations.

Change-Id: I171197c90f72d3496a385465937b7666cbf7e308
Signed-off-by: Timothy Pearson <tpearson@raptorengineeringinc.com>
Reviewed-on: http://review.coreboot.org/12020
Tested-by: build bot (Jenkins)
Reviewed-by: Alexandru Gagniuc <mr.nuke.me@gmail.com>

22 files changed, 59 insertions, 29 deletions

diff --git a/src/mainboard/advansus/a785e-i/romstage.c b/src/mainboard/advansus/a785e-i/romstage.c
index b17fb2bcf1..55c94a63ad 100644
--- a/src/mainboard/advansus/a785e-i/romstage.c
+++ b/src/mainboard/advansus/a785e-i/romstage.c
@@ -151,7 +151,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/amd/bimini_fam10/romstage.c b/src/mainboard/amd/bimini_fam10/romstage.c
index 11d54797ea..be381a12ba 100644
--- a/src/mainboard/amd/bimini_fam10/romstage.c
+++ b/src/mainboard/amd/bimini_fam10/romstage.c
@@ -143,7 +143,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/amd/mahogany_fam10/romstage.c b/src/mainboard/amd/mahogany_fam10/romstage.c
index 5b39bb7304..3428aab9ab 100644
--- a/src/mainboard/amd/mahogany_fam10/romstage.c
+++ b/src/mainboard/amd/mahogany_fam10/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c b/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c
index 5131f36d9a..0c84b6d1e0 100644
--- a/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c
+++ b/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c
@@ -251,7 +251,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/amd/tilapia_fam10/romstage.c b/src/mainboard/amd/tilapia_fam10/romstage.c
index 0804b32eca..68281e7455 100644
--- a/src/mainboard/amd/tilapia_fam10/romstage.c
+++ b/src/mainboard/amd/tilapia_fam10/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/asus/kfsn4-dre/romstage.c b/src/mainboard/asus/kfsn4-dre/romstage.c
index 8686be725b..e7ad0b1b65 100644
--- a/src/mainboard/asus/kfsn4-dre/romstage.c
+++ b/src/mainboard/asus/kfsn4-dre/romstage.c
@@ -284,7 +284,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 	if (IS_ENABLED(CONFIG_LOGICAL_CPUS)) {
 		/* Core0 on each node is configured. Now setup any additional cores. */
 		printk(BIOS_DEBUG, "start_other_cores()\n");
-		start_other_cores();
+		start_other_cores(bsp_apicid);
 		post_code(0x37);
 		wait_all_other_cores_started(bsp_apicid);
 	}
diff --git a/src/mainboard/asus/kgpe-d16/romstage.c b/src/mainboard/asus/kgpe-d16/romstage.c
index 3a682302bc..be91eaa29a 100644
--- a/src/mainboard/asus/kgpe-d16/romstage.c
+++ b/src/mainboard/asus/kgpe-d16/romstage.c
@@ -93,7 +93,18 @@ static void switch_spd_mux(uint8_t channel)
 	pci_write_config8(PCI_DEV(0, 0x14, 0), 0x54, byte);
 }
 
-static const uint8_t spd_addr[] = {
+static const uint8_t spd_addr_fam15[] = {
+	// Socket 0 Node 0 ("Node 0")
+	RC00, DIMM0, DIMM1, 0, 0, DIMM2, DIMM3, 0, 0,
+	// Socket 0 Node 1 ("Node 1")
+	RC00, DIMM4, DIMM5, 0, 0, DIMM6, DIMM7, 0, 0,
+	// Socket 1 Node 0 ("Node 2")
+	RC01, DIMM0, DIMM1, 0, 0, DIMM2, DIMM3, 0, 0,
+	// Socket 1 Node 1 ("Node 3")
+	RC01, DIMM4, DIMM5, 0, 0, DIMM6, DIMM7, 0, 0,
+};
+
+static const uint8_t spd_addr_fam10[] = {
 	// Socket 0 Node 0 ("Node 0")
 	RC00, DIMM0, DIMM1, 0, 0, DIMM2, DIMM3, 0, 0,
 	// Socket 0 Node 1 ("Node 1")
@@ -113,10 +124,10 @@ static void activate_spd_rom(const struct mem_controller *ctrl) {
 		switch_spd_mux(0x2);
 	} else if (ctrl->node_id == 1) {
 		printk(BIOS_DEBUG, "enable_spd_node1()\n");
-		switch_spd_mux((sysinfo->nodes <= 2)?0x2:0x3);
+		switch_spd_mux((is_fam15h() || (sysinfo->nodes <= 2))?0x2:0x3);
 	} else if (ctrl->node_id == 2) {
 		printk(BIOS_DEBUG, "enable_spd_node2()\n");
-		switch_spd_mux((sysinfo->nodes <= 2)?0x3:0x2);
+		switch_spd_mux((is_fam15h() || (sysinfo->nodes <= 2))?0x3:0x2);
 	} else if (ctrl->node_id == 3) {
 		printk(BIOS_DEBUG, "enable_spd_node3()\n");
 		switch_spd_mux(0x3);
@@ -303,18 +314,25 @@ static void execute_memory_test(void)
 
 void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 {
+	uint32_t esp;
+	__asm__ volatile (
+		"movl %%esp, %0"
+		: "=r" (esp)
+		);
+
 	struct sys_info *sysinfo = &sysinfo_car;
 
 	uint32_t bsp_apicid = 0, val;
 	uint8_t byte;
 	msr_t msr;
 
-	timestamp_init(timestamp_get());
-	timestamp_add_now(TS_START_ROMSTAGE);
-
 	int s3resume = acpi_is_wakeup_s3();
 
 	if (!cpu_init_detectedx && boot_cpu()) {
+		/* Initial timestamp */
+		timestamp_init(timestamp_get());
+		timestamp_add_now(TS_START_ROMSTAGE);
+
 		/* Nothing special needs to be done to find bus 0 */
 		/* Allow the HT devices to be found */
 		set_bsp_node_CHtExtNodeCfgEn();
@@ -338,6 +356,8 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 		pci_write_config8(PCI_DEV(0, 0x14, 3), 0x78, byte);
 	}
 
+	printk(BIOS_SPEW, "Initial stack pointer: %08x\n", esp);
+
 	post_code(0x30);
 
 	if (bist == 0)
@@ -396,7 +416,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 	if (IS_ENABLED(CONFIG_LOGICAL_CPUS)) {
 		/* Core0 on each node is configured. Now setup any additional cores. */
 		printk(BIOS_DEBUG, "start_other_cores()\n");
-		start_other_cores();
+		start_other_cores(bsp_apicid);
 		post_code(0x37);
 		wait_all_other_cores_started(bsp_apicid);
 	}
@@ -454,7 +474,10 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 
 	/* It's the time to set ctrl in sysinfo now; */
 	printk(BIOS_DEBUG, "fill_mem_ctrl() detected %d nodes\n", sysinfo->nodes);
-	fill_mem_ctrl(sysinfo->nodes, sysinfo->ctrl, spd_addr);
+	if (is_fam15h())
+		fill_mem_ctrl(sysinfo->nodes, sysinfo->ctrl, spd_addr_fam15);
+	else
+		fill_mem_ctrl(sysinfo->nodes, sysinfo->ctrl, spd_addr_fam10);
 	post_code(0x3D);
 
 #if 0
@@ -526,5 +549,12 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  */
 BOOL AMD_CB_ManualBUIDSwapList (u8 node, u8 link, const u8 **List)
 {
+	/* Force BUID to 0 */
+	static const u8 swaplist[] = {0, 0, 0xFF, 0, 0xFF};
+	if ((node == 0) && (link == 1)) {	/* BSP SB link */
+		*List = swaplist;
+		return 1;
+	}
+
 	return 0;
 }
diff --git a/src/mainboard/asus/m4a78-em/romstage.c b/src/mainboard/asus/m4a78-em/romstage.c
index f717ed8352..844fc93c79 100644
--- a/src/mainboard/asus/m4a78-em/romstage.c
+++ b/src/mainboard/asus/m4a78-em/romstage.c
@@ -147,7 +147,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/asus/m4a785-m/romstage.c b/src/mainboard/asus/m4a785-m/romstage.c
index 47d884cfdb..e43dbbc8ad 100644
--- a/src/mainboard/asus/m4a785-m/romstage.c
+++ b/src/mainboard/asus/m4a785-m/romstage.c
@@ -147,7 +147,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/asus/m5a88-v/romstage.c b/src/mainboard/asus/m5a88-v/romstage.c
index b69fd80518..1630497e26 100644
--- a/src/mainboard/asus/m5a88-v/romstage.c
+++ b/src/mainboard/asus/m5a88-v/romstage.c
@@ -148,7 +148,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/avalue/eax-785e/romstage.c b/src/mainboard/avalue/eax-785e/romstage.c
index ed85c8258d..ff1aa9e36a 100644
--- a/src/mainboard/avalue/eax-785e/romstage.c
+++ b/src/mainboard/avalue/eax-785e/romstage.c
@@ -152,7 +152,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/gigabyte/ma785gm/romstage.c b/src/mainboard/gigabyte/ma785gm/romstage.c
index c5a04330d2..baf49afda8 100644
--- a/src/mainboard/gigabyte/ma785gm/romstage.c
+++ b/src/mainboard/gigabyte/ma785gm/romstage.c
@@ -142,7 +142,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/gigabyte/ma785gmt/romstage.c b/src/mainboard/gigabyte/ma785gmt/romstage.c
index 743463da5f..b8ab282bb8 100644
--- a/src/mainboard/gigabyte/ma785gmt/romstage.c
+++ b/src/mainboard/gigabyte/ma785gmt/romstage.c
@@ -142,7 +142,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/gigabyte/ma78gm/romstage.c b/src/mainboard/gigabyte/ma78gm/romstage.c
index aee7fbec35..43bae3ca0d 100644
--- a/src/mainboard/gigabyte/ma78gm/romstage.c
+++ b/src/mainboard/gigabyte/ma78gm/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/hp/dl165_g6_fam10/romstage.c b/src/mainboard/hp/dl165_g6_fam10/romstage.c
index bf1b73456b..84e28f7fc9 100644
--- a/src/mainboard/hp/dl165_g6_fam10/romstage.c
+++ b/src/mainboard/hp/dl165_g6_fam10/romstage.c
@@ -156,7 +156,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/iei/kino-780am2-fam10/romstage.c b/src/mainboard/iei/kino-780am2-fam10/romstage.c
index bb0ba572b6..7b001761b0 100644
--- a/src/mainboard/iei/kino-780am2-fam10/romstage.c
+++ b/src/mainboard/iei/kino-780am2-fam10/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/jetway/pa78vm5/romstage.c b/src/mainboard/jetway/pa78vm5/romstage.c
index 9144f3282e..8c87563c9e 100644
--- a/src/mainboard/jetway/pa78vm5/romstage.c
+++ b/src/mainboard/jetway/pa78vm5/romstage.c
@@ -150,7 +150,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/msi/ms9652_fam10/romstage.c b/src/mainboard/msi/ms9652_fam10/romstage.c
index cb4d02feba..f4413edead 100644
--- a/src/mainboard/msi/ms9652_fam10/romstage.c
+++ b/src/mainboard/msi/ms9652_fam10/romstage.c
@@ -173,7 +173,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	printk(BIOS_DEBUG, "wait_all_other_cores_started()\n");
 	wait_all_other_cores_started(bsp_apicid);
diff --git a/src/mainboard/supermicro/h8dmr_fam10/romstage.c b/src/mainboard/supermicro/h8dmr_fam10/romstage.c
index c29470f766..bf5093f417 100644
--- a/src/mainboard/supermicro/h8dmr_fam10/romstage.c
+++ b/src/mainboard/supermicro/h8dmr_fam10/romstage.c
@@ -167,7 +167,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/supermicro/h8qme_fam10/romstage.c b/src/mainboard/supermicro/h8qme_fam10/romstage.c
index eacc87f925..13a101185c 100644
--- a/src/mainboard/supermicro/h8qme_fam10/romstage.c
+++ b/src/mainboard/supermicro/h8qme_fam10/romstage.c
@@ -234,7 +234,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/supermicro/h8scm_fam10/romstage.c b/src/mainboard/supermicro/h8scm_fam10/romstage.c
index 6e34f0ac88..7312683fc0 100644
--- a/src/mainboard/supermicro/h8scm_fam10/romstage.c
+++ b/src/mainboard/supermicro/h8scm_fam10/romstage.c
@@ -158,7 +158,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/tyan/s2912_fam10/romstage.c b/src/mainboard/tyan/s2912_fam10/romstage.c
index 9f25d271c4..ed211d9c86 100644
--- a/src/mainboard/tyan/s2912_fam10/romstage.c
+++ b/src/mainboard/tyan/s2912_fam10/romstage.c
@@ -169,7 +169,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif