cpu/amd/fam10h-fam15h: Update Fam15h APIC config and startup sequence

This fixes Family 15h multiple package support; the previous code
hung in CAR setup and romstage when more than one CPU package was
installed for a variety of loosely related reasons.

TEST: Booted ASUS KGPE-D16 with two Opteron 6328 processors
and several different RDIMM configurations.

Change-Id: I171197c90f72d3496a385465937b7666cbf7e308
Signed-off-by: Timothy Pearson <tpearson@raptorengineeringinc.com>
Reviewed-on: http://review.coreboot.org/12020
Tested-by: build bot (Jenkins)
Reviewed-by: Alexandru Gagniuc <mr.nuke.me@gmail.com>

32 files changed, 457 insertions, 123 deletions

diff --git a/src/cpu/amd/car/cache_as_ram.inc b/src/cpu/amd/car/cache_as_ram.inc
index cbb1e39854..53056038b7 100644
--- a/src/cpu/amd/car/cache_as_ram.inc
+++ b/src/cpu/amd/car/cache_as_ram.inc
@@ -548,8 +548,23 @@ CAR_FAM10_ap:
 	/* Fam10h NB config bit 54 was not set */
 	rolb	%cl, %bl
 roll_cfg:
+	jmp_if_not_fam15h(ap_apicid_ready)
+	cmp	$0x5, %ecx
+	jne	ap_apicid_ready
 
-	/* Calculate stack pointer. */
+	/* This is a multi-node CPU
+	 * Adjust the maximum APIC ID to a more reasonable value
+	 * given that no 32-core Family 15h processors exist
+	 */
+	movl	%ebx, %ecx
+	and	$0x0f, %ecx		/* Get lower 4 bits of CPU number */
+	and	$0x60, %ebx		/* Get node ID */
+	shrl	$0x1, %ebx		/* Shift node ID part of APIC ID down by 1 */
+	or	%ecx, %ebx		/* Recombine node ID and CPU number */
+
+ap_apicid_ready:
+
+	/* Calculate stack pointer using adjusted APIC ID stored in ebx */
 	movl	$CacheSizeAPStack, %eax
 	mull	%ebx
 	movl	$(CacheBase + (CacheSize - (CacheSizeBSPStack + CacheSizeBSPSlush))), %esp
diff --git a/src/cpu/amd/family_10h-family_15h/defaults.h b/src/cpu/amd/family_10h-family_15h/defaults.h
index 94468c5ae2..b6c9bc4c93 100644
--- a/src/cpu/amd/family_10h-family_15h/defaults.h
+++ b/src/cpu/amd/family_10h-family_15h/defaults.h
@@ -240,18 +240,37 @@ static const struct {
 	{ 0, 0x68, (AMD_DR_B0 | AMD_DR_B1),
 	  AMD_PTYPE_SVR, 0x00200000, 0x00600000 },	/* [22:21] DsNpReqLmt0 = 01b */
 
-	{ 0, 0x84, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL,
+	{ 0, 0x84, AMD_FAM10_ALL, AMD_PTYPE_ALL,
 	  0x00002000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
 
-	{ 0, 0xA4, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL,
+	{ 0, 0xA4, AMD_FAM10_ALL, AMD_PTYPE_ALL,
 	  0x00002000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
 
-	{ 0, 0xC4, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL,
+	{ 0, 0xC4, AMD_FAM10_ALL, AMD_PTYPE_ALL,
 	  0x00002000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
 
-	{ 0, 0xE4, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL,
+	{ 0, 0xE4, AMD_FAM10_ALL, AMD_PTYPE_ALL,
 	  0x00002000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
 
+	/* FIXME
+	 * Non-C32 packages only
+	 */
+	{ 0, 0x84, AMD_FAM15_ALL, AMD_PTYPE_ALL,
+	  0x00000000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
+
+	{ 0, 0xA4, AMD_FAM15_ALL, AMD_PTYPE_ALL,
+	  0x00000000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
+
+	{ 0, 0xC4, AMD_FAM15_ALL, AMD_PTYPE_ALL,
+	  0x00000000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
+
+	{ 0, 0xE4, AMD_FAM15_ALL, AMD_PTYPE_ALL,
+	  0x00000000, 0x00002000 },	/* [13] LdtStopTriEn = 1 */
+
+	/* FIXME
+	 * C32 package is not supported at this time
+	 */
+
 	/* Link Global Retry Control Register */
 	{ 0, 0x150, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL,
 	  0x00073900, 0x00073F00 },
@@ -610,38 +629,79 @@ static const struct {
 	{ 0x530A, AMD_DR_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_ALL,
 	  0x00004400, 0x00006400 },	/* HT_PHY_DLL_REG */
 
-	{ 0xCF, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	{ 0xCF, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	  0x00000000, 0x000000FF },	/* Provide clear setting for logical
+					   completeness */
+
+	{ 0xDF, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	  0x00000000, 0x000000FF },	/* Provide clear setting for logical
+					   completeness */
+
+	{ 0xCF, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
+	  0x0000006D, 0x000000FF },	/* HT_PHY_HT1_FIFO_PTR_OPT_VALUE */
+
+	{ 0xDF, AMD_FAM10_ALL, AMD_PTYPE_ALL,  HTPHY_LINKTYPE_HT1,
+	  0x0000006D, 0x000000FF }, 	/* HT_PHY_HT1_FIFO_PTR_OPT_VALUE */
+
+	/* Link Phy Receiver Loop Filter Registers */
+	{ 0xD1, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	  0x08040000, 0x3FFFC000 },	/* [29:22] LfcMax = 20h,
+					   [21:14] LfcMin = 10h */
+
+	{ 0xC1, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	  0x08040000, 0x3FFFC000 },	/* [29:22] LfcMax = 20h,
+					   [21:14] LfcMin = 10h */
+
+	{ 0xD1, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
+	  0x04020000, 0x3FFFC000 },	/* [29:22] LfcMax = 10h,
+					   [21:14] LfcMin = 08h */
+
+	{ 0xC1, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
+	  0x04020000, 0x3FFFC000 },	/* [29:22] LfcMax = 10h,
+					   [21:14] LfcMin = 08h */
+
+	{ 0xC0, AMD_FAM10_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_ALL,
+	  0x40040000, 0xe01F0000 },	/* [31:29] RttCtl = 02h,
+					  [20:16] RttIndex = 04h */
+
+/* FIXME
+ * Causes lockups for some reason when more than one package is installed
+ * Debug and reactivate!
+ */
+// #if 0
+	{ 0xCF, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
 	  0x00000000, 0x000000FF },	/* Provide clear setting for logical
 					   completeness */
 
-	{ 0xDF, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	{ 0xDF, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
 	  0x00000000, 0x000000FF },	/* Provide clear setting for logical
 					   completeness */
 
-	{ 0xCF, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
+	{ 0xCF, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
 	  0x0000006D, 0x000000FF },	/* HT_PHY_HT1_FIFO_PTR_OPT_VALUE */
 
-	{ 0xDF, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL,  HTPHY_LINKTYPE_HT1,
+	{ 0xDF, AMD_FAM15_ALL, AMD_PTYPE_ALL,  HTPHY_LINKTYPE_HT1,
 	  0x0000006D, 0x000000FF }, 	/* HT_PHY_HT1_FIFO_PTR_OPT_VALUE */
 
 	/* Link Phy Receiver Loop Filter Registers */
-	{ 0xD1, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	{ 0xD1, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
 	  0x08040000, 0x3FFFC000 },	/* [29:22] LfcMax = 20h,
 					   [21:14] LfcMin = 10h */
 
-	{ 0xC1, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
+	{ 0xC1, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT3,
 	  0x08040000, 0x3FFFC000 },	/* [29:22] LfcMax = 20h,
 					   [21:14] LfcMin = 10h */
 
-	{ 0xD1, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
+	{ 0xD1, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
 	  0x04020000, 0x3FFFC000 },	/* [29:22] LfcMax = 10h,
 					   [21:14] LfcMin = 08h */
 
-	{ 0xC1, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
+	{ 0xC1, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_HT1,
 	  0x04020000, 0x3FFFC000 },	/* [29:22] LfcMax = 10h,
 					   [21:14] LfcMin = 08h */
 
-	{ 0xC0, (AMD_FAM10_ALL | AMD_FAM15_ALL), AMD_PTYPE_ALL, HTPHY_LINKTYPE_ALL,
+	{ 0xC0, AMD_FAM15_ALL, AMD_PTYPE_ALL, HTPHY_LINKTYPE_ALL,
 	  0x40040000, 0xe01F0000 },	/* [31:29] RttCtl = 02h,
-								   [20:16] RttIndex = 04h */
+					  [20:16] RttIndex = 04h */
+// #endif
 };
diff --git a/src/cpu/amd/family_10h-family_15h/fidvid.c b/src/cpu/amd/family_10h-family_15h/fidvid.c
index 3af18bd06b..8a006cbbb3 100644
--- a/src/cpu/amd/family_10h-family_15h/fidvid.c
+++ b/src/cpu/amd/family_10h-family_15h/fidvid.c
@@ -629,44 +629,45 @@ static void prep_fid_change(void)
 }
 
 static void waitCurrentPstate(u32 target_pstate) {
-  msr_t initial_msr = rdmsr(TSC_MSR);
-  msr_t pstate_msr = rdmsr(CUR_PSTATE_MSR);
-  msr_t tsc_msr;
-  u8 timedout ;
-
-  /* paranoia ? I fear when we run fixPsNbVidBeforeWR we can enter a
-   * P1 that is a copy of P0, therefore has the same NB DID but the
-   * TSC will count twice per tick, so we have to wait for twice the
-   * count to achieve the desired timeout. But I'm likely to
-   * misunderstand this...
-   */
-  u32 corrected_timeout = (    (pstate_msr.lo==1)
-			    && (!(rdmsr(0xC0010065).lo & NB_DID_M_ON)) ) ?
-                          WAIT_PSTATE_TIMEOUT*2 : WAIT_PSTATE_TIMEOUT  ;
-  msr_t timeout;
-
-  timeout.lo = initial_msr.lo + corrected_timeout ;
-  timeout.hi = initial_msr.hi;
-  if ( (((u32)0xffffffff) - initial_msr.lo) < corrected_timeout ) {
-     timeout.hi++;
-  }
-
-  // assuming TSC ticks at 1.25 ns per tick (800 MHz)
-  do {
-      pstate_msr = rdmsr(CUR_PSTATE_MSR);
-      tsc_msr = rdmsr(TSC_MSR);
-      timedout = (tsc_msr.hi > timeout.hi)
-        	|| ((tsc_msr.hi == timeout.hi) && (tsc_msr.lo > timeout.lo ));
-  } while ( (pstate_msr.lo != target_pstate) && (! timedout) ) ;
-
-  if (pstate_msr.lo != target_pstate) {
-    msr_t limit_msr = rdmsr(0xc0010061);
-    printk(BIOS_ERR, "*** Time out waiting for P-state %01x. Current P-state %01x P-state current limit MSRC001_0061=%08x %08x\n", target_pstate, pstate_msr.lo, limit_msr.hi, limit_msr.lo);
-
-    do { // should we just go on instead ?
-      pstate_msr = rdmsr(CUR_PSTATE_MSR);
-    } while ( pstate_msr.lo != target_pstate  ) ;
-  }
+	msr_t initial_msr = rdmsr(TSC_MSR);
+	msr_t pstate_msr = rdmsr(CUR_PSTATE_MSR);
+	msr_t tsc_msr;
+	u8 timedout ;
+
+	/* paranoia ? I fear when we run fixPsNbVidBeforeWR we can enter a
+	* P1 that is a copy of P0, therefore has the same NB DID but the
+	* TSC will count twice per tick, so we have to wait for twice the
+	* count to achieve the desired timeout. But I'm likely to
+	* misunderstand this...
+	*/
+	u32 corrected_timeout = ((pstate_msr.lo==1)
+				&& (!(rdmsr(0xC0010065).lo & NB_DID_M_ON)) ) ?
+				WAIT_PSTATE_TIMEOUT*2 : WAIT_PSTATE_TIMEOUT;
+	msr_t timeout;
+
+	timeout.lo = initial_msr.lo + corrected_timeout ;
+	timeout.hi = initial_msr.hi;
+	if ( (((u32)0xffffffff) - initial_msr.lo) < corrected_timeout ) {
+		timeout.hi++;
+	}
+
+	// assuming TSC ticks at 1.25 ns per tick (800 MHz)
+	do {
+		pstate_msr = rdmsr(CUR_PSTATE_MSR);
+		tsc_msr = rdmsr(TSC_MSR);
+		timedout = (tsc_msr.hi > timeout.hi)
+			|| ((tsc_msr.hi == timeout.hi) && (tsc_msr.lo > timeout.lo ));
+	} while ( (pstate_msr.lo != target_pstate) && (! timedout) ) ;
+
+	if (pstate_msr.lo != target_pstate) {
+		msr_t limit_msr = rdmsr(0xc0010061);
+		printk(BIOS_ERR, "*** APIC ID %02x: timed out waiting for P-state %01x. Current P-state %01x P-state current limit MSRC001_0061=%08x %08x\n",
+			cpuid_ebx(0x00000001) >> 24, target_pstate, pstate_msr.lo, limit_msr.hi, limit_msr.lo);
+
+		do { // should we just go on instead ?
+			pstate_msr = rdmsr(CUR_PSTATE_MSR);
+		} while ( pstate_msr.lo != target_pstate  ) ;
+	}
 }
 
 static void set_pstate(u32 nonBoostedPState) {
@@ -1060,13 +1061,13 @@ static int init_fidvid_bsp(u32 bsp_apicid, u32 nodes)
 	   APs and BSP */
 	ap_apicidx.num = 0;
 
-	for_each_ap(bsp_apicid, CONFIG_SET_FIDVID_CORE_RANGE, store_ap_apicid, &ap_apicidx);
+	for_each_ap(bsp_apicid, CONFIG_SET_FIDVID_CORE_RANGE, -1, store_ap_apicid, &ap_apicidx);
 
 	for (i = 0; i < ap_apicidx.num; i++) {
 		init_fidvid_bsp_stage1(ap_apicidx.apicid[i], &fv);
 	}
 #else
-	for_each_ap(bsp_apicid, CONFIG_SET_FIDVID_CORE0_ONLY, init_fidvid_bsp_stage1, &fv);
+	for_each_ap(bsp_apicid, CONFIG_SET_FIDVID_CORE0_ONLY, -1, init_fidvid_bsp_stage1, &fv);
 #endif
 
 	print_debug_fv("common_fid = ", fv.common_fid);
diff --git a/src/cpu/amd/family_10h-family_15h/init_cpus.c b/src/cpu/amd/family_10h-family_15h/init_cpus.c
index e4721a43f3..a421bfcc03 100644
--- a/src/cpu/amd/family_10h-family_15h/init_cpus.c
+++ b/src/cpu/amd/family_10h-family_15h/init_cpus.c
@@ -55,6 +55,8 @@ static void set_EnableCf8ExtCfg(void)
 static void set_EnableCf8ExtCfg(void) { }
 #endif
 
+// #define DEBUG_HT_SETUP 1
+// #define FAM10_AP_NODE_SEQUENTIAL_START 1
 
 typedef void (*process_ap_t) (u32 apicid, void *gp);
 
@@ -139,8 +141,8 @@ uint32_t get_boot_apic_id(uint8_t node, uint32_t core) {
 //core range = 1 : core 0 only
 //core range = 2 : cores other than core0
 
-static void for_each_ap(u32 bsp_apicid, u32 core_range, process_ap_t process_ap,
-			void *gp)
+static void for_each_ap(uint32_t bsp_apicid, uint32_t core_range, int8_t node,
+			process_ap_t process_ap, void *gp)
 {
 	// here assume the OS don't change our apicid
 	u32 ap_apicid;
@@ -161,6 +163,9 @@ static void for_each_ap(u32 bsp_apicid, u32 core_range, process_ap_t process_ap,
 	}
 
 	for (i = 0; i < nodes; i++) {
+		if ((node >= 0) && (i != node))
+			continue;
+
 		cores_found = get_core_num_in_bsp(i);
 
 		u32 jstart, jend;
@@ -276,7 +281,7 @@ void wait_all_other_cores_started(u32 bsp_apicid)
 {
 	// all aps other than core0
 	printk(BIOS_DEBUG, "started ap apicid: ");
-	for_each_ap(bsp_apicid, 2, wait_ap_started, (void *)0);
+	for_each_ap(bsp_apicid, 2, -1, wait_ap_started, (void *)0);
 	printk(BIOS_DEBUG, "\n");
 }
 
@@ -369,8 +374,10 @@ static u32 init_cpus(u32 cpu_init_detectedx, struct sys_info *sysinfo)
 	/* NB_CFG MSR is shared between cores, so we need make sure
 	   core0 is done at first --- use wait_all_core0_started  */
 	if (id.coreid == 0) {
-		set_apicid_cpuid_lo();	/* only set it on core0 */
-		set_EnableCf8ExtCfg();	/* only set it on core0 */
+		/* Set InitApicIdCpuIdLo / EnableCf8ExtCfg on core0 only */
+		if (!is_fam15h())
+			set_apicid_cpuid_lo();
+		set_EnableCf8ExtCfg();
 #if CONFIG_ENABLE_APIC_EXT_ID
 		enable_apic_ext_id(id.nodeid);
 #endif
@@ -423,6 +430,7 @@ static u32 init_cpus(u32 cpu_init_detectedx, struct sys_info *sysinfo)
 	}
 	// Mark the core as started.
 	lapic_write(LAPIC_MSG_REG, (apicid << 24) | F10_APSTATE_STARTED);
+	printk(BIOS_DEBUG, "CPU APICID %02x start flag set\n", apicid);
 
 	if (apicid != bsp_apicid) {
 		/* Setup each AP's cores MSRs.
@@ -584,6 +592,34 @@ static void setup_remote_node(u8 node)
 }
 #endif				/* CONFIG_MAX_PHYSICAL_CPUS > 1 */
 
+//it is running on core0 of node0
+static void start_other_cores(uint32_t bsp_apicid)
+{
+	u32 nodes;
+	u32 nodeid;
+
+	// disable multi_core
+	if (read_option(multi_core, 0) != 0)  {
+		printk(BIOS_DEBUG, "Skip additional core init\n");
+		return;
+	}
+
+	nodes = get_nodes();
+
+	for (nodeid = 0; nodeid < nodes; nodeid++) {
+		u32 cores = get_core_num_in_bsp(nodeid);
+		printk(BIOS_DEBUG, "init node: %02x  cores: %02x pass 1\n", nodeid, cores);
+		if (cores > 0) {
+			real_start_other_core(nodeid, cores);
+#ifdef FAM10_AP_NODE_SEQUENTIAL_START
+			printk(BIOS_DEBUG, "waiting for core start on node %d...\n", nodeid);
+			for_each_ap(bsp_apicid, 2, nodeid, wait_ap_started, (void *)0);
+			printk(BIOS_DEBUG, "...started\n");
+#endif
+		}
+	}
+}
+
 static void AMD_Errata281(u8 node, uint64_t revision, u32 platform)
 {
 	/* Workaround for Transaction Scheduling Conflict in
@@ -843,6 +879,10 @@ static void AMD_SetHtPhyRegister(u8 node, u8 link, u8 entry)
 
 	phyBase = ((u32) link << 3) | 0x180;
 
+	/* Determine if link is connected and abort if not */
+	if (!(pci_read_config32(NODE_PCI(node, 0), 0x98 + (link * 0x20)) & 0x1))
+		return;
+
 	/* Get the portal control register's initial value
 	 * and update it to access the desired phy register
 	 */
@@ -1005,10 +1045,11 @@ static void cpuSetAMDPCI(u8 node)
 	 * Hypertransport initialization has taken place.  Also note
 	 * that it is run for the first core on each node
 	 */
-	u8 i, j;
+	uint8_t i;
+	uint8_t j;
 	u32 platform;
 	u32 val;
-	u8 offset;
+	uint8_t offset;
 	uint32_t dword;
 	uint64_t revision;
 
@@ -1035,6 +1076,17 @@ static void cpuSetAMDPCI(u8 node)
 		}
 	}
 
+#ifdef DEBUG_HT_SETUP
+	/* Dump link settings */
+	for (i = 0; i < 4; i++) {
+		for (j = 0; j < 4; j++) {
+			printk(BIOS_DEBUG, "Node %d link %d: type register: %08x control register: %08x extended control sublink 0: %08x 1: %08x\n", i, j,
+				pci_read_config32(NODE_PCI(i, 0), 0x98 + (j * 0x20)), pci_read_config32(NODE_PCI(i, 0), 0x84 + (j * 0x20)),
+				pci_read_config32(NODE_PCI(i, 0), 0x170 + (j * 0x4)), pci_read_config32(NODE_PCI(i, 0), 0x180 + (j * 0x4)));
+		}
+	}
+#endif
+
 	for (i = 0; i < ARRAY_SIZE(fam10_htphy_default); i++) {
 		if ((fam10_htphy_default[i].revision & revision) &&
 		    (fam10_htphy_default[i].platform & platform)) {
diff --git a/src/cpu/amd/quadcore/quadcore.c b/src/cpu/amd/quadcore/quadcore.c
index 0f7c728105..3ca7f3e0e7 100644
--- a/src/cpu/amd/quadcore/quadcore.c
+++ b/src/cpu/amd/quadcore/quadcore.c
@@ -27,21 +27,6 @@
 uint32_t get_boot_apic_id(uint8_t node, uint32_t core);
 uint32_t wait_cpu_state(uint32_t apicid, uint32_t state, uint32_t state2);
 
-static inline uint8_t is_fam15h(void)
-{
-	uint8_t fam15h = 0;
-	uint32_t family;
-
-	family = cpuid_eax(0x80000001);
-	family = ((family & 0xf00000) >> 16) | ((family & 0xf00) >> 8);
-
-	if (family >= 0x6f)
-		/* Family 15h or later */
-		fam15h = 1;
-
-	return fam15h;
-}
-
 static u32 get_core_num_in_bsp(u32 nodeid)
 {
 	u32 dword;
@@ -137,6 +122,7 @@ static void real_start_other_core(uint32_t nodeid, uint32_t cores)
 	}
 }
 
+#if (!IS_ENABLED(CONFIG_CPU_AMD_MODEL_10XXX))
 //it is running on core0 of node0
 static void start_other_cores(void)
 {
@@ -153,9 +139,10 @@ static void start_other_cores(void)
 
 	for (nodeid = 0; nodeid < nodes; nodeid++) {
 		u32 cores = get_core_num_in_bsp(nodeid);
-		printk(BIOS_DEBUG, "init node: %02x  cores: %02x pass 1 \n", nodeid, cores);
+		printk(BIOS_DEBUG, "init node: %02x  cores: %02x pass 1\n", nodeid, cores);
 		if (cores > 0) {
 			real_start_other_core(nodeid, cores);
 		}
 	}
 }
+#endif
diff --git a/src/cpu/amd/quadcore/quadcore_id.c b/src/cpu/amd/quadcore/quadcore_id.c
index acfdb4967a..99e6d68be4 100644
--- a/src/cpu/amd/quadcore/quadcore_id.c
+++ b/src/cpu/amd/quadcore/quadcore_id.c
@@ -104,7 +104,6 @@ struct node_core_id get_node_core_id(u32 nb_cfg_54)
 			id.nodeid = apicid & 0x7;
 		}
 	}
-
 	if (fam15h && dual_node) {
 		/* Coreboot expects each separate processor die to be on a different nodeid.
 		 * Since the code above returns nodeid 0 even on internal node 1 some fixup is needed...
diff --git a/src/mainboard/advansus/a785e-i/romstage.c b/src/mainboard/advansus/a785e-i/romstage.c
index b17fb2bcf1..55c94a63ad 100644
--- a/src/mainboard/advansus/a785e-i/romstage.c
+++ b/src/mainboard/advansus/a785e-i/romstage.c
@@ -151,7 +151,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/amd/bimini_fam10/romstage.c b/src/mainboard/amd/bimini_fam10/romstage.c
index 11d54797ea..be381a12ba 100644
--- a/src/mainboard/amd/bimini_fam10/romstage.c
+++ b/src/mainboard/amd/bimini_fam10/romstage.c
@@ -143,7 +143,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/amd/mahogany_fam10/romstage.c b/src/mainboard/amd/mahogany_fam10/romstage.c
index 5b39bb7304..3428aab9ab 100644
--- a/src/mainboard/amd/mahogany_fam10/romstage.c
+++ b/src/mainboard/amd/mahogany_fam10/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c b/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c
index 5131f36d9a..0c84b6d1e0 100644
--- a/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c
+++ b/src/mainboard/amd/serengeti_cheetah_fam10/romstage.c
@@ -251,7 +251,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/amd/tilapia_fam10/romstage.c b/src/mainboard/amd/tilapia_fam10/romstage.c
index 0804b32eca..68281e7455 100644
--- a/src/mainboard/amd/tilapia_fam10/romstage.c
+++ b/src/mainboard/amd/tilapia_fam10/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/asus/kfsn4-dre/romstage.c b/src/mainboard/asus/kfsn4-dre/romstage.c
index 8686be725b..e7ad0b1b65 100644
--- a/src/mainboard/asus/kfsn4-dre/romstage.c
+++ b/src/mainboard/asus/kfsn4-dre/romstage.c
@@ -284,7 +284,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 	if (IS_ENABLED(CONFIG_LOGICAL_CPUS)) {
 		/* Core0 on each node is configured. Now setup any additional cores. */
 		printk(BIOS_DEBUG, "start_other_cores()\n");
-		start_other_cores();
+		start_other_cores(bsp_apicid);
 		post_code(0x37);
 		wait_all_other_cores_started(bsp_apicid);
 	}
diff --git a/src/mainboard/asus/kgpe-d16/romstage.c b/src/mainboard/asus/kgpe-d16/romstage.c
index 3a682302bc..be91eaa29a 100644
--- a/src/mainboard/asus/kgpe-d16/romstage.c
+++ b/src/mainboard/asus/kgpe-d16/romstage.c
@@ -93,7 +93,18 @@ static void switch_spd_mux(uint8_t channel)
 	pci_write_config8(PCI_DEV(0, 0x14, 0), 0x54, byte);
 }
 
-static const uint8_t spd_addr[] = {
+static const uint8_t spd_addr_fam15[] = {
+	// Socket 0 Node 0 ("Node 0")
+	RC00, DIMM0, DIMM1, 0, 0, DIMM2, DIMM3, 0, 0,
+	// Socket 0 Node 1 ("Node 1")
+	RC00, DIMM4, DIMM5, 0, 0, DIMM6, DIMM7, 0, 0,
+	// Socket 1 Node 0 ("Node 2")
+	RC01, DIMM0, DIMM1, 0, 0, DIMM2, DIMM3, 0, 0,
+	// Socket 1 Node 1 ("Node 3")
+	RC01, DIMM4, DIMM5, 0, 0, DIMM6, DIMM7, 0, 0,
+};
+
+static const uint8_t spd_addr_fam10[] = {
 	// Socket 0 Node 0 ("Node 0")
 	RC00, DIMM0, DIMM1, 0, 0, DIMM2, DIMM3, 0, 0,
 	// Socket 0 Node 1 ("Node 1")
@@ -113,10 +124,10 @@ static void activate_spd_rom(const struct mem_controller *ctrl) {
 		switch_spd_mux(0x2);
 	} else if (ctrl->node_id == 1) {
 		printk(BIOS_DEBUG, "enable_spd_node1()\n");
-		switch_spd_mux((sysinfo->nodes <= 2)?0x2:0x3);
+		switch_spd_mux((is_fam15h() || (sysinfo->nodes <= 2))?0x2:0x3);
 	} else if (ctrl->node_id == 2) {
 		printk(BIOS_DEBUG, "enable_spd_node2()\n");
-		switch_spd_mux((sysinfo->nodes <= 2)?0x3:0x2);
+		switch_spd_mux((is_fam15h() || (sysinfo->nodes <= 2))?0x3:0x2);
 	} else if (ctrl->node_id == 3) {
 		printk(BIOS_DEBUG, "enable_spd_node3()\n");
 		switch_spd_mux(0x3);
@@ -303,18 +314,25 @@ static void execute_memory_test(void)
 
 void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 {
+	uint32_t esp;
+	__asm__ volatile (
+		"movl %%esp, %0"
+		: "=r" (esp)
+		);
+
 	struct sys_info *sysinfo = &sysinfo_car;
 
 	uint32_t bsp_apicid = 0, val;
 	uint8_t byte;
 	msr_t msr;
 
-	timestamp_init(timestamp_get());
-	timestamp_add_now(TS_START_ROMSTAGE);
-
 	int s3resume = acpi_is_wakeup_s3();
 
 	if (!cpu_init_detectedx && boot_cpu()) {
+		/* Initial timestamp */
+		timestamp_init(timestamp_get());
+		timestamp_add_now(TS_START_ROMSTAGE);
+
 		/* Nothing special needs to be done to find bus 0 */
 		/* Allow the HT devices to be found */
 		set_bsp_node_CHtExtNodeCfgEn();
@@ -338,6 +356,8 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 		pci_write_config8(PCI_DEV(0, 0x14, 3), 0x78, byte);
 	}
 
+	printk(BIOS_SPEW, "Initial stack pointer: %08x\n", esp);
+
 	post_code(0x30);
 
 	if (bist == 0)
@@ -396,7 +416,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 	if (IS_ENABLED(CONFIG_LOGICAL_CPUS)) {
 		/* Core0 on each node is configured. Now setup any additional cores. */
 		printk(BIOS_DEBUG, "start_other_cores()\n");
-		start_other_cores();
+		start_other_cores(bsp_apicid);
 		post_code(0x37);
 		wait_all_other_cores_started(bsp_apicid);
 	}
@@ -454,7 +474,10 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 
 	/* It's the time to set ctrl in sysinfo now; */
 	printk(BIOS_DEBUG, "fill_mem_ctrl() detected %d nodes\n", sysinfo->nodes);
-	fill_mem_ctrl(sysinfo->nodes, sysinfo->ctrl, spd_addr);
+	if (is_fam15h())
+		fill_mem_ctrl(sysinfo->nodes, sysinfo->ctrl, spd_addr_fam15);
+	else
+		fill_mem_ctrl(sysinfo->nodes, sysinfo->ctrl, spd_addr_fam10);
 	post_code(0x3D);
 
 #if 0
@@ -526,5 +549,12 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  */
 BOOL AMD_CB_ManualBUIDSwapList (u8 node, u8 link, const u8 **List)
 {
+	/* Force BUID to 0 */
+	static const u8 swaplist[] = {0, 0, 0xFF, 0, 0xFF};
+	if ((node == 0) && (link == 1)) {	/* BSP SB link */
+		*List = swaplist;
+		return 1;
+	}
+
 	return 0;
 }
diff --git a/src/mainboard/asus/m4a78-em/romstage.c b/src/mainboard/asus/m4a78-em/romstage.c
index f717ed8352..844fc93c79 100644
--- a/src/mainboard/asus/m4a78-em/romstage.c
+++ b/src/mainboard/asus/m4a78-em/romstage.c
@@ -147,7 +147,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/asus/m4a785-m/romstage.c b/src/mainboard/asus/m4a785-m/romstage.c
index 47d884cfdb..e43dbbc8ad 100644
--- a/src/mainboard/asus/m4a785-m/romstage.c
+++ b/src/mainboard/asus/m4a785-m/romstage.c
@@ -147,7 +147,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/asus/m5a88-v/romstage.c b/src/mainboard/asus/m5a88-v/romstage.c
index b69fd80518..1630497e26 100644
--- a/src/mainboard/asus/m5a88-v/romstage.c
+++ b/src/mainboard/asus/m5a88-v/romstage.c
@@ -148,7 +148,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/avalue/eax-785e/romstage.c b/src/mainboard/avalue/eax-785e/romstage.c
index ed85c8258d..ff1aa9e36a 100644
--- a/src/mainboard/avalue/eax-785e/romstage.c
+++ b/src/mainboard/avalue/eax-785e/romstage.c
@@ -152,7 +152,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/gigabyte/ma785gm/romstage.c b/src/mainboard/gigabyte/ma785gm/romstage.c
index c5a04330d2..baf49afda8 100644
--- a/src/mainboard/gigabyte/ma785gm/romstage.c
+++ b/src/mainboard/gigabyte/ma785gm/romstage.c
@@ -142,7 +142,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/gigabyte/ma785gmt/romstage.c b/src/mainboard/gigabyte/ma785gmt/romstage.c
index 743463da5f..b8ab282bb8 100644
--- a/src/mainboard/gigabyte/ma785gmt/romstage.c
+++ b/src/mainboard/gigabyte/ma785gmt/romstage.c
@@ -142,7 +142,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/gigabyte/ma78gm/romstage.c b/src/mainboard/gigabyte/ma78gm/romstage.c
index aee7fbec35..43bae3ca0d 100644
--- a/src/mainboard/gigabyte/ma78gm/romstage.c
+++ b/src/mainboard/gigabyte/ma78gm/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/hp/dl165_g6_fam10/romstage.c b/src/mainboard/hp/dl165_g6_fam10/romstage.c
index bf1b73456b..84e28f7fc9 100644
--- a/src/mainboard/hp/dl165_g6_fam10/romstage.c
+++ b/src/mainboard/hp/dl165_g6_fam10/romstage.c
@@ -156,7 +156,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/iei/kino-780am2-fam10/romstage.c b/src/mainboard/iei/kino-780am2-fam10/romstage.c
index bb0ba572b6..7b001761b0 100644
--- a/src/mainboard/iei/kino-780am2-fam10/romstage.c
+++ b/src/mainboard/iei/kino-780am2-fam10/romstage.c
@@ -145,7 +145,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/jetway/pa78vm5/romstage.c b/src/mainboard/jetway/pa78vm5/romstage.c
index 9144f3282e..8c87563c9e 100644
--- a/src/mainboard/jetway/pa78vm5/romstage.c
+++ b/src/mainboard/jetway/pa78vm5/romstage.c
@@ -150,7 +150,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
  #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
  #endif
diff --git a/src/mainboard/msi/ms9652_fam10/romstage.c b/src/mainboard/msi/ms9652_fam10/romstage.c
index cb4d02feba..f4413edead 100644
--- a/src/mainboard/msi/ms9652_fam10/romstage.c
+++ b/src/mainboard/msi/ms9652_fam10/romstage.c
@@ -173,7 +173,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	printk(BIOS_DEBUG, "wait_all_other_cores_started()\n");
 	wait_all_other_cores_started(bsp_apicid);
diff --git a/src/mainboard/supermicro/h8dmr_fam10/romstage.c b/src/mainboard/supermicro/h8dmr_fam10/romstage.c
index c29470f766..bf5093f417 100644
--- a/src/mainboard/supermicro/h8dmr_fam10/romstage.c
+++ b/src/mainboard/supermicro/h8dmr_fam10/romstage.c
@@ -167,7 +167,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/supermicro/h8qme_fam10/romstage.c b/src/mainboard/supermicro/h8qme_fam10/romstage.c
index eacc87f925..13a101185c 100644
--- a/src/mainboard/supermicro/h8qme_fam10/romstage.c
+++ b/src/mainboard/supermicro/h8qme_fam10/romstage.c
@@ -234,7 +234,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/supermicro/h8scm_fam10/romstage.c b/src/mainboard/supermicro/h8scm_fam10/romstage.c
index 6e34f0ac88..7312683fc0 100644
--- a/src/mainboard/supermicro/h8scm_fam10/romstage.c
+++ b/src/mainboard/supermicro/h8scm_fam10/romstage.c
@@ -158,7 +158,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/mainboard/tyan/s2912_fam10/romstage.c b/src/mainboard/tyan/s2912_fam10/romstage.c
index 9f25d271c4..ed211d9c86 100644
--- a/src/mainboard/tyan/s2912_fam10/romstage.c
+++ b/src/mainboard/tyan/s2912_fam10/romstage.c
@@ -169,7 +169,7 @@ void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
 #if CONFIG_LOGICAL_CPUS
 	/* Core0 on each node is configured. Now setup any additional cores. */
 	printk(BIOS_DEBUG, "start_other_cores()\n");
-	start_other_cores();
+	start_other_cores(bsp_apicid);
 	post_code(0x37);
 	wait_all_other_cores_started(bsp_apicid);
 #endif
diff --git a/src/northbridge/amd/amdht/h3finit.c b/src/northbridge/amd/amdht/h3finit.c
index c9a310b9b8..bcdcfa2ae6 100644
--- a/src/northbridge/amd/amdht/h3finit.c
+++ b/src/northbridge/amd/amdht/h3finit.c
@@ -385,13 +385,49 @@ static u8 convertNodeToLink(u8 srcNode, u8 targetNode, sMainData *pDat)
  */
 static void htDiscoveryFloodFill(sMainData *pDat)
 {
-	u8 currentNode = 0;
-	u8 currentLink;
+	uint8_t currentNode = 0;
+	uint8_t currentLink;
+	uint8_t currentLinkID;
+
+	/* NOTE
+	 * Each node inside a dual node (socket G34) processor must share
+	 * an adjacent node ID.  Alter the link scan order such that the
+	 * other internal node is always scanned first...
+	 */
+	uint8_t currentLinkScanOrder_Default[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+	uint8_t currentLinkScanOrder_G34_Fam10[8] = {1, 0, 2, 3, 4, 5, 6, 7};
+	uint8_t currentLinkScanOrder_G34_Fam15[8] = {2, 0, 1, 3, 4, 5, 6, 7};
+
+	uint8_t fam15h = 0;
+	uint8_t rev_gte_d = 0;
+	uint8_t dual_node = 0;
+	uint32_t f3xe8;
+	uint32_t family;
+	uint32_t model;
+
+	f3xe8 = pci_read_config32(NODE_PCI(0, 3), 0xe8);
+
+	family = model = cpuid_eax(0x80000001);
+	model = ((model & 0xf0000) >> 12) | ((model & 0xf0) >> 4);
+	family = ((family & 0xf00000) >> 16) | ((family & 0xf00) >> 8);
+
+	if (family >= 0x6f) {
+		/* Family 15h or later */
+		fam15h = 1;
+	}
+
+	if ((model >= 0x8) || fam15h)
+		/* Revision D or later */
+		rev_gte_d = 1;
+
+	if (rev_gte_d)
+		 /* Check for dual node capability */
+		if (f3xe8 & 0x20000000)
+			dual_node = 1;
 
 	/* Entries are always added in pairs, the even indices are the 'source'
 	 * side closest to the BSP, the odd indices are the 'destination' side
 	 */
-
 	while (currentNode <= pDat->NodesDiscovered)
 	{
 		u32 temp;
@@ -419,11 +455,24 @@ static void htDiscoveryFloodFill(sMainData *pDat)
 		/* Enable routing tables on currentNode*/
 		pDat->nb->enableRoutingTables(currentNode, pDat->nb);
 
-		for (currentLink = 0; currentLink < pDat->nb->maxLinks; currentLink++)
+		for (currentLinkID = 0; currentLinkID < pDat->nb->maxLinks; currentLinkID++)
 		{
 			BOOL linkfound;
 			u8 token;
 
+			if (currentLinkID < 8) {
+				if (dual_node) {
+					if (fam15h)
+						currentLink = currentLinkScanOrder_G34_Fam15[currentLinkID];
+					else
+						currentLink = currentLinkScanOrder_G34_Fam10[currentLinkID];
+				} else {
+					currentLink = currentLinkScanOrder_Default[currentLinkID];
+				}
+			} else {
+				currentLink = currentLinkID;
+			}
+
 			if (pDat->HtBlock->AMD_CB_IgnoreLink && pDat->HtBlock->AMD_CB_IgnoreLink(currentNode, currentLink))
 				continue;
 
diff --git a/src/northbridge/amd/amdht/h3ncmn.c b/src/northbridge/amd/amdht/h3ncmn.c
index ac7d393b24..01c0e48d0c 100644
--- a/src/northbridge/amd/amdht/h3ncmn.c
+++ b/src/northbridge/amd/amdht/h3ncmn.c
@@ -47,8 +47,9 @@
 #define REG_NODE_ID_0X60		0x60
 #define REG_UNIT_ID_0X64		0x64
 #define REG_LINK_TRANS_CONTROL_0X68	0x68
-#define REG_LINK_INIT_CONTROL_0X6C	0x6C
+#define REG_LINK_INIT_CONTROL_0X6C	0x6c
 #define REG_HT_CAP_BASE_0X80		0x80
+#define REG_NORTHBRIDGE_CFG_3X8C	0x8c
 #define REG_HT_LINK_RETRY0_0X130	0x130
 #define REG_HT_TRAFFIC_DIST_0X164	0x164
 #define REG_HT_LINK_EXT_CONTROL0_0X170	0x170
@@ -87,6 +88,21 @@
  ***			FAMILY/NORTHBRIDGE SPECIFIC FUNCTIONS		***
  ***************************************************************************/
 
+static inline uint8_t is_fam15h(void)
+{
+	uint8_t fam15h = 0;
+	uint32_t family;
+
+	family = cpuid_eax(0x80000001);
+	family = ((family & 0xf00000) >> 16) | ((family & 0xf00) >> 8);
+
+	if (family >= 0x6f)
+		/* Family 15h or later */
+		fam15h = 1;
+
+	return fam15h;
+}
+
 /***************************************************************************//**
  *
  * SBDFO
@@ -215,8 +231,18 @@ static void writeRoutingTable(u8 node, u8 target, u8 link, cNorthBridge *nb)
 
 static void writeNodeID(u8 node, u8 nodeID, cNorthBridge *nb)
 {
-	u32 temp = nodeID;
+	u32 temp;
 	ASSERT((node < nb->maxNodes) && (nodeID < nb->maxNodes));
+	if (is_fam15h()) {
+		temp = 1;
+		AmdPCIWriteBits(MAKE_SBDFO(makePCISegmentFromNode(node),
+					makePCIBusFromNode(node),
+					makePCIDeviceFromNode(node),
+					CPU_NB_FUNC_03,
+					REG_NORTHBRIDGE_CFG_3X8C),
+					22, 22, &temp);
+	}
+	temp = nodeID;
 	AmdPCIWriteBits(MAKE_SBDFO(makePCISegmentFromNode(node),
 				makePCIBusFromNode(node),
 				makePCIDeviceFromNode(node),
diff --git a/src/northbridge/amd/amdht/ht_wrapper.c b/src/northbridge/amd/amdht/ht_wrapper.c
index 1f38b0c032..18c0ac4456 100644
--- a/src/northbridge/amd/amdht/ht_wrapper.c
+++ b/src/northbridge/amd/amdht/ht_wrapper.c
@@ -82,22 +82,129 @@ static  u32 get_nodes(void)
 	return nodes;
 }
 
+static const char * event_class_string_decodes[] = {
+	[HT_EVENT_CLASS_CRITICAL] = "CRITICAL",
+	[HT_EVENT_CLASS_ERROR] = "ERROR",
+	[HT_EVENT_CLASS_HW_FAULT] = "HARDWARE FAULT",
+	[HT_EVENT_CLASS_WARNING] = "WARNING",
+	[HT_EVENT_CLASS_INFO] = "INFO"
+};
+
+static const char * event_string_decodes[] = {
+	[HT_EVENT_COH_EVENTS] = "HT_EVENT_COH_EVENTS",
+	[HT_EVENT_COH_NO_TOPOLOGY] = "HT_EVENT_COH_NO_TOPOLOGY",
+	[HT_EVENT_COH_LINK_EXCEED] = "HT_EVENT_COH_LINK_EXCEED",
+	[HT_EVENT_COH_FAMILY_FEUD] = "HT_EVENT_COH_FAMILY_FEUD",
+	[HT_EVENT_COH_NODE_DISCOVERED] = "HT_EVENT_COH_NODE_DISCOVERED",
+	[HT_EVENT_COH_MPCAP_MISMATCH] = "HT_EVENT_COH_MPCAP_MISMATCH",
+	[HT_EVENT_NCOH_EVENTS] = "HT_EVENT_NCOH_EVENTS",
+	[HT_EVENT_NCOH_BUID_EXCEED] = "HT_EVENT_NCOH_BUID_EXCEED",
+	[HT_EVENT_NCOH_LINK_EXCEED] = "HT_EVENT_NCOH_LINK_EXCEED",
+	[HT_EVENT_NCOH_BUS_MAX_EXCEED] = "HT_EVENT_NCOH_BUS_MAX_EXCEED",
+	[HT_EVENT_NCOH_CFG_MAP_EXCEED] = "HT_EVENT_NCOH_CFG_MAP_EXCEED",
+	[HT_EVENT_NCOH_DEVICE_FAILED] = "HT_EVENT_NCOH_DEVICE_FAILED",
+	[HT_EVENT_NCOH_AUTO_DEPTH] = "HT_EVENT_NCOH_AUTO_DEPTH",
+	[HT_EVENT_OPT_EVENTS] = "HT_EVENT_OPT_EVENTS",
+	[HT_EVENT_OPT_REQUIRED_CAP_RETRY] = "HT_EVENT_OPT_REQUIRED_CAP_RETRY",
+	[HT_EVENT_OPT_REQUIRED_CAP_GEN3] = "HT_EVENT_OPT_REQUIRED_CAP_GEN3",
+	[HT_EVENT_HW_EVENTS] = "HT_EVENT_HW_EVENTS",
+	[HT_EVENT_HW_SYNCHFLOOD] = "HT_EVENT_HW_SYNCHFLOOD",
+	[HT_EVENT_HW_HTCRC] = "HT_EVENT_HW_HTCRC"
+};
 
 /**
  * void AMD_CB_EventNotify (u8 evtClass, u16 event, const u8 *pEventData0)
  */
 static void AMD_CB_EventNotify (u8 evtClass, u16 event, const u8 *pEventData0)
 {
-	u8 i;
+	uint8_t i;
+	uint8_t log_level;
+	uint8_t dump_event_detail;
 
-	printk(BIOS_DEBUG, "AMD_CB_EventNotify()\n");
-	printk(BIOS_DEBUG, " event class: %02x\n event: %04x\n data: ", evtClass, event);
+	printk(BIOS_DEBUG, "AMD_CB_EventNotify(): ");
 
-	for (i = 0; i < *pEventData0; i++) {
-		printk(BIOS_DEBUG, " %02x ", *(pEventData0 + i));
+	/* Decode event */
+	dump_event_detail = 1;
+	switch (evtClass) {
+		case HT_EVENT_CLASS_CRITICAL:
+		case HT_EVENT_CLASS_ERROR:
+		case HT_EVENT_CLASS_HW_FAULT:
+		case HT_EVENT_CLASS_WARNING:
+		case HT_EVENT_CLASS_INFO:
+			log_level = BIOS_DEBUG;
+			printk(log_level, event_class_string_decodes[evtClass]);
+			break;
+		default:
+			log_level = BIOS_DEBUG;
+			printk(log_level, "UNKNOWN");
+			break;
 	}
-	printk(BIOS_DEBUG, "\n");
+	printk(log_level, ": ");
 
+	switch(event) {
+		case HT_EVENT_COH_EVENTS:
+		case HT_EVENT_COH_NO_TOPOLOGY:
+		case HT_EVENT_COH_LINK_EXCEED:
+		case HT_EVENT_COH_FAMILY_FEUD:
+			printk(log_level, event_string_decodes[event]);
+			break;
+		case HT_EVENT_COH_NODE_DISCOVERED:
+			{
+				printk(log_level, "HT_EVENT_COH_NODE_DISCOVERED");
+				sHtEventCohNodeDiscovered *evt = (sHtEventCohNodeDiscovered*)pEventData0;
+				printk(log_level, ": node %d link %d new node: %d",
+					evt->node, evt->link, evt->newNode);
+				dump_event_detail = 0;
+				break;
+			}
+		case HT_EVENT_COH_MPCAP_MISMATCH:
+		case HT_EVENT_NCOH_EVENTS:
+		case HT_EVENT_NCOH_BUID_EXCEED:
+		case HT_EVENT_NCOH_LINK_EXCEED:
+		case HT_EVENT_NCOH_BUS_MAX_EXCEED:
+		case HT_EVENT_NCOH_CFG_MAP_EXCEED:
+			printk(log_level, event_string_decodes[event]);
+			break;
+		case HT_EVENT_NCOH_DEVICE_FAILED:
+			{
+				printk(log_level, event_string_decodes[event]);
+				sHtEventNcohDeviceFailed *evt = (sHtEventNcohDeviceFailed*)pEventData0;
+				printk(log_level, ": node %d link %d depth: %d attemptedBUID: %d",
+					evt->node, evt->link, evt->depth, evt->attemptedBUID);
+				dump_event_detail = 0;
+				break;
+			}
+		case HT_EVENT_NCOH_AUTO_DEPTH:
+			{
+				printk(log_level, event_string_decodes[event]);
+				sHtEventNcohAutoDepth *evt = (sHtEventNcohAutoDepth*)pEventData0;
+				printk(log_level, ": node %d link %d depth: %d",
+					evt->node, evt->link, evt->depth);
+				dump_event_detail = 0;
+				break;
+			}
+		case HT_EVENT_OPT_EVENTS:
+		case HT_EVENT_OPT_REQUIRED_CAP_RETRY:
+		case HT_EVENT_OPT_REQUIRED_CAP_GEN3:
+		case HT_EVENT_HW_EVENTS:
+		case HT_EVENT_HW_SYNCHFLOOD:
+		case HT_EVENT_HW_HTCRC:
+			printk(log_level, event_string_decodes[event]);
+			break;
+		default:
+			printk(log_level, "HT_EVENT_UNKNOWN");
+			break;
+	}
+	printk(log_level, "\n");
+
+	if (dump_event_detail) {
+		printk(BIOS_DEBUG, " event class: %02x\n event: %04x\n data: ", evtClass, event);
+
+		for (i = 0; i < *pEventData0; i++) {
+			printk(BIOS_DEBUG, " %02x ", *(pEventData0 + i));
+		}
+		printk(BIOS_DEBUG, "\n");
+	}
 }
 
 /**
@@ -206,9 +313,10 @@ void amd_ht_fixup(struct sys_info *sysinfo) {
 				for (node = 0; node < node_count; node++) {
 					f3xe8 = pci_read_config32(NODE_PCI(node, 3), 0xe8);
 					uint8_t internal_node_number = ((f3xe8 & 0xc0000000) >> 30);
-					printk(BIOS_DEBUG, "amd_ht_fixup(): node %d (internal node ID %d): disabling defective HT link\n", node, internal_node_number);
+					printk(BIOS_DEBUG, "amd_ht_fixup(): node %d (internal node ID %d): disabling defective HT link", node, internal_node_number);
 					if (internal_node_number == 0) {
 						uint8_t package_link_3_connected = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x98:0xd8) & 0x1;
+						printk(BIOS_DEBUG, " (L3 connected: %d)\n", package_link_3_connected);
 						if (package_link_3_connected) {
 							/* Set WidthIn and WidthOut to 0 */
 							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x84:0xc4);
@@ -230,15 +338,21 @@ void amd_ht_fixup(struct sys_info *sysinfo) {
 						}
 					} else if (internal_node_number == 1) {
 						uint8_t package_link_3_connected = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0xf8:0xb8) & 0x1;
+						printk(BIOS_DEBUG, " (L3 connected: %d)\n", package_link_3_connected);
 						if (package_link_3_connected) {
 							/* Set WidthIn and WidthOut to 0 */
 							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0xe4:0xa4);
 							dword &= ~0x77000000;
 							pci_write_config32(NODE_PCI(node, 0), (fam15h)?0xe4:0xa4, dword);
 							/* Set Ganged to 1 */
-							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x18c:0x174);
+							/* WARNING
+							 * The Family 15h BKDG states that 0x18c should be set,
+							 * however this is in error.  0x17c is the correct control
+							 * register (sublink 0) for these processors...
+							 */
+							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x17c:0x174);
 							dword |= 0x00000001;
-							pci_write_config32(NODE_PCI(node, 0), (fam15h)?0x18c:0x174, dword);
+							pci_write_config32(NODE_PCI(node, 0), (fam15h)?0x17c:0x174, dword);
 						} else {
 							/* Set ConnDly to 1 */
 							dword = pci_read_config32(NODE_PCI(node, 0), 0x16c);
diff --git a/src/northbridge/amd/amdmct/mct_ddr3/mct_d.c b/src/northbridge/amd/amdmct/mct_ddr3/mct_d.c
index 25c4042c23..bfe7515cf1 100644
--- a/src/northbridge/amd/amdmct/mct_ddr3/mct_d.c
+++ b/src/northbridge/amd/amdmct/mct_ddr3/mct_d.c
@@ -5451,6 +5451,7 @@ static void mct_InitialMCT_D(struct MCTStatStruc *pMCTstat, struct DCTStatStruc
 		cpu_divisor = (0x1 << cpu_did);
 		pMCTstat->TSCFreq = (100 * (cpu_fid + 0x10)) / cpu_divisor;
 
+		printk(BIOS_DEBUG, "mct_InitialMCT_D: mct_ForceNBPState0_En_Fam15\n");
 		mct_ForceNBPState0_En_Fam15(pMCTstat, pDCTstat);
 	} else {
 		/* K10 BKDG v3.62 section 2.8.9.2 */