CONFIG_USE_PRINTK_IN_CAR and ht chain id for HTX support in

serengeti_cheeatah


git-svn-id: svn://svn.coreboot.org/coreboot/trunk@2439 2b7e53f0-3cfb-0310-b3e9-8179ed1497e1

10 files changed, 247 insertions, 97 deletions

diff --git a/src/northbridge/amd/amdk8/amdk8.h b/src/northbridge/amd/amdk8/amdk8.h
index 56b92a2349..095f6eace4 100644
--- a/src/northbridge/amd/amdk8/amdk8.h
+++ b/src/northbridge/amd/amdk8/amdk8.h
@@ -238,6 +238,35 @@
 #define NonCoherent       (1 << 2)
 #define ConnectionPending (1 << 4)
 
+#include "raminit.h"
+//struct definitions
+
+#if RAMINIT_SYSINFO==1
+struct link_pair_st {
+        device_t udev;
+        uint32_t upos;
+        uint32_t uoffs;
+        device_t dev;
+        uint32_t pos;
+        uint32_t offs;
+
+} __attribute__((packed));
+
+struct sys_info {
+        uint8_t ctrl_present[NODE_NUMS];
+        struct mem_controller ctrl[NODE_NUMS];
+
+        uint32_t nodes;
+        struct link_pair_st link_pair[16];// enough? only in_conherent
+        uint32_t link_pair_num;
+        uint32_t ht_c_num;
+        uint32_t sbdn;
+        uint32_t sblk;
+        uint32_t sbbusn;
+} __attribute__((packed));
+#endif
+
+
 #endif
 
 #endif /* AMDK8_H */
diff --git a/src/northbridge/amd/amdk8/amdk8_f.h b/src/northbridge/amd/amdk8/amdk8_f.h
index 7901d08525..d1fcb8bb37 100644
--- a/src/northbridge/amd/amdk8/amdk8_f.h
+++ b/src/northbridge/amd/amdk8/amdk8_f.h
@@ -499,7 +499,7 @@ struct sys_info {
         uint8_t ctrl_present[NODE_NUMS];
         struct mem_info meminfo[NODE_NUMS];
 	struct mem_controller ctrl[NODE_NUMS];
-        uint8_t mem_trained[NODE_NUMS];
+	uint8_t mem_trained[NODE_NUMS]; //0: no dimm, 1: trained, 0x80: not started, 0x81: recv1 fail, 0x82: Pos Fail, 0x83:recv2 fail
         uint32_t tom_k;
         uint32_t tom2_k;
 
@@ -518,21 +518,23 @@ struct sys_info {
 	uint32_t sbbusn;
 } __attribute__((packed));
 
-#if MEM_TRAIN_SEQ == 1
+#ifdef __ROMCC__
+static void soft_reset(void);
+#endif
 
 static void wait_all_core0_mem_trained(struct sys_info *sysinfo)
 {
+
         int i;
         uint32_t mask = 0;
+	unsigned needs_reset = 0;
+
 
 	if(sysinfo->nodes == 1) return; // in case only one cpu installed	
 
         for(i=1; i<sysinfo->nodes; i++) {
-                if (!sysinfo->ctrl_present[ i ])
-                        continue;
-
                 /* Skip everything if I don't have any memory on this controller */
-                if(sysinfo->meminfo[i].dimm_mask==0x00) continue;
+                if(sysinfo->mem_trained[i]==0x00) continue;
 
                 mask |= (1<<i);
 
@@ -541,21 +543,49 @@ static void wait_all_core0_mem_trained(struct sys_info *sysinfo)
         i = 1;
         while(1) {
 		if(mask & (1<<i)) {
-			if((sysinfo->mem_trained[i])) {
+			if((sysinfo->mem_trained[i])!=0x80) {
 				mask &= ~(1<<i);
 			}
 		}
 
                 if(!mask) break;
 
+#if 0
 		/* cpu_relax */
 		__asm__ __volatile__("rep;nop": : :"memory");
+#endif
 
                 i++;
                 i%=sysinfo->nodes;
 	}
 
-}
+	for(i=0; i<sysinfo->nodes; i++) {
+#ifdef __ROMCC__
+		print_debug("mem_trained["); print_debug_hex8(i); print_debug("]="); print_debug_hex8(sysinfo->mem_trained[i]); print_debug("\r\n");
+#else
+		printk_debug("mem_trained[%02x]=%02x\n", i, sysinfo->mem_trained[i]); 
 #endif
+                switch(sysinfo->mem_trained[i]) {
+		case 0: //don't need train
+		case 1: //trained
+			break;
+		case 0x81: //recv1: fail
+		case 0x82: //Pos :fail
+		case 0x83: //recv2: fail
+			needs_reset = 1;
+			break;
+		}
+	}
+	if(needs_reset) {
+#ifdef __ROMCC__
+		print_debug("mem trained failed\r\n");
+		soft_reset();
+#else
+		printk_debug("mem trained failed\n"); 
+		hard_reset();
+#endif
+	}
+
+}
 
 #endif /* AMDK8_F_H */
diff --git a/src/northbridge/amd/amdk8/coherent_ht_car.c b/src/northbridge/amd/amdk8/coherent_ht_car.c
index c0c4b338b9..b731a89415 100644
--- a/src/northbridge/amd/amdk8/coherent_ht_car.c
+++ b/src/northbridge/amd/amdk8/coherent_ht_car.c
@@ -113,7 +113,7 @@ typedef uint32_t u32;
 static inline void print_linkn (const char *strval, uint8_t byteval) 
 {
 #if 1
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 	printk_debug("%s%02x\r\n", strval, byteval); 
 #else
 	print_debug(strval); print_debug_hex8(byteval); print_debug("\r\n");
@@ -285,14 +285,14 @@ static uint16_t read_freq_cap(device_t dev, uint8_t pos)
 	freq_cap = pci_read_config16(dev, pos);
 	freq_cap &= ~(1 << HT_FREQ_VENDOR); /* Ignore Vendor HT frequencies */
 
-#if K8_REV_F_SUPPORT == 0
 #if K8_HT_FREQ_1G_SUPPORT == 1
+    #if K8_REV_F_SUPPORT == 0
 	if (!is_cpu_pre_e0())
+    #endif 
 	{
 		return freq_cap;
 	}
 #endif
-#endif
 
 	id = pci_read_config32(dev, 0);
 
@@ -1503,7 +1503,7 @@ static unsigned setup_smp(void)
 		nodes = setup_smp8();
 #endif
 
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 	printk_debug("%02x nodes initialized.\r\n", nodes);
 #else
 	print_debug_hex8(nodes);
@@ -1613,11 +1613,9 @@ static void coherent_ht_finalize(unsigned nodes)
 	 */
 
 	print_spew("coherent_ht_finalize\r\n");
-
 #if K8_REV_F_SUPPORT == 0
 	rev_a0 = is_cpu_rev_a0();
 #endif
-
 	for (node = 0; node < nodes; node++) {
 		device_t dev;
 		uint32_t val;
diff --git a/src/northbridge/amd/amdk8/debug.c b/src/northbridge/amd/amdk8/debug.c
index ca45cbe519..42104d43af 100644
--- a/src/northbridge/amd/amdk8/debug.c
+++ b/src/northbridge/amd/amdk8/debug.c
@@ -6,7 +6,7 @@
 static inline void print_debug_addr(const char *str, void *val)
 {
 #if CACHE_AS_RAM_ADDRESS_DEBUG == 1
-        #if CONFIG_USE_INIT==1
+	#if CONFIG_USE_PRINTK_IN_CAR
                 printk_debug("------Address debug: %s%x------\r\n", str, val);
         #else
 		print_debug ("------Address debug: "); print_debug(str); print_debug_hex32(val); print_debug("------\r\n");
@@ -17,15 +17,15 @@ static inline void print_debug_addr(const char *str, void *val)
 #if 1
 static void print_debug_pci_dev(unsigned dev)
 {
-#if CONFIG_USE_INIT
-	printk_debug("PCI: %02x:%02x.%02x", (dev>>16) & 0xff, (dev>>11) & 0x1f, (dev>>8) & 0x7);
+#if CONFIG_USE_PRINTK_IN_CAR
+	printk_debug("PCI: %02x:%02x.%02x", (dev>>20) & 0xff, (dev>>15) & 0x1f, (dev>>12) & 0x7);
 #else
 	print_debug("PCI: ");
-	print_debug_hex8((dev >> 16) & 0xff);
+	print_debug_hex8((dev >> 20) & 0xff);
 	print_debug_char(':');
-	print_debug_hex8((dev >> 11) & 0x1f);
+	print_debug_hex8((dev >> 15) & 0x1f);
 	print_debug_char('.');
-	print_debug_hex8((dev >> 8) & 7);
+	print_debug_hex8((dev >> 12) & 7);
 #endif
 }
 
@@ -43,14 +43,14 @@ static void print_pci_devices(void)
 			continue;
 		}
 		print_debug_pci_dev(dev);
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 		printk_debug(" %04x:%04x\r\n", (id & 0xffff), (id>>16));
 #else
 		print_debug(" ");
 		print_debug_hex32(id);
 		print_debug("\r\n");
 #endif
-		if(((dev>>8) & 0x07) == 0) {
+		if(((dev>>12) & 0x07) == 0) {
 			uint8_t hdr_type;
 			hdr_type = pci_read_config8(dev, PCI_HEADER_TYPE);
 			if((hdr_type & 0x80) != 0x80) {
@@ -68,7 +68,7 @@ static void dump_pci_device(unsigned dev)
 	for(i = 0; i < 256; i++) {
 		unsigned char val;
 		if ((i & 0x0f) == 0) {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
                         printk_debug("\r\n%02x:",i);
 #else
 			print_debug("\r\n");
@@ -77,7 +77,7 @@ static void dump_pci_device(unsigned dev)
 #endif
 		}
 		val = pci_read_config8(dev, i);
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 		printk_debug(" %02x", val);
 #else
 		print_debug_char(' ');
@@ -87,6 +87,39 @@ static void dump_pci_device(unsigned dev)
 	print_debug("\r\n");
 }
 
+#if K8_REV_F_SUPPORT == 1
+static uint32_t pci_read_config32_index_wait(device_t dev, uint32_t index_reg, uint32_t index);
+static void dump_pci_device_index_wait(unsigned dev, uint32_t index_reg)
+{
+        int i;
+        print_debug_pci_dev(dev);
+	print_debug(" -- index_reg="); print_debug_hex32(index_reg); 
+
+        for(i = 0; i < 0x40; i++) {
+                uint32_t val;
+		int j;
+#if CONFIG_USE_PRINTK_IN_CAR
+                printk_debug("\r\n%02x:",i);
+#else
+                print_debug("\r\n");
+                print_debug_hex8(i);
+                print_debug_char(':');
+#endif
+                val = pci_read_config32_index_wait(dev, index_reg, i);
+		for(j=0;j<4;j++) {
+#if CONFIG_USE_PRINTK_IN_CAR
+	                printk_debug(" %02x", val & 0xff);
+#else
+	                print_debug_char(' '); print_debug_hex8(val&0xff);
+#endif
+			val >>= 8;
+		}
+
+        }
+        print_debug("\r\n");
+}
+#endif
+
 static void dump_pci_devices(void)
 {
 	device_t dev;
@@ -102,7 +135,7 @@ static void dump_pci_devices(void)
 		}
 		dump_pci_device(dev);
 
-                if(((dev>>8) & 0x07) == 0) {
+                if(((dev>>12) & 0x07) == 0) {
                         uint8_t hdr_type;
                         hdr_type = pci_read_config8(dev, PCI_HEADER_TYPE);
                         if((hdr_type & 0x80) != 0x80) {
@@ -127,7 +160,7 @@ static void dump_pci_devices_on_bus(unsigned busn)
                 }
                 dump_pci_device(dev);
 
-                if(((dev>>8) & 0x07) == 0) {
+                if(((dev>>12) & 0x07) == 0) {
                         uint8_t hdr_type;
                         hdr_type = pci_read_config8(dev, PCI_HEADER_TYPE);
                         if((hdr_type & 0x80) != 0x80) {
@@ -137,6 +170,11 @@ static void dump_pci_devices_on_bus(unsigned busn)
         }
 }
 
+#ifndef DEBUG_SMBUS
+#define DEBUG_SMBUS 0
+#endif
+
+#if DEBUG_SMBUS == 1
 static void dump_spd_registers(const struct mem_controller *ctrl)
 {
 	int i;
@@ -146,7 +184,7 @@ static void dump_spd_registers(const struct mem_controller *ctrl)
 		device = ctrl->channel0[i];
 		if (device) {
 			int j;
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 			printk_debug("dimm: %02x.0: %02x", i, device);
 #else
 			print_debug("dimm: "); 
@@ -158,7 +196,7 @@ static void dump_spd_registers(const struct mem_controller *ctrl)
 				int status;
 				unsigned char byte;
 				if ((j & 0xf) == 0) {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 					printk_debug("\r\n%02x: ", j);
 #else
 					print_debug("\r\n");
@@ -171,7 +209,7 @@ static void dump_spd_registers(const struct mem_controller *ctrl)
 					break;
 				}
 				byte = status & 0xff;
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 				printk_debug("%02x ", byte);
 #else
 				print_debug_hex8(byte);
@@ -183,7 +221,7 @@ static void dump_spd_registers(const struct mem_controller *ctrl)
 		device = ctrl->channel1[i];
 		if (device) {
 			int j;
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
                         printk_debug("dimm: %02x.1: %02x", i, device);
 #else
 			print_debug("dimm: "); 
@@ -195,7 +233,7 @@ static void dump_spd_registers(const struct mem_controller *ctrl)
 				int status;
 				unsigned char byte;
 				if ((j & 0xf) == 0) {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
                                         printk_debug("\r\n%02x: ", j);
 #else
 					print_debug("\r\n");
@@ -208,7 +246,7 @@ static void dump_spd_registers(const struct mem_controller *ctrl)
 					break;
 				}
 				byte = status & 0xff;
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
                                 printk_debug("%02x ", byte);
 #else
 				print_debug_hex8(byte);
@@ -226,7 +264,7 @@ static void dump_smbus_registers(void)
         for(device = 1; device < 0x80; device++) {
                 int j;
 		if( smbus_read_byte(device, 0) < 0 ) continue;
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 		printk_debug("smbus: %02x", device);
 #else
                 print_debug("smbus: ");
@@ -240,7 +278,7 @@ static void dump_smbus_registers(void)
 				break;
                         }
                         if ((j & 0xf) == 0) {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 				printk_debug("\r\n%02x: ",j);
 #else
                 	        print_debug("\r\n");
@@ -249,7 +287,7 @@ static void dump_smbus_registers(void)
 #endif
                         }
                         byte = status & 0xff;
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
                         printk_debug("%02x ", byte);
 #else
                         print_debug_hex8(byte);
@@ -259,13 +297,14 @@ static void dump_smbus_registers(void)
                 print_debug("\r\n");
 	}	
 }
+#endif
 
 static void dump_io_resources(unsigned port) 
 {
 
 	int i;
         udelay(2000);
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 	printk_debug("%04x:\r\n", port);
 #else
         print_debug_hex16(port);
@@ -274,7 +313,7 @@ static void dump_io_resources(unsigned port)
         for(i=0;i<256;i++) {
                 uint8_t val;
                 if ((i & 0x0f) == 0) {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 			printk_debug("%02x:", i);
 #else
                         print_debug_hex8(i);
@@ -282,7 +321,7 @@ static void dump_io_resources(unsigned port)
 #endif
                 }
                 val = inb(port);
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 		printk_debug(" %02x",val);
 #else
                 print_debug_char(' ');
@@ -301,7 +340,7 @@ static void dump_mem(unsigned start, unsigned end)
 	print_debug("dump_mem:");
         for(i=start;i<end;i++) {
 		if((i & 0xf)==0) {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
 			printk_debug("\r\n%08x:", i);
 #else	
 			print_debug("\r\n");
@@ -309,7 +348,7 @@ static void dump_mem(unsigned start, unsigned end)
 			print_debug(":");
 #endif
 		}
-#if CONFIG_USE_INIT	
+#if CONFIG_USE_PRINTK_IN_CAR
 		printk_debug(" %02x", (unsigned char)*((unsigned char *)i));
 #else
 		print_debug(" ");
diff --git a/src/northbridge/amd/amdk8/get_sblk_pci1234.c b/src/northbridge/amd/amdk8/get_sblk_pci1234.c
index 61e89028e1..5c15c1e462 100644
--- a/src/northbridge/amd/amdk8/get_sblk_pci1234.c
+++ b/src/northbridge/amd/amdk8/get_sblk_pci1234.c
@@ -176,6 +176,7 @@ void get_sblk_pci1234(void)
         dword &=0x0300;
         dword |= 1;
         sysconf.pci1234[0] = dword;
+	sysconf.hcid[0] = 0;
 
         /*about hardcode numbering for HT_IO support
                 set the node_id and link_id that could have ht chain in the one array,
@@ -221,6 +222,7 @@ void get_sblk_pci1234(void)
                         sysconf.pci1234[i] = 0;
 			sysconf.hcdn[i] = 0x20202020;
                 }
+		sysconf.hcid[i] = 0;
         }
 
 }
diff --git a/src/northbridge/amd/amdk8/incoherent_ht.c b/src/northbridge/amd/amdk8/incoherent_ht.c
index 012caf1501..de5bfd537b 100644
--- a/src/northbridge/amd/amdk8/incoherent_ht.c
+++ b/src/northbridge/amd/amdk8/incoherent_ht.c
@@ -26,7 +26,7 @@
 
 static inline void print_linkn_in (const char *strval, uint8_t byteval)
 {
-#if CONFIG_USE_INIT
+#if CONFIG_USE_PRINTK_IN_CAR
         printk_debug("%s%02x\r\n", strval, byteval); 
 #else
         print_debug(strval); print_debug_hex8(byteval); print_debug("\r\n");
@@ -297,7 +297,7 @@ static int scan_pci_bus( unsigned bus)
 	new_bus = bus;
 
 #if 0
-#if CONFIG_USE_INIT == 1
+#if CONFIG_USE_PRINTK_IN_CAR
 	printk_debug("bus_num=%02x\r\n", bus);
 #endif
 #endif
@@ -313,7 +313,7 @@ static int scan_pci_bus( unsigned bus)
                 class = pci_read_config16(dev, PCI_CLASS_DEVICE);
 
 #if 0
-#if CONFIG_USE_INIT == 1
+#if CONFIG_USE_PRINTK_IN_CAR
 		if(hdr_type !=0xff ) {
 			printk_debug("dev=%02x fn=%02x hdr_type=%02x class=%04x\r\n", 
 				(devfn>>3)& 0x1f, (devfn & 0x7), hdr_type, class);
diff --git a/src/northbridge/amd/amdk8/raminit.c b/src/northbridge/amd/amdk8/raminit.c
index f7a12e77f8..9a028bf2e6 100644
--- a/src/northbridge/amd/amdk8/raminit.c
+++ b/src/northbridge/amd/amdk8/raminit.c
@@ -3,6 +3,9 @@
 	2004.12 yhlu add D0 support
 	2005.02 yhlu add E0 memory hole support
 */
+#if K8_REV_F_SUPPORT == 1
+        #include "raminit_f.c"
+#else
 
 #include <cpu/x86/mem.h>
 #include <cpu/x86/cache.h>
@@ -35,7 +38,7 @@ static void setup_resource_map(const unsigned int *register_values, int max)
 		unsigned where;
 		unsigned long reg;
 #if 0
-	#if CONFIG_USE_INIT
+	#if CONFIG_USE_PRINTK_IN_CAR
 		prink_debug("%08x <- %08x\r\n", register_values[i], register_values[i+2]);
 	#else
 		print_debug_hex32(register_values[i]);
@@ -66,7 +69,11 @@ static int controller_present(const struct mem_controller *ctrl)
         return pci_read_config32(ctrl->f0, 0) == 0x11001022;
 }
 
+#if RAMINIT_SYSINFO==1
+static void sdram_set_registers(const struct mem_controller *ctrl, struct sys_info *sysinfo)
+#else
 static void sdram_set_registers(const struct mem_controller *ctrl)
+#endif
 {
 	static const unsigned int register_values[] = {
 
@@ -546,7 +553,7 @@ static void sdram_set_registers(const struct mem_controller *ctrl)
 		unsigned where;
 		unsigned long reg;
 #if 0
-        #if CONFIG_USE_INIT
+	#if CONFIG_USE_PRINTK_IN_CAR
                 prink_debug("%08x <- %08x\r\n", register_values[i], register_values[i+2]);
         #else
 		print_spew_hex32(register_values[i]);
@@ -2139,7 +2146,11 @@ static long spd_set_dram_timing(const struct mem_controller *ctrl, const struct
 	return dimm_mask;
 }
 
+#if RAMINIT_SYSINFO==1
+static void sdram_set_spd_registers(const struct mem_controller *ctrl, struct sys_info *sysinfo) 
+#else
 static void sdram_set_spd_registers(const struct mem_controller *ctrl) 
+#endif
 {
 	struct spd_set_memclk_result result;
 	const struct mem_param *param;
@@ -2288,7 +2299,11 @@ static void set_hw_mem_hole(int controllers, const struct mem_controller *ctrl)
 #endif
 
 #define TIMEOUT_LOOPS 300000
+#if RAMINIT_SYSINFO == 1
+static void sdram_enable(int controllers, const struct mem_controller *ctrl, struct sys_info *sysinfo)
+#else
 static void sdram_enable(int controllers, const struct mem_controller *ctrl)
+#endif
 {
 	int i;
 
@@ -2476,3 +2491,5 @@ static void fill_mem_ctrl(int controllers, struct mem_controller *ctrl_a, const
         }
 }
 #endif
+
+#endif
diff --git a/src/northbridge/amd/amdk8/raminit_f.c b/src/northbridge/amd/amdk8/raminit_f.c
index 3c8341050e..de1a52499d 100644
--- a/src/northbridge/amd/amdk8/raminit_f.c
+++ b/src/northbridge/amd/amdk8/raminit_f.c
@@ -20,8 +20,8 @@
 
 static inline void print_raminit(const char *strval, uint32_t val)
 {
-#if CONFIG_USE_INIT
-        printk_debug("%s:%08x\r\n", strval, val);
+#if CONFIG_USE_PRINTK_IN_CAR
+        printk_debug("%s%08x\r\n", strval, val);
 #else
         print_debug(strval); print_debug_hex32(val); print_debug("\r\n");
 #endif
@@ -3005,8 +3005,22 @@ static void sdram_enable(int controllers, const struct mem_controller *ctrl, str
 
         for(i = 0; i < controllers; i++) {
                 sysinfo->mem_trained[i] = 0;
+
+                if (!sysinfo->ctrl_present[ i ])
+                        continue;
+
+                /* Skip everything if I don't have any memory on this controller */
+                if(sysinfo->meminfo[i].dimm_mask==0x00) 
+			continue;
+
+                sysinfo->mem_trained[i] = 0x80; // mem need to be trained
         }
 
+#if 0
+        dump_pci_devices();
+        dump_pci_device_index_wait(PCI_DEV(0, 0x18, 2), 0x98);
+#endif
+
 #if MEM_TRAIN_SEQ ==  0
    #if K8_REV_F_SUPPORT_F0_F1_WORKAROUND == 1
 	dqs_timing(controllers, ctrl, tsc0, sysinfo);
@@ -3021,13 +3035,11 @@ static void sdram_enable(int controllers, const struct mem_controller *ctrl, str
    #endif
 
         for(i = 0; i < controllers; i++) {
-                if (!sysinfo->ctrl_present[ i ])
-                        continue;
-
                 /* Skip everything if I don't have any memory on this controller */
-                if(sysinfo->meminfo[i].dimm_mask==0x00) continue;
+                if(sysinfo->mem_trained[i]!=0x80) 
+			continue;
 
-                dqs_timing(i, ctrl, sysinfo, 1);
+                dqs_timing(i, &ctrl[i], sysinfo, 1);
 
    #if MEM_TRAIN_SEQ == 1
                 break; // only train the first node with ram
@@ -3040,6 +3052,14 @@ static void sdram_enable(int controllers, const struct mem_controller *ctrl, str
 
 #endif
 
+#if MEM_TRAIN_SEQ != 1
+	wait_all_core0_mem_trained(sysinfo);
+#endif
+
+#if 0
+        dump_pci_devices();
+        dump_pci_device_index_wait(PCI_DEV(0, 0x18, 2), 0x98);
+#endif
 
 }
 static void fill_mem_ctrl(int controllers, struct mem_controller *ctrl_a, const uint16_t *spd_addr)
diff --git a/src/northbridge/amd/amdk8/raminit_f_dqs.c b/src/northbridge/amd/amdk8/raminit_f_dqs.c
index f77b6d6c86..9d712a7b15 100644
--- a/src/northbridge/amd/amdk8/raminit_f_dqs.c
+++ b/src/northbridge/amd/amdk8/raminit_f_dqs.c
@@ -8,7 +8,7 @@ static inline void print_debug_dqs(const char *str, unsigned val, unsigned level
 {
 #if DQS_TRAIN_DEBUG > 0
 	if(DQS_TRAIN_DEBUG > level) {
-		#if CONFIG_USE_INIT == 1
+		#if CONFIG_USE_PRINTK_IN_CAR
 		printk_debug("%s%x\r\n", str, val);
 		#else
 	        print_debug(str); print_debug_hex32(val); print_debug("\r\n");
@@ -21,7 +21,7 @@ static inline void print_debug_dqs_pair(const char *str, unsigned val, const cha
 {
 #if DQS_TRAIN_DEBUG > 0
         if(DQS_TRAIN_DEBUG > level) {
-                #if CONFIG_USE_INIT == 1
+		#if CONFIG_USE_PRINTK_IN_CAR
                 printk_debug("%s%08x%s%08x\r\n", str, val, str2, val2);
                 #else
                 print_debug(str); print_debug_hex32(val); print_debug(str2); print_debug_hex32(val2); print_debug("\r\n");
@@ -34,7 +34,7 @@ static inline void print_debug_dqs_tsc(const char *str, unsigned i, unsigned val
 {
 #if DQS_TRAIN_DEBUG > 0
         if(DQS_TRAIN_DEBUG > level) {
-                #if CONFIG_USE_INIT == 1
+		#if CONFIG_USE_PRINTK_IN_CAR
                 printk_debug("%s[%02x]=%08x%08x\r\n", str, i, val, val2);
                 #else
 		print_debug(str); print_debug("["); print_debug_hex8(i); print_debug("]="); print_debug_hex32(val); print_debug_hex32(val2); print_debug("\r\n");
@@ -45,7 +45,7 @@ static inline void print_debug_dqs_tsc(const char *str, unsigned i, unsigned val
 
 static inline void print_debug_dqs_tsc_x(const char *str, unsigned i, unsigned val, unsigned val2)
 {
-	#if CONFIG_USE_INIT == 1
+	#if CONFIG_USE_PRINTK_IN_CAR
         printk_debug("%s[%02x]=%08x%08x\r\n", str, i, val, val2);
         #else
         print_debug(str); print_debug("["); print_debug_hex8(i); print_debug("]="); print_debug_hex32(val); print_debug_hex32(val2); print_debug("\r\n");
@@ -501,7 +501,7 @@ static void InitDQSPos4RcvrEn(const struct mem_controller *ctrl)
 #define K8_REV_F_SUPPORT_F0_F1_WORKAROUND 1
 #endif
 
-static void TrainRcvrEn(const struct mem_controller *ctrl, unsigned Pass, struct sys_info *sysinfo)
+static unsigned TrainRcvrEn(const struct mem_controller *ctrl, unsigned Pass, struct sys_info *sysinfo)
 {
 
 	const static uint32_t TestPattern0[] = {
@@ -876,16 +876,14 @@ static void TrainRcvrEn(const struct mem_controller *ctrl, unsigned Pass, struct
 
 #if MEM_TRAIN_SEQ != 1  
 	/* We need tidy output for type 1 */
-	#if CONFIG_USE_INIT == 1
+	#if CONFIG_USE_PRINTK_IN_CAR
 	printk_debug(" CTLRMaxDelay=%02x", CTLRMaxDelay);
 	#else
 	print_debug(" CTLRMaxDelay="); print_debug_hex8(CTLRMaxDelay); 
 	#endif
 #endif
 
-	if(CTLRMaxDelay==0xae) {
-		soft_reset(); // try more or downgrade?
-	}
+	return (CTLRMaxDelay==0xae)?1:0;
 
 }
 
@@ -1544,24 +1542,28 @@ static void SetEccDQSRdWrPos(const struct mem_controller *ctrl, struct sys_info
 	}
 }
 
-static void train_DqsRcvrEn(const struct mem_controller *ctrl, unsigned Pass, struct sys_info *sysinfo)
+static unsigned train_DqsRcvrEn(const struct mem_controller *ctrl, unsigned Pass, struct sys_info *sysinfo)
 {
 	print_debug_dqs("\r\ntrain_DqsRcvrEn: begin ctrl ", ctrl->node_id, 0); 
-	TrainRcvrEn(ctrl, Pass, sysinfo);
+	if(TrainRcvrEn(ctrl, Pass, sysinfo)) {
+		return 1;
+	}
 	print_debug_dqs("\r\ntrain_DqsRcvrEn: end ctrl ", ctrl->node_id, 0); 
+	return 0;
 	
 }
-static  void train_DqsPos(const struct mem_controller *ctrl, struct sys_info *sysinfo)
+static unsigned train_DqsPos(const struct mem_controller *ctrl, struct sys_info *sysinfo)
 {
 	print_debug_dqs("\r\ntrain_DqsPos: begin ctrl ", ctrl->node_id, 0); 
 	if(TrainDQSRdWrPos(ctrl, sysinfo) != 0) {
                 print_err("\r\nDQS Training Rd Wr failed ctrl"); print_err_hex8(ctrl->node_id); print_err("\r\n");
-		soft_reset();
+		return 1;
 	}
 	else {
 		SetEccDQSRdWrPos(ctrl, sysinfo);
 	}
 	print_debug_dqs("\r\ntrain_DqsPos: end ctrl ", ctrl->node_id, 0); 
+	return 0;
 	
 }
 
@@ -1717,7 +1719,7 @@ static unsigned int range_to_mtrr(unsigned int reg,
                 }
                 sizek = 1 << align;
 #if MEM_TRAIN_SEQ != 1
-	#if CONFIG_USE_INIT == 1
+	#if CONFIG_USE_PRINTK_IN_CAR
                 printk_debug("Setting variable MTRR %d, base: %4dMB, range: %4dMB, type %s\r\n",
                         reg, range_startk >>10, sizek >> 10,
                         (type==MTRR_TYPE_UNCACHEABLE)?"UC":
@@ -1880,7 +1882,7 @@ static void dqs_timing(int controllers, const struct mem_controller *ctrl, struc
 
                 print_debug("DQS Training:RcvrEn:Pass1: ");
                 print_debug_hex8(i);
-                train_DqsRcvrEn(ctrl+i, 1, sysinfo);
+                if(train_DqsRcvrEn(ctrl+i, 1, sysinfo)) goto out;
        	        print_debug(" done\r\n");
         }
 
@@ -1899,7 +1901,7 @@ static void dqs_timing(int controllers, const struct mem_controller *ctrl, struc
 
                 print_debug("DQS Training:DQSPos: ");
                 print_debug_hex8(i);
-                train_DqsPos(ctrl+i, sysinfo);
+                if(train_DqsPos(ctrl+i, sysinfo)) goto out;
                 print_debug(" done\r\n");
         }
 
@@ -1913,11 +1915,12 @@ static void dqs_timing(int controllers, const struct mem_controller *ctrl, struc
 
                 print_debug("DQS Training:RcvrEn:Pass2: ");
                 print_debug_hex8(i);
-                train_DqsRcvrEn(ctrl+i, 2, sysinfo);
+                if(train_DqsRcvrEn(ctrl+i, 2, sysinfo)) goto out;
                 print_debug(" done\r\n");
 		sysinfo->mem_trained[i]=1;
         }
 
+out:
 	tsc[4] = rdtsc();
 	clear_mtrr_dqs(sysinfo->tom2_k);
 
@@ -1942,14 +1945,14 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
 
          tsc_t tsc[4];
 
+	if(sysinfo->mem_trained[i] != 0x80) return;
 
 #if MEM_TRAIN_SEQ == 1
-	if(sysinfo->mem_trained[i]) return;
         //need to enable mtrr, so dqs training could access the test address
         setup_mtrr_dqs(sysinfo->tom_k, sysinfo->tom2_k);
 #endif
 
-	fill_mem_cs_sysinfo(i, ctrl+i, sysinfo);
+	fill_mem_cs_sysinfo(i, ctrl, sysinfo);
 
 	if(v) {
 	        tsc[0] = rdtsc();
@@ -1957,7 +1960,10 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
 	        print_debug("set DQS timing:RcvrEn:Pass1: ");
 	        print_debug_hex8(i);
 	}
-        train_DqsRcvrEn(ctrl+i, 1,  sysinfo);
+        if(train_DqsRcvrEn(ctrl, 1,  sysinfo)) {
+		sysinfo->mem_trained[i]=0x81; //
+		goto out;
+	}
 
 	if(v) {
 	        print_debug(" done\r\n");
@@ -1966,7 +1972,10 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
 	        print_debug_hex8(i);
 	}
 
-        train_DqsPos(ctrl+i, sysinfo);
+        if(train_DqsPos(ctrl, sysinfo)) {
+		sysinfo->mem_trained[i]=0x82; //
+		goto out;
+	}
 	
 	if(v) {
 	        print_debug(" done\r\n");
@@ -1975,7 +1984,10 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
 	        print_debug("set DQS timing:RcvrEn:Pass2: ");
 	        print_debug_hex8(i);
 	}
-        train_DqsRcvrEn(ctrl+i, 2,  sysinfo);
+        if(train_DqsRcvrEn(ctrl, 2,  sysinfo)){
+		sysinfo->mem_trained[i]=0x83; //
+		goto out;
+	}
 
 	if(v) {
 	        print_debug(" done\r\n");
@@ -1983,6 +1995,7 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
 	        tsc[3] = rdtsc();
 	}
 
+out:
 #if MEM_TRAIN_SEQ == 1
         clear_mtrr_dqs(sysinfo->tom2_k);
 #endif
@@ -1992,8 +2005,10 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
         	      print_debug_dqs_tsc_x("Total DQS Training : tsc ", ii,  tsc[ii].hi, tsc[ii].lo);
 	        }
 	}
-
-	sysinfo->mem_trained[i]=1;
+	
+	if(sysinfo->mem_trained[i] == 0x80) {
+		sysinfo->mem_trained[i]=1;
+	}
 
 }
 #endif
@@ -2001,7 +2016,7 @@ static void dqs_timing(int i, const struct mem_controller *ctrl, struct sys_info
 #if MEM_TRAIN_SEQ == 1
 static void train_ram(unsigned nodeid, struct sys_info *sysinfo, struct sys_info *sysinfox)
 {
-	dqs_timing(nodeid, sysinfo->ctrl,sysinfo, 0); // keep the output tidy
+	dqs_timing(nodeid, &sysinfo->ctrl[nodeid], sysinfo, 0); // keep the output tidy
 //      memcpy(&sysinfox->dqs_rcvr_dly_a[nodeid * 2 * 8],&sysinfo->dqs_rcvr_dly_a[nodeid * 2 * 8], 2*8);
 //      memcpy(&sysinfox->dqs_delay_a[nodeid * 2 * 2 * 9], &sysinfo->dqs_delay_a[nodeid * 2 * 2 * 9], 2 * 2 * 9);
 	sysinfox->mem_trained[nodeid] = sysinfo->mem_trained[nodeid];
@@ -2014,23 +2029,23 @@ static inline void train_ram_on_node(unsigned nodeid, unsigned coreid, struct sy
 	struct sys_info *sysinfox = ((CONFIG_LB_MEM_TOPK<<10) - DCACHE_RAM_GLOBAL_VAR_SIZE);
 	wait_till_sysinfo_in_ram(); // use pci to get it
 
-	if(sysinfox->mem_trained[nodeid] == 0) {
-		if (sysinfox->ctrl_present[ nodeid ] &&  sysinfox->meminfo[nodeid].dimm_mask) {
-			sysinfo->tom_k = sysinfox->tom_k;
-			sysinfo->tom2_k = sysinfox->tom2_k;
-			sysinfo->meminfo[nodeid].is_Width128 = sysinfox->meminfo[nodeid].is_Width128;
-			set_top_mem_ap(sysinfo->tom_k, sysinfo->tom2_k); // keep the ap's tom consistent with bsp's
-		#if CONFIG_AP_CODE_IN_CAR == 0
-			print_debug("CODE IN ROM AND RUN ON NODE:"); print_debug_hex8(nodeid); print_debug("\r\n");
-			train_ram(nodeid, sysinfo, sysinfox);
-		#else
-			/* Can copy dqs_timing to ap cache and run from cache?
-			* we need linuxbios_ap_car.rom? and treat it as linuxbios_ram.rom for ap ?
-			*/
-			copy_and_run_ap_code_in_car(retcall);
-			// will go back by jump
-		#endif
-		}
+	if(sysinfox->mem_trained[nodeid] == 0x80) {
+		sysinfo->tom_k = sysinfox->tom_k;
+		sysinfo->tom2_k = sysinfox->tom2_k;
+		sysinfo->meminfo[nodeid].is_Width128 = sysinfox->meminfo[nodeid].is_Width128;
+		sysinfo->mem_trained[nodeid] = sysinfox->mem_trained[nodeid];
+		memcpy(&sysinfo->ctrl[nodeid], &sysinfox->ctrl[nodeid], sizeof(struct mem_controller));
+		set_top_mem_ap(sysinfo->tom_k, sysinfo->tom2_k); // keep the ap's tom consistent with bsp's
+	#if CONFIG_AP_CODE_IN_CAR == 0
+		print_debug("CODE IN ROM AND RUN ON NODE:"); print_debug_hex8(nodeid); print_debug("\r\n");
+		train_ram(nodeid, sysinfo, sysinfox);
+	#else
+		/* Can copy dqs_timing to ap cache and run from cache?
+		* we need linuxbios_ap_car.rom? and treat it as linuxbios_ram.rom for ap ?
+		*/
+		copy_and_run_ap_code_in_car(retcall);
+		// will go back by jump
+	#endif
 	}
 }
 #endif
diff --git a/src/northbridge/amd/amdk8/setup_resource_map.c b/src/northbridge/amd/amdk8/setup_resource_map.c
index 6b710334e4..b64b4ee5e5 100644
--- a/src/northbridge/amd/amdk8/setup_resource_map.c
+++ b/src/northbridge/amd/amdk8/setup_resource_map.c
@@ -12,7 +12,7 @@ static void setup_resource_map_offset(const unsigned int *register_values, int m
                 unsigned where;
                 unsigned long reg;
 #if 0
-        #if CONFIG_USE_INIT
+	#if CONFIG_USE_PRINTK_IN_CAR
                 prink_debug("%08x <- %08x\r\n", register_values[i] +  offset_pci_dev, register_values[i+2]);
         #else
                 print_debug_hex32(register_values[i] + offset_pci_dev);
@@ -56,7 +56,7 @@ static void setup_resource_map_x_offset(const unsigned int *register_values, int
 #endif
 	for(i = 0; i < max; i += 4) {
 #if RES_DEBUG
-	#if CONFIG_USE_INIT
+	#if CONFIG_USE_PRINTK_IN_CAR
                 printk_debug("%04x: %02x %08x <- & %08x | %08x\r\n", 
 			i>>2, register_values[i], 
 			register_values[i+1] + ( (register_values[i]==RES_PCI_IO) ? offset_pci_dev : 0), 
@@ -151,7 +151,7 @@ static void setup_resource_map_x(const unsigned int *register_values, int max)
 #endif
         for(i = 0; i < max; i += 4) {
 #if RES_DEBUG
-        #if CONFIG_USE_INIT
+	#if CONFIG_USE_PRINTK_IN_CAR
                 printk_debug("%04x: %02x %08x <- & %08x | %08x\r\n",
                         i/4, register_values[i],register_values[i+1], register_values[i+2], register_values[i+3]);
         #else