Intel e7505: enable ECC scrubbing

It takes about 3 seconds to scrub 8GiB DDR266 RAM.

After ECC scrub XIP cache is disabled for system stability. There is
very little to do in romstage after ECC scrub, especially when RAM
debug messages are turned off. So the delay caused by this is hardly
noticeable.

Cache for complete ROM is re-enabled before ramstage is decompressed,
and it has no unstability issues. So the code required to re-enable
cache for ROM currently already exists in cache-as-ram_ht.inc.

A Kconfig option HW_SCRUBBER enables the scrub to be run on hard
reboots and power-ons.

Change-Id: Icf27acf73240c06b58091f1229efc0f01cca3f85
Signed-off-by: Kyösti Mälkki <kyosti.malkki@gmail.com>
Reviewed-on: http://review.coreboot.org/905
Tested-by: build bot (Jenkins)
Reviewed-by: Patrick Georgi <patrick@georgi-clan.de>

5 files changed, 160 insertions, 41 deletions

diff --git a/src/mainboard/aopen/dxplplusu/Kconfig b/src/mainboard/aopen/dxplplusu/Kconfig
index da03491d87..b6fbf45745 100644
--- a/src/mainboard/aopen/dxplplusu/Kconfig
+++ b/src/mainboard/aopen/dxplplusu/Kconfig
@@ -13,6 +13,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	select UDELAY_TSC
 	select HAVE_ACPI_TABLES
 	select BOARD_ROMSIZE_KB_512
+	select HW_SCRUBBER
 
 config MAINBOARD_DIR
 	string
diff --git a/src/mainboard/aopen/dxplplusu/romstage.c b/src/mainboard/aopen/dxplplusu/romstage.c
index 73e445b7ce..ee900e9e12 100644
--- a/src/mainboard/aopen/dxplplusu/romstage.c
+++ b/src/mainboard/aopen/dxplplusu/romstage.c
@@ -68,8 +68,23 @@ void main(unsigned long bist)
 	// If this is a warm boot, some initialization can be skipped
 	if (!bios_reset_detected()) {
 		enable_smbus();
-		sdram_initialize(ARRAY_SIZE(memctrl), memctrl);
+
+		/* The real MCH initialisation. */
+		e7505_mch_init(memctrl);
+
+		/*
+		 * ECC scrub invalidates cache, so all stack in CAR
+		 * is lost. Only return addresses from main() and
+		 * scrub_ecc() are recovered to stack via xmm0-xmm3.
+		 */
+#if CONFIG_HW_SCRUBBER
+		unsigned long ret_addr = (unsigned long)((unsigned long*)&bist - 1);
+		e7505_mch_scrub_ecc(ret_addr);
+#endif
+
+		/* Hook for post ECC scrub settings and debug. */
+		e7505_mch_done(memctrl);
 	}
 
-	print_debug("SDRAM is up.\n");
+	printk(BIOS_DEBUG, "SDRAM is up.\n");
 }
diff --git a/src/northbridge/intel/e7505/Kconfig b/src/northbridge/intel/e7505/Kconfig
index 578ab866d8..8ef1a36013 100644
--- a/src/northbridge/intel/e7505/Kconfig
+++ b/src/northbridge/intel/e7505/Kconfig
@@ -1,4 +1,33 @@
+##
+## This file is part of the coreboot project.
+##
+## Copyright (C) 2007-2012 coresystems GmbH
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; version 2 of the License.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+##
+
 config NORTHBRIDGE_INTEL_E7505
 	bool
+
+if NORTHBRIDGE_INTEL_E7505
+
+config NORTHBRIDGE_SPECIFIC_OPTIONS # dummy
+	def_bool y
 	select HAVE_DEBUG_RAM_SETUP
 
+config HW_SCRUBBER
+	bool
+	default n
+
+endif
diff --git a/src/northbridge/intel/e7505/raminit.c b/src/northbridge/intel/e7505/raminit.c
index b06f81d3d6..1d10a8cd73 100644
--- a/src/northbridge/intel/e7505/raminit.c
+++ b/src/northbridge/intel/e7505/raminit.c
@@ -11,6 +11,9 @@
 
 /* converted to C 6/2004 yhlu */
 
+#include <cpu/x86/mtrr.h>
+#include <cpu/x86/cache.h>
+#include <cpu/x86/msr.h>
 #include <assert.h>
 #include <spd.h>
 #include <sdram_mode.h>
@@ -918,44 +921,93 @@ static void configure_e7501_ram_addresses(const struct mem_controller
 }
 
 /**
- * If we're configured to use ECC, initialize the SDRAM and clear the E7501's
- * ECC error flags.
+ * Execute ECC full-speed scrub once and leave scrubber disabled.
+ *
+ * NOTE: All cache and stack is lost during ECC scrub loop.
  */
-#ifdef __ROMCC__
-static void initialize_ecc(void)
+static void __attribute__((always_inline))
+		initialize_ecc(unsigned long ret_addr, unsigned long ret_addr2)
 {
-	uint32_t dram_controller_mode;
-
-	/* Test to see if ECC support is enabled */
-	dram_controller_mode = pci_read_config32(MCHDEV, DRC);
-	dram_controller_mode >>= 20;
-	dram_controller_mode &= 3;
-	if (dram_controller_mode == 2) {
-		uint8_t byte;
+	uint16_t scrubbed = pci_read_config16(MCHDEV, MCHCFGNS) & 0x08;
 
+	if (!scrubbed) {
 		RAM_DEBUG_MESSAGE("Initializing ECC state...\n");
-		pci_write_config8(MCHDEV, MCHCFGNS, 0x01);
 
-		// Wait for scrub cycle to complete
+		/* ECC scrub flushes cache-lines and stack, need to
+		 * store return address from romstage.c:main().
+		 */
+		asm volatile(
+			"movd %0, %%xmm0;"
+			"movd (%0), %%xmm1;"
+			"movd %1, %%xmm2;"
+			"movd (%1), %%xmm3;"
+			:: "r" (ret_addr), "r" (ret_addr2) :
+		);
+
+		/* NOTE: All cache is lost during this loop.
+		 * Make sure PCI access does not use stack.
+		 */
+
+		pci_write_config16(MCHDEV, MCHCFGNS, 0x01);
 		do {
-			byte =
-			    pci_read_config8(MCHDEV, MCHCFGNS);
-		} while ((byte & 0x08) == 0);
+			scrubbed = pci_read_config16(MCHDEV, MCHCFGNS);
+		} while (! (scrubbed & 0x08));
+		pci_write_config16(MCHDEV, MCHCFGNS, (scrubbed & ~0x07) | 0x04);
+
+		/* Some problem remains with XIP cache from ROM, so for
+		 * now, I disable XIP and also invalidate cache (again)
+		 * before the remaining small portion of romstage.
+		 *
+		 * Adding NOPs here has unexpected results, making
+		 * the first do_printk()/vtxprintf() after ECC scrub
+		 * fail midway. Sometimes vtxprintf() dumps strings
+		 * completely but with every 4th (fourth) character as "/".
+		 *
+		 * An inlined dump to console of the same string,
+		 * before vtxprintf() call, is successful. So the
+		 * source string should be completely in cache already.
+		 *
+		 * I need to review this again with CPU microcode
+		 * update applied pre-CAR.
+		 */
+
+		/* Disable and invalidate all cache. */
+		msr_t xip_mtrr = rdmsr(MTRRphysMask_MSR(1));
+		xip_mtrr.lo &= ~MTRRphysMaskValid;
+		invd();
+		wrmsr(MTRRphysMask_MSR(1), xip_mtrr);
+		invd();
 
-		pci_write_config8(MCHDEV, MCHCFGNS, (byte & 0xfc) | 0x04);
 		RAM_DEBUG_MESSAGE("ECC state initialized.\n");
 
-		/* Clear the ECC error bits */
-		pci_write_config8(RASDEV, DRAM_FERR, 0x03);
-		pci_write_config8(RASDEV, DRAM_NERR, 0x03);
+		/* Recover IP for return from main. */
+		asm volatile(
+			"movd %%xmm0, %%edi;"
+			"movd %%xmm1, (%%edi);"
+			"movd %%xmm2, %%edi;"
+			"movd %%xmm3, (%%edi);"
+			 ::: "edi"
+		);
 
-		// Clear DRAM Interface error bits (write-one-clear)
-		pci_write_config32(RASDEV, FERR_GLOBAL, 1 << 18);
-		pci_write_config32(RASDEV, NERR_GLOBAL, 1 << 18);
+#if CONFIG_DEBUG_RAM_SETUP
+		unsigned int a1, a2;
+		asm volatile("movd %%xmm2, %%eax;" : "=a" (a1) ::);
+		asm volatile("movd %%xmm3, %%eax;" : "=a" (a2) ::);
+		printk(BIOS_DEBUG, "return EIP @ %x = %x\n", a1, a2);
+		asm volatile("movd %%xmm0, %%eax;" : "=a" (a1) ::);
+		asm volatile("movd %%xmm1, %%eax;" : "=a" (a2) ::);
+		printk(BIOS_DEBUG, "return EIP @ %x = %x\n", a1, a2);
+#endif
 	}
 
+	/* Clear the ECC error bits. */
+	pci_write_config8(RASDEV, DRAM_FERR, 0x03);
+	pci_write_config8(RASDEV, DRAM_NERR, 0x03);
+
+	/* Clear DRAM Interface error bits. */
+	pci_write_config32(RASDEV, FERR_GLOBAL, 1 << 18);
+	pci_write_config32(RASDEV, NERR_GLOBAL, 1 << 18);
 }
-#endif
 
 /**
  * Program the DRAM Timing register (DRT) of the E7501 (except for CAS#
@@ -1669,18 +1721,19 @@ static void sdram_enable(const struct mem_controller *ctrl)
 	dram_controller_mode |= (1 << 29);
 	pci_write_config32(MCHDEV, DRC, dram_controller_mode);
 	EXTRA_DELAY;
+}
 
-#ifdef __ROMCC__
-	/* Problems with cache-as-ram, disable for now */
-	initialize_ecc();
-#endif
-
-	dram_controller_mode = pci_read_config32(MCHDEV, DRC);	/* FCS_EN */
-	dram_controller_mode |= (1 << 17);	// NOTE: undocumented reserved bit
+/**
+ * @param ctrl PCI addresses of memory controller functions, and SMBus
+ *             addresses of DIMM slots on the mainboard.
+ */
+static void sdram_post_ecc(const struct mem_controller *ctrl)
+{
+	/* Fast CS# Enable. */
+	uint32_t dram_controller_mode = pci_read_config32(MCHDEV, DRC);
+	dram_controller_mode = pci_read_config32(MCHDEV, DRC);
+	dram_controller_mode |= (1 << 17);
 	pci_write_config32(MCHDEV, DRC, dram_controller_mode);
-
-	RAM_DEBUG_MESSAGE("Northbridge following SDRAM init:\n");
-	DUMPNORTH();
 }
 
 /**
@@ -1815,7 +1868,7 @@ static void sdram_set_registers(const struct mem_controller *ctrl)
  *
  *
  */
-void sdram_initialize(int controllers, const struct mem_controller *memctrl)
+void e7505_mch_init(const struct mem_controller *memctrl)
 {
 	RAM_DEBUG_MESSAGE("Northbridge prior to SDRAM init:\n");
 	DUMPNORTH();
@@ -1825,6 +1878,27 @@ void sdram_initialize(int controllers, const struct mem_controller *memctrl)
 	sdram_enable(memctrl);
 }
 
+/**
+ * Scrub and reset error counts for ECC dimms.
+ *
+ * NOTE: this will invalidate cache and disable XIP cache for the
+ * short remaining part of romstage.
+ */
+void e7505_mch_scrub_ecc(unsigned long ret_addr)
+{
+	unsigned long ret_addr2 = (unsigned long)((unsigned long*)&ret_addr-1);
+	if ((pci_read_config32(MCHDEV, DRC)>>20 & 3) == 2)
+		initialize_ecc(ret_addr, ret_addr2);
+}
+
+void e7505_mch_done(const struct mem_controller *memctrl)
+{
+	sdram_post_ecc(memctrl);
+
+	RAM_DEBUG_MESSAGE("Northbridge following SDRAM init:\n");
+	DUMPNORTH();
+}
+
 static int bios_reset_detected(void)
 {
 	uint32_t dword = pci_read_config32(MCHDEV, DRC);
diff --git a/src/northbridge/intel/e7505/raminit.h b/src/northbridge/intel/e7505/raminit.h
index df0e9291a3..a38d722d15 100644
--- a/src/northbridge/intel/e7505/raminit.h
+++ b/src/northbridge/intel/e7505/raminit.h
@@ -15,8 +15,8 @@ struct mem_controller {
 	uint16_t channel1[MAX_DIMM_SOCKETS_PER_CHANNEL];
 };
 
-#ifndef __ROMCC__
-void sdram_initialize(int controllers, const struct mem_controller *ctrl);
-#endif
+void e7505_mch_init(const struct mem_controller *memctrl);
+void e7505_mch_scrub_ecc(unsigned long ret_addr);
+void e7505_mch_done(const struct mem_controller *memctrl);
 
 #endif /* RAMINIT_H */