Synchronize rdtsc instructions

The CPU can arbitrarily reorder calls to rdtsc, significantly
reducing the precision of timing using the CPUs time stamp counter.
Unfortunately the method of synchronizing rdtsc is different
on AMD and Intel CPUs. There is a generic method, using the cpuid
instruction, but that uses up a lot of registers, and is very slow.
Hence, use the correct lfence/mfence instructions (for CPUs that
we know support it)

Change-Id: I17ecb48d283f38f23148c13159aceda704c64ea5
Signed-off-by: Stefan Reinauer <reinauer@google.com>
Reviewed-on: http://review.coreboot.org/1422
Reviewed-by: Alexandru Gagniuc <mr.nuke.me@gmail.com>
Tested-by: build bot (Jenkins)

10 files changed, 40 insertions, 4 deletions

diff --git a/src/cpu/amd/agesa/Kconfig b/src/cpu/amd/agesa/Kconfig
index 5ec5ce8b1d..b7e69e015a 100644
--- a/src/cpu/amd/agesa/Kconfig
+++ b/src/cpu/amd/agesa/Kconfig
@@ -25,6 +25,7 @@ config CPU_AMD_AGESA
         default y if CPU_AMD_AGESA_FAMILY15
         default y if CPU_AMD_AGESA_FAMILY15_TN
         default n
+	select TSC_SYNC_LFENCE
 
 if CPU_AMD_AGESA
 
diff --git a/src/cpu/amd/model_10xxx/Kconfig b/src/cpu/amd/model_10xxx/Kconfig
index 221d0449ac..0890771e34 100644
--- a/src/cpu/amd/model_10xxx/Kconfig
+++ b/src/cpu/amd/model_10xxx/Kconfig
@@ -3,6 +3,7 @@ config CPU_AMD_MODEL_10XXX
 	select SSE
 	select SSE2
 	select MMCONF_SUPPORT_DEFAULT
+	select TSC_SYNC_LFENCE
 
 if CPU_AMD_MODEL_10XXX
 config CPU_ADDR_BITS
diff --git a/src/cpu/amd/model_fxx/Kconfig b/src/cpu/amd/model_fxx/Kconfig
index e04605bfbe..0afc4b0f4c 100644
--- a/src/cpu/amd/model_fxx/Kconfig
+++ b/src/cpu/amd/model_fxx/Kconfig
@@ -3,6 +3,7 @@ config CPU_AMD_MODEL_FXX
 	select MMX
 	select SSE
 	select SSE2
+	select TSC_SYNC_LFENCE
 
 if CPU_AMD_MODEL_FXX
 config UDELAY_IO
diff --git a/src/cpu/intel/model_1067x/Kconfig b/src/cpu/intel/model_1067x/Kconfig
index b079922275..852c9cdf0c 100644
--- a/src/cpu/intel/model_1067x/Kconfig
+++ b/src/cpu/intel/model_1067x/Kconfig
@@ -2,3 +2,4 @@ config CPU_INTEL_MODEL_1067X
 	bool
 	select SMP
 	select SSE2
+	select TSC_SYNC_MFENCE
diff --git a/src/cpu/intel/model_106cx/Kconfig b/src/cpu/intel/model_106cx/Kconfig
index 103ed50d3e..7a75ec1ba3 100644
--- a/src/cpu/intel/model_106cx/Kconfig
+++ b/src/cpu/intel/model_106cx/Kconfig
@@ -4,9 +4,9 @@ config CPU_INTEL_MODEL_106CX
 	select SSE2
 	select UDELAY_LAPIC
 	select AP_IN_SIPI_WAIT
+	select TSC_SYNC_MFENCE
 
 config CPU_ADDR_BITS
 	int
 	default 32
 
-
diff --git a/src/cpu/intel/model_206ax/Kconfig b/src/cpu/intel/model_206ax/Kconfig
index 9cc6edd94e..6635868453 100644
--- a/src/cpu/intel/model_206ax/Kconfig
+++ b/src/cpu/intel/model_206ax/Kconfig
@@ -14,6 +14,7 @@ config CPU_SPECIFIC_OPTIONS
 	select SMM_TSEG
 	select MICROCODE_IN_CBFS
 	#select AP_IN_SIPI_WAIT
+	select TSC_SYNC_MFENCE
 
 config BOOTBLOCK_CPU_INIT
 	string
diff --git a/src/cpu/intel/model_6ex/Kconfig b/src/cpu/intel/model_6ex/Kconfig
index 31d24bd68f..e2b1986132 100644
--- a/src/cpu/intel/model_6ex/Kconfig
+++ b/src/cpu/intel/model_6ex/Kconfig
@@ -4,3 +4,4 @@ config CPU_INTEL_MODEL_6EX
 	select SSE2
 	select UDELAY_LAPIC
 	select AP_IN_SIPI_WAIT
+	select TSC_SYNC_MFENCE
diff --git a/src/cpu/intel/model_6fx/Kconfig b/src/cpu/intel/model_6fx/Kconfig
index 851685cb06..4517f17fa6 100644
--- a/src/cpu/intel/model_6fx/Kconfig
+++ b/src/cpu/intel/model_6fx/Kconfig
@@ -4,3 +4,4 @@ config CPU_INTEL_MODEL_6FX
 	select SSE2
 	select UDELAY_LAPIC
 	select AP_IN_SIPI_WAIT
+	select TSC_SYNC_MFENCE
diff --git a/src/cpu/x86/Kconfig b/src/cpu/x86/Kconfig
index 07e9d9ecbf..9a96aeadd8 100644
--- a/src/cpu/x86/Kconfig
+++ b/src/cpu/x86/Kconfig
@@ -23,6 +23,22 @@ config TSC_CALIBRATE_WITH_IO
 	bool
 	default n
 
+config TSC_SYNC_LFENCE
+	bool
+	default n
+	help
+	  The CPU driver should select this if the CPU needs
+	  to execute an lfence instruction in order to synchronize
+	  rdtsc. This is true for all modern AMD CPUs.
+
+config TSC_SYNC_MFENCE
+	bool
+	default n
+	help
+	  The CPU driver should select this if the CPU needs
+	  to execute an mfence instruction in order to synchronize
+	  rdtsc. This is true for all modern Intel CPUs.
+
 config XIP_ROM_SIZE
 	hex
 	default ROM_SIZE if ROMCC
diff --git a/src/include/cpu/x86/tsc.h b/src/include/cpu/x86/tsc.h
index c57362755a..6ce7f5fc7d 100644
--- a/src/include/cpu/x86/tsc.h
+++ b/src/include/cpu/x86/tsc.h
@@ -1,6 +1,14 @@
 #ifndef CPU_X86_TSC_H
 #define CPU_X86_TSC_H
 
+#if CONFIG_TSC_SYNC_MFENCE
+#define TSC_SYNC "mfence\n"
+#elif CONFIG_TSC_SYNC_LFENCE
+#define TSC_SYNC "lfence\n"
+#else
+#define TSC_SYNC
+#endif
+
 struct tsc_struct {
 	unsigned lo;
 	unsigned hi;
@@ -10,10 +18,11 @@ typedef struct tsc_struct tsc_t;
 static inline tsc_t rdtsc(void)
 {
 	tsc_t res;
-	__asm__ __volatile__ (
+	asm volatile (
+		TSC_SYNC
 		"rdtsc"
 		: "=a" (res.lo), "=d"(res.hi) /* outputs */
-		);
+	);
 	return res;
 }
 
@@ -22,7 +31,11 @@ static inline tsc_t rdtsc(void)
 static inline unsigned long long rdtscll(void)
 {
 	unsigned long long val;
-	asm volatile ("rdtsc" : "=A" (val));
+	asm volatile (
+		TSC_SYNC
+		"rdtsc"
+		: "=A" (val)
+	);
 	return val;
 }
 #endif