arm: Redesign, clarify and clean up cache related code

This patch changes several cache-related pieces to be cleaner, faster or
more correct. The largest point is removing the old
arm_invalidate_caches() function and surrounding bootblock code to
initialize SCTLR and replace it with an all-assembly function that takes
care of cache and SCTLR initialization to bring the system to a known
state. It runs without stack and before coreboot makes any write
accesses to be as compatible as possible with whatever state the system
was left in by preceeding code. This also finally fixes the dreaded
icache bug that wasted hundreds of milliseconds during boot.

Old-Change-Id: I7bb4995af8184f6383f8e3b1b870b0662bde8bd4
Signed-off-by: Julius Werner <jwerner@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/183890
(cherry picked from commit 07a35925dc957919bf88dfc90515971a36e81b97)

nyan_big: apply cache-related changes from nyan

This applies the same changes from 07a3592 that were applied to nyan.

Old-Change-Id: Idcbe85436d7a2f65fcd751954012eb5f4bec0b6c
Reviewed-on: https://chromium-review.googlesource.com/184551
Commit-Queue: David Hendricks <dhendrix@chromium.org>
Tested-by: David Hendricks <dhendrix@chromium.org>
Reviewed-by: David Hendricks <dhendrix@chromium.org>
(cherry picked from commit 4af27f02614da41c611aee2c6d175b1b948428ea)

Squashed the followup patch for nyan_big into the original patch.

Change-Id: Id14aef7846355ea2da496e55da227b635aca409e
Signed-off-by: Isaac Christensen <isaac.christensen@se-eng.com>
(cherry picked from commit 4cbf25f8eca3a12bbfec5b015953c0fc2b69c877)
Signed-off-by: Marc Jones <marc.jones@se-eng.com>
Reviewed-on: http://review.coreboot.org/6993
Tested-by: build bot (Jenkins)
Reviewed-by: Stefan Reinauer <stefan.reinauer@coreboot.org>

3 files changed, 14 insertions, 85 deletions

diff --git a/payloads/libpayload/arch/arm/cache.c b/payloads/libpayload/arch/arm/cache.c
index defe640654..63715bc5a6 100644
--- a/payloads/libpayload/arch/arm/cache.c
+++ b/payloads/libpayload/arch/arm/cache.c
@@ -38,26 +38,9 @@
 
 void tlb_invalidate_all(void)
 {
-	/*
-	 * FIXME: ARMv7 Architecture Ref. Manual claims that the distinction
-	 * instruction vs. data TLBs is deprecated in ARMv7, however this does
-	 * not seem to be the case as of Cortex-A15.
-	 */
+	/* TLBIALL includes dTLB and iTLB on systems that have them. */
 	tlbiall();
-	dtlbiall();
-	itlbiall();
-	isb();
 	dsb();
-}
-
-void icache_invalidate_all(void)
-{
-	/*
-	 * icache can be entirely invalidated with one operation.
-	 * Note: If branch predictors are architecturally-visible, ICIALLU
-	 * also performs a BPIALL operation (B2-1283 in arch manual)
-	 */
-	iciallu();
 	isb();
 }
 
@@ -135,6 +118,11 @@ void dcache_invalidate_by_mva(void const *addr, size_t len)
 	dcache_op_mva(addr, len, OP_DCIMVAC);
 }
 
+/*
+ * CAUTION: This implementation assumes that coreboot never uses non-identity
+ * page tables for pages containing executed code. If you ever want to violate
+ * this assumption, have fun figuring out the associated problems on your own.
+ */
 void dcache_mmu_disable(void)
 {
 	uint32_t sctlr;
@@ -150,58 +138,14 @@ void dcache_mmu_enable(void)
 	uint32_t sctlr;
 
 	sctlr = read_sctlr();
-	dcache_clean_invalidate_all();
 	sctlr |= SCTLR_C | SCTLR_M;
 	write_sctlr(sctlr);
 }
 
-void arm_invalidate_caches(void)
+void cache_sync_instructions(void)
 {
-	uint32_t clidr;
-	int level;
-
-	/* Invalidate branch predictor */
-	bpiall();
-
-	/* Iterate thru each cache identified in CLIDR and invalidate */
-	clidr = read_clidr();
-	for (level = 0; level < 7; level++) {
-		unsigned int ctype = (clidr >> (level * 3)) & 0x7;
-		uint32_t csselr;
-
-		switch(ctype) {
-		case 0x0:
-			/* no cache */
-			break;
-		case 0x1:
-			/* icache only */
-			csselr = (level << 1) | 1;
-			write_csselr(csselr);
-			icache_invalidate_all();
-			break;
-		case 0x2:
-		case 0x4:
-			/* dcache only or unified cache */
-			csselr = level << 1;
-			write_csselr(csselr);
-			dcache_invalidate_all();
-			break;
-		case 0x3:
-			/* separate icache and dcache */
-			csselr = (level << 1) | 1;
-			write_csselr(csselr);
-			icache_invalidate_all();
-
-			csselr = level << 1;
-			write_csselr(csselr);
-			dcache_invalidate_all();
-			break;
-		default:
-			/* reserved */
-			break;
-		}
-	}
-
-	/* Invalidate TLB */
-	tlb_invalidate_all();
+	dcache_clean_all();	/* includes trailing DSB (in assembly) */
+	iciallu();		/* includes BPIALLU (architecturally) */
+	dsb();
+	isb();
 }
diff --git a/payloads/libpayload/include/arm/arch/cache.h b/payloads/libpayload/include/arm/arch/cache.h
index 5210dfe6a8..470eb55108 100644
--- a/payloads/libpayload/include/arm/arch/cache.h
+++ b/payloads/libpayload/include/arm/arch/cache.h
@@ -93,18 +93,6 @@ static inline void isb(void)
  * Low-level TLB maintenance operations
  */
 
-/* invalidate entire data TLB */
-static inline void dtlbiall(void)
-{
-	asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0) : "memory");
-}
-
-/* invalidate entire instruction TLB */
-static inline void itlbiall(void)
-{
-	asm volatile ("mcr p15, 0, %0, c8, c5, 0" : : "r" (0));
-}
-
 /* invalidate entire unified TLB */
 static inline void tlbiall(void)
 {
@@ -313,8 +301,8 @@ void dcache_mmu_disable(void);
 /* dcache and MMU enable */
 void dcache_mmu_enable(void);
 
-/* icache invalidate all (on current level given by CSSELR) */
-void icache_invalidate_all(void);
+/* perform all icache/dcache maintenance needed after loading new code */
+void cache_sync_instructions(void);
 
 /* tlb invalidate all */
 void tlb_invalidate_all(void);
@@ -323,9 +311,6 @@ void tlb_invalidate_all(void);
  * Generalized setup/init functions
  */
 
-/* invalidate all caches on ARM */
-void arm_invalidate_caches(void);
-
 /* mmu initialization (set page table address, set permissions, etc) */
 void mmu_init(void);
 
diff --git a/payloads/libpayload/include/x86/arch/cache.h b/payloads/libpayload/include/x86/arch/cache.h
index 396048802f..ffefcdbe19 100644
--- a/payloads/libpayload/include/x86/arch/cache.h
+++ b/payloads/libpayload/include/x86/arch/cache.h
@@ -41,6 +41,6 @@
 #define dcache_invalidate_by_mva(addr, len)
 #define dcache_clean_invalidate_all()
 #define dcache_clean_invalidate_by_mva(addr, len)
-#define icache_invalidate_all()
+#define cache_sync_instructions()
 
 #endif