armv7: import updated cache/MMU stuff from coreboot

This imports the newest cache and MMU code from coreboot. This
time it's so new that it hasn't even been checked in to coreboot.

However, this version at least allows DMA to work properly for the
MSHC driver. So even if we rebase a few more times, this version is
at least a step in the right direction.

Note: This omits the stuff that sets up dcache policy since
libpayload should not need to worry about that and it depends
on cbmem stuff.

Change-Id: Idd42b083e8019634aaaa44d5bf5b51db6c3912f5
Signed-off-by: David Hendricks <dhendrix@google.com>
Reviewed-on: http://review.coreboot.org/2975
Reviewed-by: David Hendricks <dhendrix@chromium.org>
Tested-by: build bot (Jenkins)

2 files changed, 203 insertions, 40 deletions

diff --git a/payloads/libpayload/arch/armv7/cache.c b/payloads/libpayload/arch/armv7/cache.c
index 60d5f742b9..9fac442f17 100644
--- a/payloads/libpayload/arch/armv7/cache.c
+++ b/payloads/libpayload/arch/armv7/cache.c
@@ -26,7 +26,9 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * cache.c: Low-level cache operations for ARMv7
+ * cache.c: Cache maintenance routines for ARMv7-A and ARMv7-R
+ *
+ * Reference: ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition
  */
 
 #include <inttypes.h>
@@ -52,8 +54,8 @@ void tlb_invalidate_all(void)
 {
 	/*
 	 * FIXME: ARMv7 Architecture Ref. Manual claims that the distinction
-	 * instruction vs. data TLBs is deprecated in ARMv7. But that doesn't
-	 * really seem true for Cortex-A15?
+	 * instruction vs. data TLBs is deprecated in ARMv7, however this does
+	 * not seem to be the case as of Cortex-A15.
 	 */
 	tlbiall();
 	dtlbiall();
@@ -64,7 +66,8 @@ void tlb_invalidate_all(void)
 
 void icache_invalidate_all(void)
 {
-	/* icache can be entirely invalidated with one operation.
+	/*
+	 * icache can be entirely invalidated with one operation.
 	 * Note: If branch predictors are architecturally-visible, ICIALLU
 	 * also performs a BPIALL operation (B2-1283 in arch manual)
 	 */
@@ -74,10 +77,17 @@ void icache_invalidate_all(void)
 
 enum dcache_op {
 	OP_DCCISW,
-	OP_DCISW
+	OP_DCISW,
+	OP_DCCIMVAC,
+	OP_DCCMVAC,
 };
 
-/* do a dcache operation on entire cache by set/way */
+/*
+ * Do a dcache operation on entire cache by set/way. This is done for
+ * portability because mapping of memory address to cache location is
+ * implementation defined (See note on "Requirements for operations by
+ * set/way" in arch ref. manual).
+ */
 static void dcache_op_set_way(enum dcache_op op)
 {
 	uint32_t ccsidr;
@@ -101,6 +111,8 @@ static void dcache_op_set_way(enum dcache_op op)
 	/* FIXME: do we need to use CTR.DminLine here? */
 	linesize_bytes = (1 << ((ccsidr & 0x7) + 2)) * 4;
 
+	dsb();
+
 	/*
 	 * Set/way operations require an interesting bit packing. See section
 	 * B4-35 in the ARMv7 Architecture Reference Manual:
@@ -134,8 +146,7 @@ static void dcache_op_set_way(enum dcache_op op)
 			}
 		}
 	}
-
-	dsb();
+	isb();
 }
 
 void dcache_clean_invalidate_all(void)
@@ -161,18 +172,116 @@ static unsigned int line_bytes(void)
 	return size;
 }
 
+/*
+ * Do a dcache operation by modified virtual address. This is useful for
+ * maintaining coherency in drivers which do DMA transfers and only need to
+ * perform cache maintenance on a particular memory range rather than the
+ * entire cache.
+ */
+static void dcache_op_mva(unsigned long addr,
+		unsigned long len, enum dcache_op op)
+{
+	unsigned long line, linesize;
+
+	linesize = line_bytes();
+	line = addr & ~(linesize - 1);
+
+	dsb();
+	while (line < addr + len) {
+		switch(op) {
+		case OP_DCCIMVAC:
+			dccimvac(line);
+			break;
+		default:
+			break;
+		}
+		line += linesize;
+	}
+	isb();
+}
+
+void dcache_clean_by_mva(unsigned long addr, unsigned long len)
+{
+	dcache_op_mva(addr, len, OP_DCCMVAC);
+}
+
 void dcache_clean_invalidate_by_mva(unsigned long addr, unsigned long len)
 {
-	unsigned long line, i;
+	dcache_op_mva(addr, len, OP_DCCIMVAC);
+}
+
+void dcache_mmu_disable(void)
+{
+	uint32_t sctlr, csselr;
+
+	/* ensure L1 data/unified cache is selected */
+	csselr = read_csselr();
+	csselr &= ~0xf;
+	write_csselr(csselr);
+
+	dcache_clean_invalidate_all();
+
+	sctlr = read_sctlr();
+	sctlr &= ~(SCTLR_C | SCTLR_M);
+	write_sctlr(sctlr);
+}
+
+
+void dcache_mmu_enable(void)
+{
+	uint32_t sctlr;
 
-	line = line_bytes();
-	for (i = addr & ~(line - 1); i < addr + len - 1; i += line)
-		dccimvac(addr);
+	sctlr = read_sctlr();
+	dcache_clean_invalidate_all();
+	sctlr |= SCTLR_C | SCTLR_M;
+	write_sctlr(sctlr);
 }
 
-/* FIXME: wrapper around imported mmu_setup() for now */
-extern void mmu_setup(unsigned long start, unsigned long size);
-void mmu_setup_by_mva(unsigned long start, unsigned long size)
+void armv7_invalidate_caches(void)
 {
-	mmu_setup(start, size);
+	uint32_t clidr;
+	int level;
+
+	/* Invalidate branch predictor */
+	bpiall();
+
+	/* Iterate thru each cache identified in CLIDR and invalidate */
+	clidr = read_clidr();
+	for (level = 0; level < 7; level++) {
+		unsigned int ctype = (clidr >> (level * 3)) & 0x7;
+		uint32_t csselr;
+
+		switch(ctype) {
+		case 0x0:
+			/* no cache */
+			break;
+		case 0x1:
+			/* icache only */
+			csselr = (level << 1) | 1;
+			write_csselr(csselr);
+			icache_invalidate_all();
+			break;
+		case 0x2:
+		case 0x4:
+			/* dcache only or unified cache */
+			dcache_invalidate_all();
+			break;
+		case 0x3:
+			/* separate icache and dcache */
+			csselr = (level << 1) | 1;
+			write_csselr(csselr);
+			icache_invalidate_all();
+
+			csselr = level < 1;
+			write_csselr(csselr);
+			dcache_invalidate_all();
+			break;
+		default:
+			/* reserved */
+			break;
+		}
+	}
+
+	/* Invalidate TLB */
+	tlb_invalidate_all();
 }
diff --git a/payloads/libpayload/include/armv7/arch/cache.h b/payloads/libpayload/include/armv7/arch/cache.h
index 5125b8c449..f074a3b158 100644
--- a/payloads/libpayload/include/armv7/arch/cache.h
+++ b/payloads/libpayload/include/armv7/arch/cache.h
@@ -25,6 +25,8 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
+ * cache.h: Cache maintenance API for ARMv7
  */
 
 #ifndef ARMV7_CACHE_H
@@ -91,7 +93,7 @@ static inline void isb(void)
 /* invalidate entire data TLB */
 static inline void dtlbiall(void)
 {
-	asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0));
+	asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0) : "memory");
 }
 
 /* invalidate entire instruction TLB */
@@ -103,7 +105,33 @@ static inline void itlbiall(void)
 /* invalidate entire unified TLB */
 static inline void tlbiall(void)
 {
-	asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0));
+	asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0) : "memory");
+}
+
+/* write data access control register (DACR) */
+static inline void write_dacr(uint32_t val)
+{
+	asm volatile ("mcr p15, 0, %0, c3, c0, 0" : : "r" (val));
+}
+
+/* write translation table base register 0 (TTBR0) */
+static inline void write_ttbr0(uint32_t val)
+{
+	asm volatile ("mcr p15, 0, %0, c2, c0, 0" : : "r" (val) : "memory");
+}
+
+/* read translation table base control register (TTBCR) */
+static inline uint32_t read_ttbcr(void)
+{
+	uint32_t val = 0;
+	asm volatile ("mrc p15, 0, %0, c2, c0, 2" : "=r" (val));
+	return val;
+}
+
+/* write translation table base control register (TTBCR) */
+static inline void write_ttbcr(uint32_t val)
+{
+	asm volatile ("mcr p15, 0, %0, c2, c0, 2" : : "r" (val) : "memory");
 }
 
 /*
@@ -119,31 +147,31 @@ static inline void bpiall(void)
 /* data cache clean and invalidate by MVA to PoC */
 static inline void dccimvac(unsigned long mva)
 {
-	asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva));
+	asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva) : "memory");
 }
 
 /* data cache invalidate by set/way */
 static inline void dccisw(uint32_t val)
 {
-	asm volatile ("mcr p15, 0, %0, c7, c14, 2" : : "r" (val));
-}
-
-/* data cache invalidate by set/way */
-static inline void dcisw(uint32_t val)
-{
-	asm volatile ("mcr p15, 0, %0, c7, c6, 2" : : "r" (val));
+	asm volatile ("mcr p15, 0, %0, c7, c14, 2" : : "r" (val) : "memory");
 }
 
 /* data cache clean by MVA to PoC */
 static inline void dccmvac(unsigned long mva)
 {
-	asm volatile ("mcr p15, 0, %0, c7, c10, 1" : : "r" (mva));
+	asm volatile ("mcr p15, 0, %0, c7, c10, 1" : : "r" (mva) : "memory");
 }
 
 /* data cache invalidate by MVA to PoC */
 static inline void dcimvac(unsigned long mva)
 {
-	asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva));
+	asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva) : "memory");
+}
+
+/* data cache invalidate by set/way */
+static inline void dcisw(uint32_t val)
+{
+	asm volatile ("mcr p15, 0, %0, c7, c6, 2" : : "r" (val) : "memory");
 }
 
 /* instruction cache invalidate all by PoU */
@@ -195,12 +223,12 @@ static inline void write_csselr(uint32_t val)
 static inline unsigned int read_sctlr(void)
 {
 	unsigned int val;
-	asm volatile ("mrc p15, 0, %0, c1, c0, 0" : "=r" (val) : : "cc");
+	asm volatile ("mrc p15, 0, %0, c1, c0, 0" : "=r" (val));
 	return val;
 }
 
 /* write system control register (SCTLR) */
-static inline void write_sctlr(unsigned int val)
+static inline void write_sctlr(uint32_t val)
 {
 	asm volatile ("mcr p15, 0, %0, c1, c0, 0" : : "r" (val) : "cc");
 	isb();
@@ -210,22 +238,48 @@ static inline void write_sctlr(unsigned int val)
  * Cache maintenance API
  */
 
-/* invalidate all TLBs */
-void tlb_invalidate_all(void);
-
-/* clean and invalidate entire dcache on current level (given by CCSELR) */
+/* dcache clean and invalidate all (on current level given by CCSELR) */
 void dcache_clean_invalidate_all(void);
 
-/* invalidate entire dcache on current level (given by CCSELR) */
-void dcache_invalidate_all(void);
+/* dcache clean by modified virtual address to PoC */
+void dcache_clean_by_mva(unsigned long addr, unsigned long len);
 
-/* invalidate and clean dcache by machine virtual address to PoC */
+/* dcache clean and invalidate by modified virtual address to PoC */
 void dcache_clean_invalidate_by_mva(unsigned long addr, unsigned long len);
 
-/* invalidate entire icache on current level (given by CSSELR) */
+/* dcache invalidate all (on current level given by CCSELR) */
+void dcache_invalidate_all(void);
+
+/* dcache and MMU disable */
+void dcache_mmu_disable(void);
+
+/* dcache and MMU enable */
+void dcache_mmu_enable(void);
+
+/* icache invalidate all (on current level given by CSSELR) */
 void icache_invalidate_all(void);
 
-/* MMU setup by machine virtual address */
-void mmu_setup_by_mva(unsigned long start, unsigned long size);
+/* tlb invalidate all */
+void tlb_invalidate_all(void);
+
+/*
+ * Generalized setup/init functions
+ */
+
+/* invalidate all caches on ARMv7 */
+void armv7_invalidate_caches(void);
+
+/* mmu initialization (set page table address, set permissions, etc) */
+void mmu_init(void);
+
+enum dcache_policy {
+	DCACHE_OFF,
+	DCACHE_WRITEBACK,
+	DCACHE_WRITETHROUGH,
+};
+
+/* mmu range configuration (set dcache policy) */
+void mmu_config_range(unsigned long start_mb, unsigned long size_mb,
+						enum dcache_policy policy);
 
 #endif /* ARMV7_CACHE_H */