/* * Copyright (c) 2010, 2012-2018 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall * not be construed as granting a license to any other intellectual * property including but not limited to intellectual property relating * to a hardware implementation of the functionality of the software * licensed hereunder. You may use the software subject to the license * terms below provided that you ensure that this notice is replicated * unmodified and in its entirety in all distributions of the software, * modified or unmodified, in source code or in binary form. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer; * redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution; * neither the name of the copyright holders nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Authors: Ali Saidi * Giacomo Gabrielli */ #include "arch/arm/table_walker.hh" #include #include "arch/arm/faults.hh" #include "arch/arm/stage2_mmu.hh" #include "arch/arm/system.hh" #include "arch/arm/tlb.hh" #include "cpu/base.hh" #include "cpu/thread_context.hh" #include "debug/Checkpoint.hh" #include "debug/Drain.hh" #include "debug/TLB.hh" #include "debug/TLBVerbose.hh" #include "dev/dma_device.hh" #include "sim/system.hh" using namespace ArmISA; TableWalker::TableWalker(const Params *p) : MemObject(p), stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId), isStage2(p->is_stage2), tlb(NULL), currState(NULL), pending(false), numSquashable(p->num_squash_per_cycle), pendingReqs(0), pendingChangeTick(curTick()), doL1DescEvent([this]{ doL1DescriptorWrapper(); }, name()), doL2DescEvent([this]{ doL2DescriptorWrapper(); }, name()), doL0LongDescEvent([this]{ doL0LongDescriptorWrapper(); }, name()), doL1LongDescEvent([this]{ doL1LongDescriptorWrapper(); }, name()), doL2LongDescEvent([this]{ doL2LongDescriptorWrapper(); }, name()), doL3LongDescEvent([this]{ doL3LongDescriptorWrapper(); }, name()), LongDescEventByLevel { &doL0LongDescEvent, &doL1LongDescEvent, &doL2LongDescEvent, &doL3LongDescEvent }, doProcessEvent([this]{ processWalkWrapper(); }, name()) { sctlr = 0; // Cache system-level properties if (FullSystem) { ArmSystem *armSys = dynamic_cast(p->sys); assert(armSys); haveSecurity = armSys->haveSecurity(); _haveLPAE = armSys->haveLPAE(); _haveVirtualization = armSys->haveVirtualization(); physAddrRange = armSys->physAddrRange(); _haveLargeAsid64 = armSys->haveLargeAsid64(); } else { haveSecurity = _haveLPAE = _haveVirtualization = false; _haveLargeAsid64 = false; physAddrRange = 32; } } TableWalker::~TableWalker() { ; } void TableWalker::setMMU(Stage2MMU *m, MasterID master_id) { stage2Mmu = m; port = &m->getPort(); masterId = master_id; } void TableWalker::init() { fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n"); fatal_if(!port, "Table walker must have a valid port\n"); fatal_if(!tlb, "Table walker must have a valid TLB\n"); } BaseMasterPort& TableWalker::getMasterPort(const std::string &if_name, PortID idx) { if (if_name == "port") { if (!isStage2) { return *port; } else { fatal("Cannot access table walker port through stage-two walker\n"); } } return MemObject::getMasterPort(if_name, idx); } TableWalker::WalkerState::WalkerState() : tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr), asid(0), vmid(0), isHyp(false), transState(nullptr), vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false), secureLookup(false), rwTable(false), userTable(false), xnTable(false), pxnTable(false), stage2Req(false), stage2Tran(nullptr), timing(false), functional(false), mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc), delayed(false), tableWalker(nullptr) { } void TableWalker::completeDrain() { if (drainState() == DrainState::Draining && stateQueues[L0].empty() && stateQueues[L1].empty() && stateQueues[L2].empty() && stateQueues[L3].empty() && pendingQueue.empty()) { DPRINTF(Drain, "TableWalker done draining, processing drain event\n"); signalDrainDone(); } } DrainState TableWalker::drain() { bool state_queues_not_empty = false; for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) { if (!stateQueues[i].empty()) { state_queues_not_empty = true; break; } } if (state_queues_not_empty || pendingQueue.size()) { DPRINTF(Drain, "TableWalker not drained\n"); return DrainState::Draining; } else { DPRINTF(Drain, "TableWalker free, no need to drain\n"); return DrainState::Drained; } } void TableWalker::drainResume() { if (params()->sys->isTimingMode() && currState) { delete currState; currState = NULL; pendingChange(); } } Fault TableWalker::walk(const RequestPtr &_req, ThreadContext *_tc, uint16_t _asid, uint8_t _vmid, bool _isHyp, TLB::Mode _mode, TLB::Translation *_trans, bool _timing, bool _functional, bool secure, TLB::ArmTranslationType tranType, bool _stage2Req) { assert(!(_functional && _timing)); ++statWalks; WalkerState *savedCurrState = NULL; if (!currState && !_functional) { // For atomic mode, a new WalkerState instance should be only created // once per TLB. For timing mode, a new instance is generated for every // TLB miss. DPRINTF(TLBVerbose, "creating new instance of WalkerState\n"); currState = new WalkerState(); currState->tableWalker = this; } else if (_functional) { // If we are mixing functional mode with timing (or even // atomic), we need to to be careful and clean up after // ourselves to not risk getting into an inconsistent state. DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n"); savedCurrState = currState; currState = new WalkerState(); currState->tableWalker = this; } else if (_timing) { // This is a translation that was completed and then faulted again // because some underlying parameters that affect the translation // changed out from under us (e.g. asid). It will either be a // misprediction, in which case nothing will happen or we'll use // this fault to re-execute the faulting instruction which should clean // up everything. if (currState->vaddr_tainted == _req->getVaddr()) { ++statSquashedBefore; return std::make_shared(); } } pendingChange(); currState->startTime = curTick(); currState->tc = _tc; // ARM DDI 0487A.f (ARMv8 ARM) pg J8-5672 // aarch32/translation/translation/AArch32.TranslateAddress dictates // even AArch32 EL0 will use AArch64 translation if EL1 is in AArch64. if (isStage2) { currState->el = EL1; currState->aarch64 = ELIs64(_tc, EL2); } else { currState->el = TLB::tranTypeEL(_tc->readMiscReg(MISCREG_CPSR), tranType); currState->aarch64 = ELIs64(_tc, currState->el == EL0 ? EL1 : currState->el); } currState->transState = _trans; currState->req = _req; currState->fault = NoFault; currState->asid = _asid; currState->vmid = _vmid; currState->isHyp = _isHyp; currState->timing = _timing; currState->functional = _functional; currState->mode = _mode; currState->tranType = tranType; currState->isSecure = secure; currState->physAddrRange = physAddrRange; /** @todo These should be cached or grabbed from cached copies in the TLB, all these miscreg reads are expensive */ currState->vaddr_tainted = currState->req->getVaddr(); if (currState->aarch64) currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted, currState->tc, currState->el); else currState->vaddr = currState->vaddr_tainted; if (currState->aarch64) { if (isStage2) { currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1); currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR_EL2); } else switch (currState->el) { case EL0: case EL1: currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1); currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1); break; case EL2: assert(_haveVirtualization); currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2); currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2); break; case EL3: assert(haveSecurity); currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3); currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3); break; default: panic("Invalid exception level"); break; } currState->hcr = currState->tc->readMiscReg(MISCREG_HCR_EL2); } else { currState->sctlr = currState->tc->readMiscReg(snsBankedIndex( MISCREG_SCTLR, currState->tc, !currState->isSecure)); currState->ttbcr = currState->tc->readMiscReg(snsBankedIndex( MISCREG_TTBCR, currState->tc, !currState->isSecure)); currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR); currState->hcr = currState->tc->readMiscReg(MISCREG_HCR); currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR); } sctlr = currState->sctlr; currState->isFetch = (currState->mode == TLB::Execute); currState->isWrite = (currState->mode == TLB::Write); statRequestOrigin[REQUESTED][currState->isFetch]++; currState->stage2Req = _stage2Req && !isStage2; bool long_desc_format = currState->aarch64 || _isHyp || isStage2 || longDescFormatInUse(currState->tc); if (long_desc_format) { // Helper variables used for hierarchical permissions currState->secureLookup = currState->isSecure; currState->rwTable = true; currState->userTable = true; currState->xnTable = false; currState->pxnTable = false; ++statWalksLongDescriptor; } else { ++statWalksShortDescriptor; } if (!currState->timing) { Fault fault = NoFault; if (currState->aarch64) fault = processWalkAArch64(); else if (long_desc_format) fault = processWalkLPAE(); else fault = processWalk(); // If this was a functional non-timing access restore state to // how we found it. if (currState->functional) { delete currState; currState = savedCurrState; } return fault; } if (pending || pendingQueue.size()) { pendingQueue.push_back(currState); currState = NULL; pendingChange(); } else { pending = true; pendingChange(); if (currState->aarch64) return processWalkAArch64(); else if (long_desc_format) return processWalkLPAE(); else return processWalk(); } return NoFault; } void TableWalker::processWalkWrapper() { assert(!currState); assert(pendingQueue.size()); pendingChange(); currState = pendingQueue.front(); // Check if a previous walk filled this request already // @TODO Should this always be the TLB or should we look in the stage2 TLB? TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid, currState->vmid, currState->isHyp, currState->isSecure, true, false, currState->el); // Check if we still need to have a walk for this request. If the requesting // instruction has been squashed, or a previous walk has filled the TLB with // a match, we just want to get rid of the walk. The latter could happen // when there are multiple outstanding misses to a single page and a // previous request has been successfully translated. if (!currState->transState->squashed() && !te) { // We've got a valid request, lets process it pending = true; pendingQueue.pop_front(); // Keep currState in case one of the processWalk... calls NULLs it WalkerState *curr_state_copy = currState; Fault f; if (currState->aarch64) f = processWalkAArch64(); else if (longDescFormatInUse(currState->tc) || currState->isHyp || isStage2) f = processWalkLPAE(); else f = processWalk(); if (f != NoFault) { curr_state_copy->transState->finish(f, curr_state_copy->req, curr_state_copy->tc, curr_state_copy->mode); delete curr_state_copy; } return; } // If the instruction that we were translating for has been // squashed we shouldn't bother. unsigned num_squashed = 0; ThreadContext *tc = currState->tc; while ((num_squashed < numSquashable) && currState && (currState->transState->squashed() || te)) { pendingQueue.pop_front(); num_squashed++; statSquashedBefore++; DPRINTF(TLB, "Squashing table walk for address %#x\n", currState->vaddr_tainted); if (currState->transState->squashed()) { // finish the translation which will delete the translation object currState->transState->finish( std::make_shared("Squashed Inst"), currState->req, currState->tc, currState->mode); } else { // translate the request now that we know it will work statWalkServiceTime.sample(curTick() - currState->startTime); tlb->translateTiming(currState->req, currState->tc, currState->transState, currState->mode); } // delete the current request delete currState; // peak at the next one if (pendingQueue.size()) { currState = pendingQueue.front(); te = tlb->lookup(currState->vaddr, currState->asid, currState->vmid, currState->isHyp, currState->isSecure, true, false, currState->el); } else { // Terminate the loop, nothing more to do currState = NULL; } } pendingChange(); // if we still have pending translations, schedule more work nextWalk(tc); currState = NULL; } Fault TableWalker::processWalk() { Addr ttbr = 0; // If translation isn't enabled, we shouldn't be here assert(currState->sctlr.m || isStage2); DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n", currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31, 32 - currState->ttbcr.n)); statWalkWaitTime.sample(curTick() - currState->startTime); if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31, 32 - currState->ttbcr.n)) { DPRINTF(TLB, " - Selecting TTBR0\n"); // Check if table walk is allowed when Security Extensions are enabled if (haveSecurity && currState->ttbcr.pd0) { if (currState->isFetch) return std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L1, isStage2, ArmFault::VmsaTran); else return std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L1, isStage2, ArmFault::VmsaTran); } ttbr = currState->tc->readMiscReg(snsBankedIndex( MISCREG_TTBR0, currState->tc, !currState->isSecure)); } else { DPRINTF(TLB, " - Selecting TTBR1\n"); // Check if table walk is allowed when Security Extensions are enabled if (haveSecurity && currState->ttbcr.pd1) { if (currState->isFetch) return std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L1, isStage2, ArmFault::VmsaTran); else return std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L1, isStage2, ArmFault::VmsaTran); } ttbr = currState->tc->readMiscReg(snsBankedIndex( MISCREG_TTBR1, currState->tc, !currState->isSecure)); currState->ttbcr.n = 0; } Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) | (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2); DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr, currState->isSecure ? "s" : "ns"); // Trickbox address check Fault f; f = testWalk(l1desc_addr, sizeof(uint32_t), TlbEntry::DomainType::NoAccess, L1); if (f) { DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); if (currState->timing) { pending = false; nextWalk(currState->tc); currState = NULL; } else { currState->tc = NULL; currState->req = NULL; } return f; } Request::Flags flag = Request::PT_WALK; if (currState->sctlr.c == 0) { flag.set(Request::UNCACHEABLE); } if (currState->isSecure) { flag.set(Request::SECURE); } bool delayed; delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data, sizeof(uint32_t), flag, L1, &doL1DescEvent, &TableWalker::doL1Descriptor); if (!delayed) { f = currState->fault; } return f; } Fault TableWalker::processWalkLPAE() { Addr ttbr, ttbr0_max, ttbr1_min, desc_addr; int tsz, n; LookupLevel start_lookup_level = L1; DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n", currState->vaddr_tainted, currState->ttbcr); statWalkWaitTime.sample(curTick() - currState->startTime); Request::Flags flag = Request::PT_WALK; if (currState->isSecure) flag.set(Request::SECURE); // work out which base address register to use, if in hyp mode we always // use HTTBR if (isStage2) { DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n"); ttbr = currState->tc->readMiscReg(MISCREG_VTTBR); tsz = sext<4>(currState->vtcr.t0sz); start_lookup_level = currState->vtcr.sl0 ? L1 : L2; } else if (currState->isHyp) { DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n"); ttbr = currState->tc->readMiscReg(MISCREG_HTTBR); tsz = currState->htcr.t0sz; } else { assert(longDescFormatInUse(currState->tc)); // Determine boundaries of TTBR0/1 regions if (currState->ttbcr.t0sz) ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1; else if (currState->ttbcr.t1sz) ttbr0_max = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz)) - 1; else ttbr0_max = (1ULL << 32) - 1; if (currState->ttbcr.t1sz) ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz)); else ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz)); // The following code snippet selects the appropriate translation table base // address (TTBR0 or TTBR1) and the appropriate starting lookup level // depending on the address range supported by the translation table (ARM // ARM issue C B3.6.4) if (currState->vaddr <= ttbr0_max) { DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n"); // Check if table walk is allowed if (currState->ttbcr.epd0) { if (currState->isFetch) return std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L1, isStage2, ArmFault::LpaeTran); else return std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L1, isStage2, ArmFault::LpaeTran); } ttbr = currState->tc->readMiscReg(snsBankedIndex( MISCREG_TTBR0, currState->tc, !currState->isSecure)); tsz = currState->ttbcr.t0sz; if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB start_lookup_level = L2; } else if (currState->vaddr >= ttbr1_min) { DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n"); // Check if table walk is allowed if (currState->ttbcr.epd1) { if (currState->isFetch) return std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L1, isStage2, ArmFault::LpaeTran); else return std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L1, isStage2, ArmFault::LpaeTran); } ttbr = currState->tc->readMiscReg(snsBankedIndex( MISCREG_TTBR1, currState->tc, !currState->isSecure)); tsz = currState->ttbcr.t1sz; if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB start_lookup_level = L2; } else { // Out of boundaries -> translation fault if (currState->isFetch) return std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L1, isStage2, ArmFault::LpaeTran); else return std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L1, isStage2, ArmFault::LpaeTran); } } // Perform lookup (ARM ARM issue C B3.6.6) if (start_lookup_level == L1) { n = 5 - tsz; desc_addr = mbits(ttbr, 39, n) | (bits(currState->vaddr, n + 26, 30) << 3); DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", desc_addr, currState->isSecure ? "s" : "ns"); } else { // Skip first-level lookup n = (tsz >= 2 ? 14 - tsz : 12); desc_addr = mbits(ttbr, 39, n) | (bits(currState->vaddr, n + 17, 21) << 3); DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", desc_addr, currState->isSecure ? "s" : "ns"); } // Trickbox address check Fault f = testWalk(desc_addr, sizeof(uint64_t), TlbEntry::DomainType::NoAccess, start_lookup_level); if (f) { DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); if (currState->timing) { pending = false; nextWalk(currState->tc); currState = NULL; } else { currState->tc = NULL; currState->req = NULL; } return f; } if (currState->sctlr.c == 0) { flag.set(Request::UNCACHEABLE); } currState->longDesc.lookupLevel = start_lookup_level; currState->longDesc.aarch64 = false; currState->longDesc.grainSize = Grain4KB; bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data, sizeof(uint64_t), flag, start_lookup_level, LongDescEventByLevel[start_lookup_level], &TableWalker::doLongDescriptor); if (!delayed) { f = currState->fault; } return f; } unsigned TableWalker::adjustTableSizeAArch64(unsigned tsz) { if (tsz < 25) return 25; if (tsz > 48) return 48; return tsz; } bool TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange) { return (currPhysAddrRange != MaxPhysAddrRange && bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange)); } Fault TableWalker::processWalkAArch64() { assert(currState->aarch64); DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n", currState->vaddr_tainted, currState->tcr); static const GrainSize GrainMap_tg0[] = { Grain4KB, Grain64KB, Grain16KB, ReservedGrain }; static const GrainSize GrainMap_tg1[] = { ReservedGrain, Grain16KB, Grain4KB, Grain64KB }; statWalkWaitTime.sample(curTick() - currState->startTime); // Determine TTBR, table size, granule size and phys. address range Addr ttbr = 0; int tsz = 0, ps = 0; GrainSize tg = Grain4KB; // grain size computed from tg* field bool fault = false; LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS; switch (currState->el) { case EL0: case EL1: if (isStage2) { DPRINTF(TLB, " - Selecting VTTBR0 (AArch64 stage 2)\n"); ttbr = currState->tc->readMiscReg(MISCREG_VTTBR_EL2); tsz = 64 - currState->vtcr.t0sz64; tg = GrainMap_tg0[currState->vtcr.tg0]; // ARM DDI 0487A.f D7-2148 // The starting level of stage 2 translation depends on // VTCR_EL2.SL0 and VTCR_EL2.TG0 LookupLevel __ = MAX_LOOKUP_LEVELS; // invalid level uint8_t sl_tg = (currState->vtcr.sl0 << 2) | currState->vtcr.tg0; static const LookupLevel SLL[] = { L2, L3, L3, __, // sl0 == 0 L1, L2, L2, __, // sl0 == 1, etc. L0, L1, L1, __, __, __, __, __ }; start_lookup_level = SLL[sl_tg]; panic_if(start_lookup_level == MAX_LOOKUP_LEVELS, "Cannot discern lookup level from vtcr.{sl0,tg0}"); ps = currState->vtcr.ps; } else { switch (bits(currState->vaddr, 63,48)) { case 0: DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1); tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); tg = GrainMap_tg0[currState->tcr.tg0]; if (bits(currState->vaddr, 63, tsz) != 0x0 || currState->tcr.epd0) fault = true; break; case 0xffff: DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n"); ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1); tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz); tg = GrainMap_tg1[currState->tcr.tg1]; if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) || currState->tcr.epd1) fault = true; break; default: // top two bytes must be all 0s or all 1s, else invalid addr fault = true; } ps = currState->tcr.ips; } break; case EL2: switch(bits(currState->vaddr, 63,48)) { case 0: DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2); tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); tg = GrainMap_tg0[currState->tcr.tg0]; break; case 0xffff: DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n"); ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL2); tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz); tg = GrainMap_tg1[currState->tcr.tg1]; if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) || currState->tcr.epd1 || !currState->hcr.e2h) fault = true; break; default: // invalid addr if top two bytes are not all 0s fault = true; } ps = currState->tcr.ps; break; case EL3: switch(bits(currState->vaddr, 63,48)) { case 0: DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3); tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); tg = GrainMap_tg0[currState->tcr.tg0]; break; default: // invalid addr if top two bytes are not all 0s fault = true; } ps = currState->tcr.ps; break; } if (fault) { Fault f; if (currState->isFetch) f = std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L0, isStage2, ArmFault::LpaeTran); else f = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L0, isStage2, ArmFault::LpaeTran); if (currState->timing) { pending = false; nextWalk(currState->tc); currState = NULL; } else { currState->tc = NULL; currState->req = NULL; } return f; } if (tg == ReservedGrain) { warn_once("Reserved granule size requested; gem5's IMPLEMENTATION " "DEFINED behavior takes this to mean 4KB granules\n"); tg = Grain4KB; } // Determine starting lookup level // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library // in ARM DDI 0487A. These table values correspond to the cascading tests // to compute the lookup level and are of the form // (grain_size + N*stride), for N = {1, 2, 3}. // A value of 64 will never succeed and a value of 0 will always succeed. if (start_lookup_level == MAX_LOOKUP_LEVELS) { struct GrainMap { GrainSize grain_size; unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS]; }; static const GrainMap GM[] = { { Grain4KB, { 39, 30, 0, 0 } }, { Grain16KB, { 47, 36, 25, 0 } }, { Grain64KB, { 64, 42, 29, 0 } } }; const unsigned *lookup = NULL; // points to a lookup_level_cutoff for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[] if (tg == GM[i].grain_size) { lookup = GM[i].lookup_level_cutoff; break; } } assert(lookup); for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) { if (tsz > lookup[L]) { start_lookup_level = (LookupLevel) L; break; } } panic_if(start_lookup_level == MAX_LOOKUP_LEVELS, "Table walker couldn't find lookup level\n"); } int stride = tg - 3; // Determine table base address int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg; Addr base_addr = mbits(ttbr, 47, base_addr_lo); // Determine physical address size and raise an Address Size Fault if // necessary int pa_range = decodePhysAddrRange64(ps); // Clamp to lower limit if (pa_range > physAddrRange) currState->physAddrRange = physAddrRange; else currState->physAddrRange = pa_range; if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) { DPRINTF(TLB, "Address size fault before any lookup\n"); Fault f; if (currState->isFetch) f = std::make_shared( currState->vaddr_tainted, ArmFault::AddressSizeLL + start_lookup_level, isStage2, ArmFault::LpaeTran); else f = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::AddressSizeLL + start_lookup_level, isStage2, ArmFault::LpaeTran); if (currState->timing) { pending = false; nextWalk(currState->tc); currState = NULL; } else { currState->tc = NULL; currState->req = NULL; } return f; } // Determine descriptor address Addr desc_addr = base_addr | (bits(currState->vaddr, tsz - 1, stride * (3 - start_lookup_level) + tg) << 3); // Trickbox address check Fault f = testWalk(desc_addr, sizeof(uint64_t), TlbEntry::DomainType::NoAccess, start_lookup_level); if (f) { DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); if (currState->timing) { pending = false; nextWalk(currState->tc); currState = NULL; } else { currState->tc = NULL; currState->req = NULL; } return f; } Request::Flags flag = Request::PT_WALK; if (currState->sctlr.c == 0) { flag.set(Request::UNCACHEABLE); } if (currState->isSecure) { flag.set(Request::SECURE); } currState->longDesc.lookupLevel = start_lookup_level; currState->longDesc.aarch64 = true; currState->longDesc.grainSize = tg; if (currState->timing) { fetchDescriptor(desc_addr, (uint8_t*) &currState->longDesc.data, sizeof(uint64_t), flag, start_lookup_level, LongDescEventByLevel[start_lookup_level], NULL); } else { fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data, sizeof(uint64_t), flag, -1, NULL, &TableWalker::doLongDescriptor); f = currState->fault; } return f; } void TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr, uint8_t texcb, bool s) { // Note: tc and sctlr local variables are hiding tc and sctrl class // variables DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s); te.shareable = false; // default value te.nonCacheable = false; te.outerShareable = false; if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) { switch(texcb) { case 0: // Stongly-ordered te.nonCacheable = true; te.mtype = TlbEntry::MemoryType::StronglyOrdered; te.shareable = true; te.innerAttrs = 1; te.outerAttrs = 0; break; case 1: // Shareable Device te.nonCacheable = true; te.mtype = TlbEntry::MemoryType::Device; te.shareable = true; te.innerAttrs = 3; te.outerAttrs = 0; break; case 2: // Outer and Inner Write-Through, no Write-Allocate te.mtype = TlbEntry::MemoryType::Normal; te.shareable = s; te.innerAttrs = 6; te.outerAttrs = bits(texcb, 1, 0); break; case 3: // Outer and Inner Write-Back, no Write-Allocate te.mtype = TlbEntry::MemoryType::Normal; te.shareable = s; te.innerAttrs = 7; te.outerAttrs = bits(texcb, 1, 0); break; case 4: // Outer and Inner Non-cacheable te.nonCacheable = true; te.mtype = TlbEntry::MemoryType::Normal; te.shareable = s; te.innerAttrs = 0; te.outerAttrs = bits(texcb, 1, 0); break; case 5: // Reserved panic("Reserved texcb value!\n"); break; case 6: // Implementation Defined panic("Implementation-defined texcb value!\n"); break; case 7: // Outer and Inner Write-Back, Write-Allocate te.mtype = TlbEntry::MemoryType::Normal; te.shareable = s; te.innerAttrs = 5; te.outerAttrs = 1; break; case 8: // Non-shareable Device te.nonCacheable = true; te.mtype = TlbEntry::MemoryType::Device; te.shareable = false; te.innerAttrs = 3; te.outerAttrs = 0; break; case 9 ... 15: // Reserved panic("Reserved texcb value!\n"); break; case 16 ... 31: // Cacheable Memory te.mtype = TlbEntry::MemoryType::Normal; te.shareable = s; if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0) te.nonCacheable = true; te.innerAttrs = bits(texcb, 1, 0); te.outerAttrs = bits(texcb, 3, 2); break; default: panic("More than 32 states for 5 bits?\n"); } } else { assert(tc); PRRR prrr = tc->readMiscReg(snsBankedIndex(MISCREG_PRRR, currState->tc, !currState->isSecure)); NMRR nmrr = tc->readMiscReg(snsBankedIndex(MISCREG_NMRR, currState->tc, !currState->isSecure)); DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr); uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0; switch(bits(texcb, 2,0)) { case 0: curr_tr = prrr.tr0; curr_ir = nmrr.ir0; curr_or = nmrr.or0; te.outerShareable = (prrr.nos0 == 0); break; case 1: curr_tr = prrr.tr1; curr_ir = nmrr.ir1; curr_or = nmrr.or1; te.outerShareable = (prrr.nos1 == 0); break; case 2: curr_tr = prrr.tr2; curr_ir = nmrr.ir2; curr_or = nmrr.or2; te.outerShareable = (prrr.nos2 == 0); break; case 3: curr_tr = prrr.tr3; curr_ir = nmrr.ir3; curr_or = nmrr.or3; te.outerShareable = (prrr.nos3 == 0); break; case 4: curr_tr = prrr.tr4; curr_ir = nmrr.ir4; curr_or = nmrr.or4; te.outerShareable = (prrr.nos4 == 0); break; case 5: curr_tr = prrr.tr5; curr_ir = nmrr.ir5; curr_or = nmrr.or5; te.outerShareable = (prrr.nos5 == 0); break; case 6: panic("Imp defined type\n"); case 7: curr_tr = prrr.tr7; curr_ir = nmrr.ir7; curr_or = nmrr.or7; te.outerShareable = (prrr.nos7 == 0); break; } switch(curr_tr) { case 0: DPRINTF(TLBVerbose, "StronglyOrdered\n"); te.mtype = TlbEntry::MemoryType::StronglyOrdered; te.nonCacheable = true; te.innerAttrs = 1; te.outerAttrs = 0; te.shareable = true; break; case 1: DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n", prrr.ds1, prrr.ds0, s); te.mtype = TlbEntry::MemoryType::Device; te.nonCacheable = true; te.innerAttrs = 3; te.outerAttrs = 0; if (prrr.ds1 && s) te.shareable = true; if (prrr.ds0 && !s) te.shareable = true; break; case 2: DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n", prrr.ns1, prrr.ns0, s); te.mtype = TlbEntry::MemoryType::Normal; if (prrr.ns1 && s) te.shareable = true; if (prrr.ns0 && !s) te.shareable = true; break; case 3: panic("Reserved type"); } if (te.mtype == TlbEntry::MemoryType::Normal){ switch(curr_ir) { case 0: te.nonCacheable = true; te.innerAttrs = 0; break; case 1: te.innerAttrs = 5; break; case 2: te.innerAttrs = 6; break; case 3: te.innerAttrs = 7; break; } switch(curr_or) { case 0: te.nonCacheable = true; te.outerAttrs = 0; break; case 1: te.outerAttrs = 1; break; case 2: te.outerAttrs = 2; break; case 3: te.outerAttrs = 3; break; } } } DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, " "outerAttrs: %d\n", te.shareable, te.innerAttrs, te.outerAttrs); te.setAttributes(false); } void TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te, LongDescriptor &lDescriptor) { assert(_haveLPAE); uint8_t attr; uint8_t sh = lDescriptor.sh(); // Different format and source of attributes if this is a stage 2 // translation if (isStage2) { attr = lDescriptor.memAttr(); uint8_t attr_3_2 = (attr >> 2) & 0x3; uint8_t attr_1_0 = attr & 0x3; DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh); if (attr_3_2 == 0) { te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered : TlbEntry::MemoryType::Device; te.outerAttrs = 0; te.innerAttrs = attr_1_0 == 0 ? 1 : 3; te.nonCacheable = true; } else { te.mtype = TlbEntry::MemoryType::Normal; te.outerAttrs = attr_3_2 == 1 ? 0 : attr_3_2 == 2 ? 2 : 1; te.innerAttrs = attr_1_0 == 1 ? 0 : attr_1_0 == 2 ? 6 : 5; te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1); } } else { uint8_t attrIndx = lDescriptor.attrIndx(); // LPAE always uses remapping of memory attributes, irrespective of the // value of SCTLR.TRE MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0; int reg_as_int = snsBankedIndex(reg, currState->tc, !currState->isSecure); uint32_t mair = currState->tc->readMiscReg(reg_as_int); attr = (mair >> (8 * (attrIndx % 4))) & 0xff; uint8_t attr_7_4 = bits(attr, 7, 4); uint8_t attr_3_0 = bits(attr, 3, 0); DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr); // Note: the memory subsystem only cares about the 'cacheable' memory // attribute. The other attributes are only used to fill the PAR register // accordingly to provide the illusion of full support te.nonCacheable = false; switch (attr_7_4) { case 0x0: // Strongly-ordered or Device memory if (attr_3_0 == 0x0) te.mtype = TlbEntry::MemoryType::StronglyOrdered; else if (attr_3_0 == 0x4) te.mtype = TlbEntry::MemoryType::Device; else panic("Unpredictable behavior\n"); te.nonCacheable = true; te.outerAttrs = 0; break; case 0x4: // Normal memory, Outer Non-cacheable te.mtype = TlbEntry::MemoryType::Normal; te.outerAttrs = 0; if (attr_3_0 == 0x4) // Inner Non-cacheable te.nonCacheable = true; else if (attr_3_0 < 0x8) panic("Unpredictable behavior\n"); break; case 0x8: case 0x9: case 0xa: case 0xb: case 0xc: case 0xd: case 0xe: case 0xf: if (attr_7_4 & 0x4) { te.outerAttrs = (attr_7_4 & 1) ? 1 : 3; } else { te.outerAttrs = 0x2; } // Normal memory, Outer Cacheable te.mtype = TlbEntry::MemoryType::Normal; if (attr_3_0 != 0x4 && attr_3_0 < 0x8) panic("Unpredictable behavior\n"); break; default: panic("Unpredictable behavior\n"); break; } switch (attr_3_0) { case 0x0: te.innerAttrs = 0x1; break; case 0x4: te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0; break; case 0x8: case 0x9: case 0xA: case 0xB: te.innerAttrs = 6; break; case 0xC: case 0xD: case 0xE: case 0xF: te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7; break; default: panic("Unpredictable behavior\n"); break; } } te.outerShareable = sh == 2; te.shareable = (sh & 0x2) ? true : false; te.setAttributes(true); te.attributes |= (uint64_t) attr << 56; } void TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, LongDescriptor &lDescriptor) { uint8_t attr; uint8_t attr_hi; uint8_t attr_lo; uint8_t sh = lDescriptor.sh(); if (isStage2) { attr = lDescriptor.memAttr(); uint8_t attr_hi = (attr >> 2) & 0x3; uint8_t attr_lo = attr & 0x3; DPRINTF(TLBVerbose, "memAttrsAArch64 MemAttr:%#x sh:%#x\n", attr, sh); if (attr_hi == 0) { te.mtype = attr_lo == 0 ? TlbEntry::MemoryType::StronglyOrdered : TlbEntry::MemoryType::Device; te.outerAttrs = 0; te.innerAttrs = attr_lo == 0 ? 1 : 3; te.nonCacheable = true; } else { te.mtype = TlbEntry::MemoryType::Normal; te.outerAttrs = attr_hi == 1 ? 0 : attr_hi == 2 ? 2 : 1; te.innerAttrs = attr_lo == 1 ? 0 : attr_lo == 2 ? 6 : 5; // Treat write-through memory as uncacheable, this is safe // but for performance reasons not optimal. te.nonCacheable = (attr_hi == 1) || (attr_hi == 2) || (attr_lo == 1) || (attr_lo == 2); } } else { uint8_t attrIndx = lDescriptor.attrIndx(); DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh); // Select MAIR uint64_t mair; switch (currState->el) { case EL0: case EL1: mair = tc->readMiscReg(MISCREG_MAIR_EL1); break; case EL2: mair = tc->readMiscReg(MISCREG_MAIR_EL2); break; case EL3: mair = tc->readMiscReg(MISCREG_MAIR_EL3); break; default: panic("Invalid exception level"); break; } // Select attributes attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx); attr_lo = bits(attr, 3, 0); attr_hi = bits(attr, 7, 4); // Memory type te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal; // Cacheability te.nonCacheable = false; if (te.mtype == TlbEntry::MemoryType::Device) { // Device memory te.nonCacheable = true; } // Treat write-through memory as uncacheable, this is safe // but for performance reasons not optimal. switch (attr_hi) { case 0x1 ... 0x3: // Normal Memory, Outer Write-through transient case 0x4: // Normal memory, Outer Non-cacheable case 0x8 ... 0xb: // Normal Memory, Outer Write-through non-transient te.nonCacheable = true; } switch (attr_lo) { case 0x1 ... 0x3: // Normal Memory, Inner Write-through transient case 0x9 ... 0xb: // Normal Memory, Inner Write-through non-transient warn_if(!attr_hi, "Unpredictable behavior"); M5_FALLTHROUGH; case 0x4: // Device-nGnRE memory or // Normal memory, Inner Non-cacheable case 0x8: // Device-nGRE memory or // Normal memory, Inner Write-through non-transient te.nonCacheable = true; } te.shareable = sh == 2; te.outerShareable = (sh & 0x2) ? true : false; // Attributes formatted according to the 64-bit PAR te.attributes = ((uint64_t) attr << 56) | (1 << 11) | // LPAE bit (te.ns << 9) | // NS bit (sh << 7); } } void TableWalker::doL1Descriptor() { if (currState->fault != NoFault) { return; } currState->l1Desc.data = htog(currState->l1Desc.data, byteOrder(currState->tc)); DPRINTF(TLB, "L1 descriptor for %#x is %#x\n", currState->vaddr_tainted, currState->l1Desc.data); TlbEntry te; switch (currState->l1Desc.type()) { case L1Descriptor::Ignore: case L1Descriptor::Reserved: if (!currState->timing) { currState->tc = NULL; currState->req = NULL; } DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n"); if (currState->isFetch) currState->fault = std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L1, isStage2, ArmFault::VmsaTran); else currState->fault = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + L1, isStage2, ArmFault::VmsaTran); return; case L1Descriptor::Section: if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) { /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is * enabled if set, do l1.Desc.setAp0() instead of generating * AccessFlag0 */ currState->fault = std::make_shared( currState->vaddr_tainted, currState->l1Desc.domain(), currState->isWrite, ArmFault::AccessFlagLL + L1, isStage2, ArmFault::VmsaTran); } if (currState->l1Desc.supersection()) { panic("Haven't implemented supersections\n"); } insertTableEntry(currState->l1Desc, false); return; case L1Descriptor::PageTable: { Addr l2desc_addr; l2desc_addr = currState->l1Desc.l2Addr() | (bits(currState->vaddr, 19, 12) << 2); DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n", l2desc_addr, currState->isSecure ? "s" : "ns"); // Trickbox address check currState->fault = testWalk(l2desc_addr, sizeof(uint32_t), currState->l1Desc.domain(), L2); if (currState->fault) { if (!currState->timing) { currState->tc = NULL; currState->req = NULL; } return; } Request::Flags flag = Request::PT_WALK; if (currState->isSecure) flag.set(Request::SECURE); bool delayed; delayed = fetchDescriptor(l2desc_addr, (uint8_t*)&currState->l2Desc.data, sizeof(uint32_t), flag, -1, &doL2DescEvent, &TableWalker::doL2Descriptor); if (delayed) { currState->delayed = true; } return; } default: panic("A new type in a 2 bit field?\n"); } } void TableWalker::doLongDescriptor() { if (currState->fault != NoFault) { return; } currState->longDesc.data = htog(currState->longDesc.data, byteOrder(currState->tc)); DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n", currState->longDesc.lookupLevel, currState->vaddr_tainted, currState->longDesc.data, currState->aarch64 ? "AArch64" : "long-desc."); if ((currState->longDesc.type() == LongDescriptor::Block) || (currState->longDesc.type() == LongDescriptor::Page)) { DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, " "xn: %d, ap: %d, af: %d, type: %d\n", currState->longDesc.lookupLevel, currState->longDesc.data, currState->longDesc.pxn(), currState->longDesc.xn(), currState->longDesc.ap(), currState->longDesc.af(), currState->longDesc.type()); } else { DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n", currState->longDesc.lookupLevel, currState->longDesc.data, currState->longDesc.type()); } TlbEntry te; switch (currState->longDesc.type()) { case LongDescriptor::Invalid: if (!currState->timing) { currState->tc = NULL; currState->req = NULL; } DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n", currState->longDesc.lookupLevel, ArmFault::TranslationLL + currState->longDesc.lookupLevel); if (currState->isFetch) currState->fault = std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + currState->longDesc.lookupLevel, isStage2, ArmFault::LpaeTran); else currState->fault = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::TranslationLL + currState->longDesc.lookupLevel, isStage2, ArmFault::LpaeTran); return; case LongDescriptor::Block: case LongDescriptor::Page: { bool fault = false; bool aff = false; // Check for address size fault if (checkAddrSizeFaultAArch64( mbits(currState->longDesc.data, MaxPhysAddrRange - 1, currState->longDesc.offsetBits()), currState->physAddrRange)) { fault = true; DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", currState->longDesc.lookupLevel); // Check for access fault } else if (currState->longDesc.af() == 0) { fault = true; DPRINTF(TLB, "L%d descriptor causing Access Fault\n", currState->longDesc.lookupLevel); aff = true; } if (fault) { if (currState->isFetch) currState->fault = std::make_shared( currState->vaddr_tainted, (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + currState->longDesc.lookupLevel, isStage2, ArmFault::LpaeTran); else currState->fault = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + currState->longDesc.lookupLevel, isStage2, ArmFault::LpaeTran); } else { insertTableEntry(currState->longDesc, true); } } return; case LongDescriptor::Table: { // Set hierarchical permission flags currState->secureLookup = currState->secureLookup && currState->longDesc.secureTable(); currState->rwTable = currState->rwTable && currState->longDesc.rwTable(); currState->userTable = currState->userTable && currState->longDesc.userTable(); currState->xnTable = currState->xnTable || currState->longDesc.xnTable(); currState->pxnTable = currState->pxnTable || currState->longDesc.pxnTable(); // Set up next level lookup Addr next_desc_addr = currState->longDesc.nextDescAddr( currState->vaddr); DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n", currState->longDesc.lookupLevel, currState->longDesc.lookupLevel + 1, next_desc_addr, currState->secureLookup ? "s" : "ns"); // Check for address size fault if (currState->aarch64 && checkAddrSizeFaultAArch64( next_desc_addr, currState->physAddrRange)) { DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", currState->longDesc.lookupLevel); if (currState->isFetch) currState->fault = std::make_shared( currState->vaddr_tainted, ArmFault::AddressSizeLL + currState->longDesc.lookupLevel, isStage2, ArmFault::LpaeTran); else currState->fault = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::AddressSizeLL + currState->longDesc.lookupLevel, isStage2, ArmFault::LpaeTran); return; } // Trickbox address check currState->fault = testWalk( next_desc_addr, sizeof(uint64_t), TlbEntry::DomainType::Client, toLookupLevel(currState->longDesc.lookupLevel +1)); if (currState->fault) { if (!currState->timing) { currState->tc = NULL; currState->req = NULL; } return; } Request::Flags flag = Request::PT_WALK; if (currState->secureLookup) flag.set(Request::SECURE); LookupLevel L = currState->longDesc.lookupLevel = (LookupLevel) (currState->longDesc.lookupLevel + 1); Event *event = NULL; switch (L) { case L1: assert(currState->aarch64); case L2: case L3: event = LongDescEventByLevel[L]; break; default: panic("Wrong lookup level in table walk\n"); break; } bool delayed; delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data, sizeof(uint64_t), flag, -1, event, &TableWalker::doLongDescriptor); if (delayed) { currState->delayed = true; } } return; default: panic("A new type in a 2 bit field?\n"); } } void TableWalker::doL2Descriptor() { if (currState->fault != NoFault) { return; } currState->l2Desc.data = htog(currState->l2Desc.data, byteOrder(currState->tc)); DPRINTF(TLB, "L2 descriptor for %#x is %#x\n", currState->vaddr_tainted, currState->l2Desc.data); TlbEntry te; if (currState->l2Desc.invalid()) { DPRINTF(TLB, "L2 descriptor invalid, causing fault\n"); if (!currState->timing) { currState->tc = NULL; currState->req = NULL; } if (currState->isFetch) currState->fault = std::make_shared( currState->vaddr_tainted, ArmFault::TranslationLL + L2, isStage2, ArmFault::VmsaTran); else currState->fault = std::make_shared( currState->vaddr_tainted, currState->l1Desc.domain(), currState->isWrite, ArmFault::TranslationLL + L2, isStage2, ArmFault::VmsaTran); return; } if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) { /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled * if set, do l2.Desc.setAp0() instead of generating AccessFlag0 */ DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n", currState->sctlr.afe, currState->l2Desc.ap()); currState->fault = std::make_shared( currState->vaddr_tainted, TlbEntry::DomainType::NoAccess, currState->isWrite, ArmFault::AccessFlagLL + L2, isStage2, ArmFault::VmsaTran); } insertTableEntry(currState->l2Desc, false); } void TableWalker::doL1DescriptorWrapper() { currState = stateQueues[L1].front(); currState->delayed = false; // if there's a stage2 translation object we don't need it any more if (currState->stage2Tran) { delete currState->stage2Tran; currState->stage2Tran = NULL; } DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data); DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data); DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted); doL1Descriptor(); stateQueues[L1].pop_front(); // Check if fault was generated if (currState->fault != NoFault) { currState->transState->finish(currState->fault, currState->req, currState->tc, currState->mode); statWalksShortTerminatedAtLevel[0]++; pending = false; nextWalk(currState->tc); currState->req = NULL; currState->tc = NULL; currState->delayed = false; delete currState; } else if (!currState->delayed) { // delay is not set so there is no L2 to do // Don't finish the translation if a stage 2 look up is underway statWalkServiceTime.sample(curTick() - currState->startTime); DPRINTF(TLBVerbose, "calling translateTiming again\n"); tlb->translateTiming(currState->req, currState->tc, currState->transState, currState->mode); statWalksShortTerminatedAtLevel[0]++; pending = false; nextWalk(currState->tc); currState->req = NULL; currState->tc = NULL; currState->delayed = false; delete currState; } else { // need to do L2 descriptor stateQueues[L2].push_back(currState); } currState = NULL; } void TableWalker::doL2DescriptorWrapper() { currState = stateQueues[L2].front(); assert(currState->delayed); // if there's a stage2 translation object we don't need it any more if (currState->stage2Tran) { delete currState->stage2Tran; currState->stage2Tran = NULL; } DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n", currState->vaddr_tainted); doL2Descriptor(); // Check if fault was generated if (currState->fault != NoFault) { currState->transState->finish(currState->fault, currState->req, currState->tc, currState->mode); statWalksShortTerminatedAtLevel[1]++; } else { statWalkServiceTime.sample(curTick() - currState->startTime); DPRINTF(TLBVerbose, "calling translateTiming again\n"); tlb->translateTiming(currState->req, currState->tc, currState->transState, currState->mode); statWalksShortTerminatedAtLevel[1]++; } stateQueues[L2].pop_front(); pending = false; nextWalk(currState->tc); currState->req = NULL; currState->tc = NULL; currState->delayed = false; delete currState; currState = NULL; } void TableWalker::doL0LongDescriptorWrapper() { doLongDescriptorWrapper(L0); } void TableWalker::doL1LongDescriptorWrapper() { doLongDescriptorWrapper(L1); } void TableWalker::doL2LongDescriptorWrapper() { doLongDescriptorWrapper(L2); } void TableWalker::doL3LongDescriptorWrapper() { doLongDescriptorWrapper(L3); } void TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level) { currState = stateQueues[curr_lookup_level].front(); assert(curr_lookup_level == currState->longDesc.lookupLevel); currState->delayed = false; // if there's a stage2 translation object we don't need it any more if (currState->stage2Tran) { delete currState->stage2Tran; currState->stage2Tran = NULL; } DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n", currState->vaddr_tainted); doLongDescriptor(); stateQueues[curr_lookup_level].pop_front(); if (currState->fault != NoFault) { // A fault was generated currState->transState->finish(currState->fault, currState->req, currState->tc, currState->mode); pending = false; nextWalk(currState->tc); currState->req = NULL; currState->tc = NULL; currState->delayed = false; delete currState; } else if (!currState->delayed) { // No additional lookups required DPRINTF(TLBVerbose, "calling translateTiming again\n"); statWalkServiceTime.sample(curTick() - currState->startTime); tlb->translateTiming(currState->req, currState->tc, currState->transState, currState->mode); statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++; pending = false; nextWalk(currState->tc); currState->req = NULL; currState->tc = NULL; currState->delayed = false; delete currState; } else { if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1) panic("Max. number of lookups already reached in table walk\n"); // Need to perform additional lookups stateQueues[currState->longDesc.lookupLevel].push_back(currState); } currState = NULL; } void TableWalker::nextWalk(ThreadContext *tc) { if (pendingQueue.size()) schedule(doProcessEvent, clockEdge(Cycles(1))); else completeDrain(); } bool TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes, Request::Flags flags, int queueIndex, Event *event, void (TableWalker::*doDescriptor)()) { bool isTiming = currState->timing; DPRINTF(TLBVerbose, "Fetching descriptor at address: 0x%x stage2Req: %d\n", descAddr, currState->stage2Req); // If this translation has a stage 2 then we know descAddr is an IPA and // needs to be translated before we can access the page table. Do that // check here. if (currState->stage2Req) { Fault fault; flags = flags | TLB::MustBeOne; if (isTiming) { Stage2MMU::Stage2Translation *tran = new Stage2MMU::Stage2Translation(*stage2Mmu, data, event, currState->vaddr); currState->stage2Tran = tran; stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes, flags); fault = tran->fault; } else { fault = stage2Mmu->readDataUntimed(currState->tc, currState->vaddr, descAddr, data, numBytes, flags, currState->functional); } if (fault != NoFault) { currState->fault = fault; } if (isTiming) { if (queueIndex >= 0) { DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", stateQueues[queueIndex].size()); stateQueues[queueIndex].push_back(currState); currState = NULL; } } else { (this->*doDescriptor)(); } } else { if (isTiming) { port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data, currState->tc->getCpuPtr()->clockPeriod(),flags); if (queueIndex >= 0) { DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", stateQueues[queueIndex].size()); stateQueues[queueIndex].push_back(currState); currState = NULL; } } else if (!currState->functional) { port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data, currState->tc->getCpuPtr()->clockPeriod(), flags); (this->*doDescriptor)(); } else { RequestPtr req = std::make_shared( descAddr, numBytes, flags, masterId); req->taskId(ContextSwitchTaskId::DMA); PacketPtr pkt = new Packet(req, MemCmd::ReadReq); pkt->dataStatic(data); port->sendFunctional(pkt); (this->*doDescriptor)(); delete pkt; } } return (isTiming); } void TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor) { TlbEntry te; // Create and fill a new page table entry te.valid = true; te.longDescFormat = longDescriptor; te.isHyp = currState->isHyp; te.asid = currState->asid; te.vmid = currState->vmid; te.N = descriptor.offsetBits(); te.vpn = currState->vaddr >> te.N; te.size = (1<isSecure; te.xn = descriptor.xn(); if (currState->aarch64) te.el = currState->el; else te.el = 1; statPageSizes[pageSizeNtoStatBin(te.N)]++; statRequestOrigin[COMPLETED][currState->isFetch]++; // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries // as global te.global = descriptor.global(currState) || isStage2; if (longDescriptor) { LongDescriptor lDescriptor = dynamic_cast(descriptor); te.xn |= currState->xnTable; te.pxn = currState->pxnTable || lDescriptor.pxn(); if (isStage2) { // this is actually the HAP field, but its stored in the same bit // possitions as the AP field in a stage 1 translation. te.hap = lDescriptor.ap(); } else { te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) | (currState->userTable && (descriptor.ap() & 0x1)); } if (currState->aarch64) memAttrsAArch64(currState->tc, te, lDescriptor); else memAttrsLPAE(currState->tc, te, lDescriptor); } else { te.ap = descriptor.ap(); memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(), descriptor.shareable()); } // Debug output DPRINTF(TLB, descriptor.dbgHeader().c_str()); DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n", te.N, te.pfn, te.size, te.global, te.valid); DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d " "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn, te.ap, static_cast(te.domain), te.asid, te.vmid, te.isHyp, te.nonCacheable, te.ns); DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n", descriptor.lookupLevel, static_cast(descriptor.domain()), descriptor.getRawData()); // Insert the entry into the TLB tlb->insert(currState->vaddr, te); if (!currState->timing) { currState->tc = NULL; currState->req = NULL; } } ArmISA::TableWalker * ArmTableWalkerParams::create() { return new ArmISA::TableWalker(this); } LookupLevel TableWalker::toLookupLevel(uint8_t lookup_level_as_int) { switch (lookup_level_as_int) { case L1: return L1; case L2: return L2; case L3: return L3; default: panic("Invalid lookup level conversion"); } } /* this method keeps track of the table walker queue's residency, so * needs to be called whenever requests start and complete. */ void TableWalker::pendingChange() { unsigned n = pendingQueue.size(); if ((currState != NULL) && (currState != pendingQueue.front())) { ++n; } if (n != pendingReqs) { Tick now = curTick(); statPendingWalks.sample(pendingReqs, now - pendingChangeTick); pendingReqs = n; pendingChangeTick = now; } } Fault TableWalker::testWalk(Addr pa, Addr size, TlbEntry::DomainType domain, LookupLevel lookup_level) { return tlb->testWalk(pa, size, currState->vaddr, currState->isSecure, currState->mode, domain, lookup_level); } uint8_t TableWalker::pageSizeNtoStatBin(uint8_t N) { /* for statPageSizes */ switch(N) { case 12: return 0; // 4K case 14: return 1; // 16K (using 16K granule in v8-64) case 16: return 2; // 64K case 20: return 3; // 1M case 21: return 4; // 2M-LPAE case 24: return 5; // 16M case 25: return 6; // 32M (using 16K granule in v8-64) case 29: return 7; // 512M (using 64K granule in v8-64) case 30: return 8; // 1G-LPAE default: panic("unknown page size"); return 255; } } void TableWalker::regStats() { ClockedObject::regStats(); statWalks .name(name() + ".walks") .desc("Table walker walks requested") ; statWalksShortDescriptor .name(name() + ".walksShort") .desc("Table walker walks initiated with short descriptors") .flags(Stats::nozero) ; statWalksLongDescriptor .name(name() + ".walksLong") .desc("Table walker walks initiated with long descriptors") .flags(Stats::nozero) ; statWalksShortTerminatedAtLevel .init(2) .name(name() + ".walksShortTerminationLevel") .desc("Level at which table walker walks " "with short descriptors terminate") .flags(Stats::nozero) ; statWalksShortTerminatedAtLevel.subname(0, "Level1"); statWalksShortTerminatedAtLevel.subname(1, "Level2"); statWalksLongTerminatedAtLevel .init(4) .name(name() + ".walksLongTerminationLevel") .desc("Level at which table walker walks " "with long descriptors terminate") .flags(Stats::nozero) ; statWalksLongTerminatedAtLevel.subname(0, "Level0"); statWalksLongTerminatedAtLevel.subname(1, "Level1"); statWalksLongTerminatedAtLevel.subname(2, "Level2"); statWalksLongTerminatedAtLevel.subname(3, "Level3"); statSquashedBefore .name(name() + ".walksSquashedBefore") .desc("Table walks squashed before starting") .flags(Stats::nozero) ; statSquashedAfter .name(name() + ".walksSquashedAfter") .desc("Table walks squashed after completion") .flags(Stats::nozero) ; statWalkWaitTime .init(16) .name(name() + ".walkWaitTime") .desc("Table walker wait (enqueue to first request) latency") .flags(Stats::pdf | Stats::nozero | Stats::nonan) ; statWalkServiceTime .init(16) .name(name() + ".walkCompletionTime") .desc("Table walker service (enqueue to completion) latency") .flags(Stats::pdf | Stats::nozero | Stats::nonan) ; statPendingWalks .init(16) .name(name() + ".walksPending") .desc("Table walker pending requests distribution") .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan) ; statPageSizes // see DDI 0487A D4-1661 .init(9) .name(name() + ".walkPageSizes") .desc("Table walker page sizes translated") .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero) ; statPageSizes.subname(0, "4K"); statPageSizes.subname(1, "16K"); statPageSizes.subname(2, "64K"); statPageSizes.subname(3, "1M"); statPageSizes.subname(4, "2M"); statPageSizes.subname(5, "16M"); statPageSizes.subname(6, "32M"); statPageSizes.subname(7, "512M"); statPageSizes.subname(8, "1G"); statRequestOrigin .init(2,2) // Instruction/Data, requests/completed .name(name() + ".walkRequestOrigin") .desc("Table walker requests started/completed, data/inst") .flags(Stats::total) ; statRequestOrigin.subname(0,"Requested"); statRequestOrigin.subname(1,"Completed"); statRequestOrigin.ysubname(0,"Data"); statRequestOrigin.ysubname(1,"Inst"); }