diff --git a/.github/workflows/autotest/gem5.cfg b/.github/workflows/autotest/gem5.cfg index 435b2d1101..2a8dcb5ff3 100644 --- a/.github/workflows/autotest/gem5.cfg +++ b/.github/workflows/autotest/gem5.cfg @@ -15,7 +15,7 @@ log_root = ./log_root compile_thread = 70 #gcpt路径 gcpt_path = /nfs/home/share/gem5_shared_tools/normal-gcb-restorer.bin -ref_so_path = /nfs/home/share/gem5_shared_tools/riscv64-nemu-4332a525-so +ref_so_path = /nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so set_var = export NEMU_HOME={ref_so_path} diff --git a/.github/workflows/gem5.yml b/.github/workflows/gem5.yml index 036f90e134..6b370b9122 100644 --- a/.github/workflows/gem5.yml +++ b/.github/workflows/gem5.yml @@ -74,7 +74,7 @@ jobs: run: CC=gcc CXX=g++ scons build/RISCV/gem5.opt --linker=gold -j64 - name: XS-GEM5 - Test xiangshan.py simulation scripts run: | - export GCBV_REF_SO="/nfs/home/share/gem5_shared_tools/riscv64-nemu-4332a525-so" + export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so" export GCB_RESTORER="/nfs/home/share/gem5_shared_tools/normal-gcb-restorer.bin" export GEM5_HOME=$(pwd) mkdir -p $GEM5_HOME/util/xs_scripts/test @@ -101,7 +101,7 @@ jobs: run: CC=gcc CXX=g++ scons build/RISCV/gem5.opt --linker=gold -j64 - name: XS-GEM5 - Test xiangshan.py simulation scripts run: | - export GCBV_REF_SO="/nfs/home/share/gem5_shared_tools/riscv64-nemu-4332a525-so" + export GCBV_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build/riscv64-nemu-interpreter-so" export GCBV_RESTORER="/nfs/home/share/gem5_shared_tools/gcbv-restorer.bin" export GEM5_HOME=$(pwd) mkdir -p $GEM5_HOME/util/xs_scripts/test_v @@ -120,7 +120,7 @@ jobs: CC=clang CXX=clang++ scons build/RISCV_CHI/gem5.opt -j 48 --gold-linker - name: XS-GEM5 - Test xiangshan.py simulation scripts run: | - export GCBV_MULTI_CORE_REF_SO="/nfs/home/share/gem5_shared_tools/riscv64-dualcore-nemu-4332a525-so" + export GCBV_MULTI_CORE_REF_SO="/nfs-nvme/home/share/zhenhao/ref-h/build-multi/riscv64-nemu-interpreter-so" export GCB_MULTI_CORE_RESTORER="/nfs/home/share/gem5_shared_tools/gcb-2core-restorer.bin" export GEM5_HOME=$(pwd) mkdir -p $GEM5_HOME/util/xs_scripts/test_multi_core diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py index ac2d0d4e4f..d5c863fc41 100644 --- a/configs/common/FSConfig.py +++ b/configs/common/FSConfig.py @@ -686,12 +686,16 @@ def makeBareMetalXiangshanSystem(mem_mode, mdesc=None, cmdline=None, np=1, ruby= self.mmcs = NemuMMC() self.mmcs.pio = self.iobus.mem_side_ports + self.plic = NemuPlic() + self.plic.pio = self.iobus.mem_side_ports + if not ruby: self.bridge.ranges = [ AddrRange(self.uartlite.pio_addr, self.uartlite.pio_addr + self.uartlite.pio_size), AddrRange(self.lint.pio_addr, self.lint.pio_addr + self.lint.pio_size), AddrRange(self.mmcs.pio_addr, self.mmcs.pio_addr + self.mmcs.pio_size), + AddrRange(self.plic.pio_addr, self.plic.pio_addr + self.plic.pio_size), ] self.workload.reset_vect = 0x80000000 diff --git a/src/arch/amdgpu/vega/faults.hh b/src/arch/amdgpu/vega/faults.hh index 67c5bfde25..4b0a9aaf74 100644 --- a/src/arch/amdgpu/vega/faults.hh +++ b/src/arch/amdgpu/vega/faults.hh @@ -62,7 +62,7 @@ class VegaFault : public FaultBase FaultName name() const override { return _name; } bool isInterrupt() const { return _interrupt; } - ExceptionCode exception() const { return _code; } + uint64_t exception() const override{ return _code; } virtual RegVal trap_value() const { return 0; } void invoke(ThreadContext *tc, const StaticInstPtr &inst) override; diff --git a/src/arch/generic/SConscript b/src/arch/generic/SConscript index e0fd998f21..aa14207825 100644 --- a/src/arch/generic/SConscript +++ b/src/arch/generic/SConscript @@ -51,6 +51,8 @@ DebugFlag('PageTableWalker', "Page table walker state machine debugging") DebugFlag('PageTableWalker2', "Page table walker state machine debugging") +DebugFlag('PageTableWalkerTwoStage', + "Page table walker state machine debugging") DebugFlag('PageTableWalker3', "Page table walker state machine debugging") DebugFlag('autoNextline', diff --git a/src/arch/generic/mmu.hh b/src/arch/generic/mmu.hh index d090f6de58..e4b241d02c 100644 --- a/src/arch/generic/mmu.hh +++ b/src/arch/generic/mmu.hh @@ -53,7 +53,7 @@ class BaseTLB; class BaseMMU : public SimObject { public: - enum Mode { Read, Write, Execute }; + enum Mode { Read, Write, Execute}; class Translation { diff --git a/src/arch/riscv/faults.cc b/src/arch/riscv/faults.cc index f72158c529..2d4cb8406b 100644 --- a/src/arch/riscv/faults.cc +++ b/src/arch/riscv/faults.cc @@ -65,7 +65,15 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) if (FullSystem) { PrivilegeMode pp = (PrivilegeMode)tc->readMiscReg(MISCREG_PRV); PrivilegeMode prv = PRV_M; + PrivilegeMode prvh = PRV_M; STATUS status = tc->readMiscReg(MISCREG_STATUS); + HSTATUS hstatus = tc->readMiscReg(MISCREG_HSTATUS); + VSSTATUS vsstatus = tc->readMiscReg(MISCREG_VSSTATUS); + auto vmode = tc->readMiscReg(MISCREG_VIRMODE); + bool hs_mode_pc = false; + if ((_code == ECALL_SUPER) && vmode){ + _code = ECALL_VS; + } // According to riscv-privileged-v1.11, if a NMI occurs at the middle // of a M-mode trap handler, the state (epc/cause) will be overwritten @@ -97,8 +105,21 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) } } + if (isInterrupt()) { + if (pp != PRV_M && (tc->readMiscReg(MISCREG_VIRMODE) == 1) && + bits(tc->readMiscReg(MISCREG_HIDELEG), _code) != 0) { + prvh = PRV_HS; + } + } else { + if (pp != PRV_M && (tc->readMiscReg(MISCREG_VIRMODE) == 1) && + bits(tc->readMiscReg(MISCREG_HEDELEG), _code) != 0) { + prvh = PRV_HS; + } + } + // Set fault registers and status MiscRegIndex cause, epc, tvec, tval; + int v = status.mprv ? status.mpv : tc->readMiscReg(MISCREG_VIRMODE); switch (prv) { case PRV_U: cause = MISCREG_UCAUSE; @@ -110,14 +131,55 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) status.uie = 0; break; case PRV_S: - cause = MISCREG_SCAUSE; - epc = MISCREG_SEPC; - tvec = MISCREG_STVEC; - tval = MISCREG_STVAL; - - status.spp = pp; - status.spie = status.sie; - status.sie = 0; + if (prvh == PRV_HS) { + cause = MISCREG_SCAUSE; + epc = MISCREG_SEPC; + tvec = MISCREG_STVEC; + //tval = MISCREG_STVAL; + tval = MISCREG_VSTVAL; + + if (hstatus.vsxl == 1) { + assert(0); + + } else { + vsstatus.spp = pp; + vsstatus.spie = vsstatus.sie; + vsstatus.sie = 0; + } + + tc->setMiscReg(MISCREG_VIRMODE, 1); + if (isInterrupt()) + tc->setMiscReg(MISCREG_VSCAUSE, _code | (1L << 63)); + else + tc->setMiscReg(MISCREG_VSCAUSE, _code); + tc->setMiscReg(MISCREG_VSEPC, tc->pcState().instAddr()); + hs_mode_pc = true; + } else { + cause = MISCREG_SCAUSE; + epc = MISCREG_SEPC; + tvec = MISCREG_STVEC; + tval = MISCREG_STVAL; + + status.spp = pp; + status.spie = status.sie; + status.sie = 0; + hstatus.gva = + (!isInterrupt()) && (_code == INST_G_PAGE || _code == LOAD_G_PAGE || _code == STORE_G_PAGE || + ((v) && ((0 <= _code && _code <= 7 && _code != 2) || _code == INST_PAGE || + _code == LOAD_PAGE || _code == STORE_PAGE))); + hstatus.spv = v; + if (v) { + hstatus.spvp = pp; + } + if ((_code == INST_PAGE || _code == LOAD_PAGE || _code == STORE_PAGE || + _code == LOAD_ADDR_MISALIGNED || _code == STORE_ADDR_MISALIGNED || _code == INST_ACCESS || + _code == LOAD_ACCESS || _code == STORE_ACCESS) || + ((_code != INST_G_PAGE) && (_code != LOAD_G_PAGE) && (_code != STORE_G_PAGE))) { + tc->setMiscReg(MISCREG_HTVAL, 0); + } + + tc->setMiscReg(MISCREG_VIRMODE, 0); + } break; case PRV_M: cause = MISCREG_MCAUSE; @@ -128,6 +190,12 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) status.mpp = pp; status.mpie = status.mie; status.mie = 0; + status.gva = + (!isInterrupt()) && (_code == INST_G_PAGE || _code == LOAD_G_PAGE || _code == STORE_G_PAGE || + ((v) && ((0 <= _code && _code <= 7 && _code != 2) || _code == INST_PAGE || + _code == LOAD_PAGE || _code == STORE_PAGE))); + status.mpv = v; + tc->setMiscReg(MISCREG_VIRMODE, 0); break; default: panic("Unknown privilege mode %d.", prv); @@ -140,14 +208,28 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) if (isInterrupt()) { _cause |= (1L << 63); } - tc->setMiscReg(cause, _cause); - tc->setMiscReg(epc, tc->pcState().instAddr()); + + if (!hs_mode_pc) { + tc->setMiscReg(cause, _cause); + tc->setMiscReg(epc, tc->pcState().instAddr()); + } + if (_cause == INST_ILLEGAL) tc->setMiscReg(tval, 0); else tc->setMiscReg(tval, trap_value()); + if (_code == INST_G_PAGE || _code == LOAD_G_PAGE || _code == STORE_G_PAGE) { + if (prv == PRV_S && (g_trap_value() != 0)) { + tc->setMiscReg(MISCREG_HTVAL, g_trap_value() >> 2); + } else if (g_trap_value() != 0) { + tc->setMiscReg(MISCREG_MTVAL2, g_trap_value() >> 2); + } + } + tc->setMiscReg(MISCREG_PRV, prv); tc->setMiscReg(MISCREG_STATUS, status); + tc->setMiscReg(MISCREG_HSTATUS, hstatus); + tc->setMiscRegNoEffect(MISCREG_VSSTATUS, vsstatus); // Temporarily mask NMI while we're in NMI handler. Otherweise, the // checkNonMaskableInterrupt will always return true and we'll be // stucked in an infinite loop. @@ -156,8 +238,16 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) } // Set PC to fault handler address - Addr addr = mbits(tc->readMiscReg(tvec), 63, 2); - if (isInterrupt() && bits(tc->readMiscReg(tvec), 1, 0) == 1) + Addr addr = 0; + auto addr_tvec = tc->readMiscReg(tvec); + if (!hs_mode_pc) { + addr = mbits(tc->readMiscReg(tvec), 63, 2); + } else { + addr = mbits(tc->readMiscReg(MISCREG_VSTVEC), 63, 2); + addr_tvec = tc->readMiscReg(MISCREG_VSTVEC); + } + + if (isInterrupt() && bits(addr_tvec, 1, 0) == 1) addr += 4 * _code; pc_state.set(addr); tc->pcState(pc_state); @@ -225,5 +315,11 @@ SyscallFault::invokeSE(ThreadContext *tc, const StaticInstPtr &inst) tc->getSystemPtr()->workload->syscall(tc); } +void +HVFault::invokeSE(ThreadContext *tc, const StaticInstPtr &inst) +{ + panic("HVFault at pc %s", tc->pcState()); +} + } // namespace RiscvISA } // namespace gem5 diff --git a/src/arch/riscv/faults.hh b/src/arch/riscv/faults.hh index 6cca267dbf..06d762c093 100644 --- a/src/arch/riscv/faults.hh +++ b/src/arch/riscv/faults.hh @@ -78,11 +78,17 @@ enum ExceptionCode : uint64_t AMO_ACCESS = 7, ECALL_USER = 8, ECALL_SUPER = 9, + ECALL_VS = 10, // ecall from vs-mode h-extention ECALL_MACHINE = 11, INST_PAGE = 12, LOAD_PAGE = 13, STORE_PAGE = 15, AMO_PAGE = 15, + INST_G_PAGE = 20, + LOAD_G_PAGE = 21, + VIRTUAL_INST = 22, + STORE_G_PAGE = 23, + AMO_G_PAGE = 23, INT_SOFTWARE_USER = 0, INT_SOFTWARE_SUPER = 1, @@ -120,8 +126,9 @@ class RiscvFault : public FaultBase { return _fault_type == FaultType::NON_MASKABLE_INTERRUPT; } - ExceptionCode exception() const { return _code; } + uint64_t exception() const override{ return _code; } virtual RegVal trap_value() const { return 0; } + virtual RegVal g_trap_value() const { return 0; } virtual void invokeSE(ThreadContext *tc, const StaticInstPtr &inst); void invoke(ThreadContext *tc, const StaticInstPtr &inst) override; @@ -228,13 +235,15 @@ class AddressFault : public RiscvFault { private: const Addr _addr; + const Addr _gPaddr; public: - AddressFault(const Addr addr, ExceptionCode code) - : RiscvFault("Address", FaultType::OTHERS, code), _addr(addr) + AddressFault(const Addr addr,const Addr gPaddr, ExceptionCode code) + : RiscvFault("Address", FaultType::OTHERS, code), _addr(addr),_gPaddr(gPaddr) {} RegVal trap_value() const override { return _addr; } + RegVal g_trap_value() const override { return _gPaddr; } }; class BreakpointFault : public RiscvFault @@ -277,6 +286,12 @@ class SyscallFault : public RiscvFault void invokeSE(ThreadContext *tc, const StaticInstPtr &inst) override; }; +class HVFault : public RiscvFault +{ + public: + HVFault() : RiscvFault("System call", FaultType::OTHERS, VIRTUAL_INST) {} + void invokeSE(ThreadContext *tc, const StaticInstPtr &inst) override; +}; } // namespace RiscvISA } // namespace gem5 diff --git a/src/arch/riscv/insts/standard.hh b/src/arch/riscv/insts/standard.hh index 92a2cfec6a..38dbd913bf 100644 --- a/src/arch/riscv/insts/standard.hh +++ b/src/arch/riscv/insts/standard.hh @@ -129,8 +129,9 @@ class CSROp : public RiscvStaticInst auto mask_it = CSRMasks.find(csr); if (mask_it == CSRMasks.end()) maskVal = mask(64); - else + else { maskVal = mask_it->second; + } } if (csr == CSR_SATP) { diff --git a/src/arch/riscv/isa.cc b/src/arch/riscv/isa.cc index ae0879165b..d9c3997bd7 100644 --- a/src/arch/riscv/isa.cc +++ b/src/arch/riscv/isa.cc @@ -69,6 +69,7 @@ namespace RiscvISA [[maybe_unused]] const std::array MiscRegNames = {{ [MISCREG_PRV] = "PRV", + [MISCREG_VIRMODE] = "VIRTUALIZATIONMODE", [MISCREG_ISA] = "ISA", [MISCREG_VENDORID] = "VENDORID", [MISCREG_ARCHID] = "ARCHID", @@ -80,6 +81,35 @@ namespace RiscvISA [MISCREG_CYCLE] = "CYCLE", [MISCREG_TIME] = "TIME", [MISCREG_INSTRET] = "INSTRET", + [MISCREG_MHPMCOUNTER3] = "MHPMCOUNTER3", + [MISCREG_MHPMCOUNTER4] = "MHPMCOUNTER4", + [MISCREG_MHPMCOUNTER5] = "MHPMCOUNTER5", + [MISCREG_MHPMCOUNTER6] = "MHPMCOUNTER6", + [MISCREG_MHPMCOUNTER7] = "MHPMCOUNTER7", + [MISCREG_MHPMCOUNTER8] = "MHPMCOUNTER8", + [MISCREG_MHPMCOUNTER9] = "MHPMCOUNTER9", + [MISCREG_MHPMCOUNTER10] = "MHPMCOUNTER10", + [MISCREG_MHPMCOUNTER11] = "MHPMCOUNTER11", + [MISCREG_MHPMCOUNTER12] = "MHPMCOUNTER12", + [MISCREG_MHPMCOUNTER13] = "MHPMCOUNTER13", + [MISCREG_MHPMCOUNTER14] = "MHPMCOUNTER14", + [MISCREG_MHPMCOUNTER15] = "MHPMCOUNTER15", + [MISCREG_MHPMCOUNTER16] = "MHPMCOUNTER16", + [MISCREG_MHPMCOUNTER17] = "MHPMCOUNTER17", + [MISCREG_MHPMCOUNTER18] = "MHPMCOUNTER18", + [MISCREG_MHPMCOUNTER19] = "MHPMCOUNTER19", + [MISCREG_MHPMCOUNTER20] = "MHPMCOUNTER20", + [MISCREG_MHPMCOUNTER21] = "MHPMCOUNTER21", + [MISCREG_MHPMCOUNTER22] = "MHPMCOUNTER22", + [MISCREG_MHPMCOUNTER23] = "MHPMCOUNTER23", + [MISCREG_MHPMCOUNTER24] = "MHPMCOUNTER24", + [MISCREG_MHPMCOUNTER25] = "MHPMCOUNTER25", + [MISCREG_MHPMCOUNTER26] = "MHPMCOUNTER26", + [MISCREG_MHPMCOUNTER27] = "MHPMCOUNTER27", + [MISCREG_MHPMCOUNTER28] = "MHPMCOUNTER28", + [MISCREG_MHPMCOUNTER29] = "MHPMCOUNTER29", + [MISCREG_MHPMCOUNTER30] = "MHPMCOUNTER30", + [MISCREG_MHPMCOUNTER31] = "MHPMCOUNTER31", [MISCREG_HPMCOUNTER03] = "HPMCOUNTER03", [MISCREG_HPMCOUNTER04] = "HPMCOUNTER04", [MISCREG_HPMCOUNTER05] = "HPMCOUNTER05", @@ -109,6 +139,7 @@ namespace RiscvISA [MISCREG_HPMCOUNTER29] = "HPMCOUNTER29", [MISCREG_HPMCOUNTER30] = "HPMCOUNTER30", [MISCREG_HPMCOUNTER31] = "HPMCOUNTER31", + [MISCREG_MCOUNTINHIBIT] = "MCOUNTINHIBIT", [MISCREG_HPMEVENT03] = "HPMEVENT03", [MISCREG_HPMEVENT04] = "HPMEVENT04", [MISCREG_HPMEVENT05] = "HPMEVENT05", @@ -139,8 +170,6 @@ namespace RiscvISA [MISCREG_HPMEVENT30] = "HPMEVENT30", [MISCREG_HPMEVENT31] = "HPMEVENT31", - [MISCREG_MTINST] = "MTINST", - [MISCREG_MTVAL2] = "MTVAL2", [MISCREG_TSELECT] = "TSELECT", [MISCREG_TDATA1] = "TDATA1", @@ -154,6 +183,7 @@ namespace RiscvISA [MISCREG_MIDELEG] = "MIDELEG", [MISCREG_MTVEC] = "MTVEC", [MISCREG_MCOUNTEREN] = "MCOUNTEREN", + [MISCREG_MENVCFG] = "MENVCFG", [MISCREG_MSCRATCH] = "MSCRATCH", [MISCREG_MEPC] = "MEPC", [MISCREG_MCAUSE] = "MCAUSE", @@ -205,6 +235,33 @@ namespace RiscvISA [MISCREG_VTYPE] = "VTYPE", [MISCREG_VLENB] = "VLENB", + [MISCREG_HSTATUS] = "HSTATUS", + [MISCREG_HEDELEG] = "HEDELEG", + [MISCREG_HIDELEG] = "HIDELEG", + [MISCREG_HIE] = "HIE", + [MISCREG_HCOUNTEREN] = "HCOUNTEREN", + [MISCREG_HGEIE] = "HGEIE", + [MISCREG_HTVAL] = "HTVAL", + [MISCREG_HIP] = "HIP", + [MISCREG_HVIP] = "HVIP", + [MISCREG_HTINST] = "HTINST", + [MISCREG_HGEIP] = "HGEIP", + [MISCREG_HENVCFG] = "HENVCFG", + [MISCREG_HGATP] = "HGATP", + [MISCREG_HTIMEDELTA] = "HTIMEDELTA", + [MISCREG_VSSTATUS] = "VSSTATUS", + [MISCREG_VSIE] = "VSIE", + [MISCREG_VSTVEC] = "VSTVEC", + [MISCREG_VSSCRATCH] = "VSSCRATCH", + [MISCREG_VSEPC] = "VSEPC", + [MISCREG_VSCAUSE] = "VSCAUSE", + [MISCREG_VSTVAL] = "VSTVAL", + [MISCREG_VSIP] = "VSIP", + [MISCREG_VSATP] = "VSATP", + [MISCREG_MTINST] = "MTINST", + [MISCREG_MTVAL2] = "MTVAL2", + + [MISCREG_NMIVEC] = "NMIVEC", [MISCREG_NMIE] = "NMIE", [MISCREG_NMIP] = "NMIP", @@ -261,7 +318,7 @@ void ISA::clear() std::fill(miscRegFile.begin(), miscRegFile.end(), 0); miscRegFile[MISCREG_PRV] = PRV_M; - miscRegFile[MISCREG_ISA] = 0x800000000034112f; + miscRegFile[MISCREG_ISA] = 0x80000000003411af; miscRegFile[MISCREG_VENDORID] = 0; miscRegFile[MISCREG_ARCHID] = 0; miscRegFile[MISCREG_IMPID] = 0; @@ -273,8 +330,8 @@ void ISA::clear() miscRegFile[MISCREG_STATUS] = (2ULL << UXL_OFFSET) | (2ULL << SXL_OFFSET) | (1ULL << FS_OFFSET); } - miscRegFile[MISCREG_MCOUNTEREN] = 0x7; - miscRegFile[MISCREG_SCOUNTEREN] = 0x7; + miscRegFile[MISCREG_MCOUNTEREN] = 0x0; + miscRegFile[MISCREG_SCOUNTEREN] = 0; // don't set it to zero; software may try to determine the supported // triggers, starting at zero. simply set a different value here. miscRegFile[MISCREG_TSELECT] = 1; @@ -282,6 +339,10 @@ void ISA::clear() miscRegFile[MISCREG_NMIE] = 1; miscRegFile[MISCREG_VTYPE] = (1lu<<63); + miscRegFile[MISCREG_HSTATUS] = (uint64_t)2<<32; + miscRegFile[MISCREG_VSSTATUS] = miscRegFile[MISCREG_STATUS] & NEMU_SSTATUS_RMASK; + miscRegFile[MISCREG_ARCHID] = 0x19; + } bool @@ -310,7 +371,18 @@ ISA::hpmCounterEnabled(int misc_reg) const RegVal ISA::readMiscRegNoEffect(int misc_reg) const { - if (misc_reg > NUM_MISCREGS || misc_reg < 0) { + if ((misc_reg >= RiscvISA::MiscRegIndex::MISCREG_PMPADDR00) && + (misc_reg < RiscvISA::MiscRegIndex::MISCREG_PMPADDR00 + 16)) { + auto mmu = dynamic_cast(tc->getMMUPtr()); + uint32_t pmp_index = misc_reg - MISCREG_PMPADDR00; + uint64_t csr_num = mmu->getPMP()->pmpcfg_from_index(pmp_index); + if (mmu->getPMP()->pmp_read_config(csr_num)) { + return miscRegFile[misc_reg] | (~mmu->getPMP()->pmpTorMask() >> 1); + } else { + return miscRegFile[misc_reg] & (mmu->getPMP()->pmpTorMask()); + } + return 0; + } else if (misc_reg > NUM_MISCREGS || misc_reg < 0) { // Illegal CSR panic("Illegal CSR index %#x\n", misc_reg); return -1; @@ -323,6 +395,40 @@ ISA::readMiscRegNoEffect(int misc_reg) const RegVal ISA::readMiscReg(int misc_reg) { + int v = readMiscRegNoEffect(MISCREG_VIRMODE); + auto pm = readMiscRegNoEffect(MISCREG_PRV); + if ((v == 1) && (misc_reg == MISCREG_SSCRATCH)) { + return readMiscRegNoEffect(MISCREG_VSSCRATCH); + } + if ((v == 1) && (misc_reg == MISCREG_SATP)) { + return readMiscRegNoEffect(MISCREG_VSATP); + } + if ((v == 1) && (misc_reg == MISCREG_SEPC)) { + return readMiscRegNoEffect(MISCREG_VSEPC); + } + if ((v == 1) && (misc_reg == MISCREG_STVAL)) { + return readMiscRegNoEffect(MISCREG_VSTVAL); + } + if ((v == 1) && (misc_reg == MISCREG_SCAUSE)) { + return readMiscRegNoEffect(MISCREG_VSCAUSE); + } + if ((v == 1) && (misc_reg == MISCREG_STVEC)) { + return readMiscRegNoEffect(MISCREG_VSTVEC); + } + if (misc_reg == MISCREG_HIE) { + auto ic = dynamic_cast(tc->getCpuPtr()->getInterruptController(tc->threadId())); + DPRINTF(RiscvMisc, "Read IE value: %#lx.\n", ic->readIE()); + return ic->readIE() & NEMU_HIE_RMASK & (readMiscRegNoEffect(MISCREG_MIDELEG) | NEMU_MIDELEG_FORCED_MASK); + } + if (misc_reg == MISCREG_HIP) { + auto ic = dynamic_cast(tc->getCpuPtr()->getInterruptController(tc->threadId())); + DPRINTF(RiscvMisc, "Read IE value: %#lx.\n", ic->readIE()); + return (ic->readIP() & NEMU_HIP_RMASK & (readMiscRegNoEffect(MISCREG_MIDELEG) | NEMU_MIDELEG_FORCED_MASK)); + } + if (misc_reg == MISCREG_HVIP) { + auto ic = dynamic_cast(tc->getCpuPtr()->getInterruptController(tc->threadId())); + return (ic->readIP() & NEMU_HVIP_MASK); + } switch (misc_reg) { case MISCREG_HARTID: return tc->contextId(); @@ -389,6 +495,10 @@ ISA::readMiscReg(int misc_reg) (readMiscRegNoEffect(MISCREG_VXRM) << 1); } break; + case MISCREG_PMPADDR00 ... MISCREG_PMPADDR15: + { + return readMiscRegNoEffect(misc_reg); + } break; default: // Try reading HPM counters // As a placeholder, all HPM counters are just cycle counters @@ -422,12 +532,54 @@ ISA::setMiscRegNoEffect(int misc_reg, RegVal val) void ISA::setMiscReg(int misc_reg, RegVal val) { + int v = readMiscReg(MISCREG_VIRMODE); if (misc_reg == MISCREG_STATUS) { DPRINTF(RiscvMisc, "setMiscReg: setting mstatus with %#lx\n", val); } + if (misc_reg == MISCREG_HSTATUS) { + DPRINTF(RiscvMisc, "setMiscReg: setting mstatus with %#lx\n", val); + } + if (misc_reg == MISCREG_IE) { + DPRINTF(RiscvMisc, "setMiscReg: setting mstatus with %#lx\n", val); + } if (misc_reg >= MISCREG_CYCLE && misc_reg <= MISCREG_HPMCOUNTER31) { // Ignore writes to HPM counters for now - warn("Ignoring write to %s.\n", CSRData.at(misc_reg).name); + if (misc_reg >= MISCREG_MHPMCOUNTER3 && misc_reg <= MISCREG_MHPMCOUNTER31) { + warn("write to misc_reg %x val %lx but now write 0\n", misc_reg, val); + setMiscRegNoEffect(misc_reg, 0); + } else { + warn("Ignoring write to %x\n", misc_reg); + } + } else if ((v == 1) && ((misc_reg == MISCREG_SSCRATCH))) { + if (misc_reg == MISCREG_SSCRATCH) { + setMiscRegNoEffect(MISCREG_VSSCRATCH, val); + } + + } else if ((v == 1) && ((misc_reg == MISCREG_STATUS)) && (readMiscRegNoEffect(MISCREG_PRV) == PRV_S)) { + auto vsstatus = readMiscRegNoEffect(MISCREG_VSSTATUS); + RegVal write_val = ((vsstatus & ~(NEMU_SSTATUS_WMASK)) | (val & NEMU_SSTATUS_WMASK)); + setMiscRegNoEffect(MISCREG_VSSTATUS, write_val); + } else if (misc_reg == MISCREG_VSSTATUS) { + auto vsstatus = readMiscRegNoEffect(MISCREG_VSSTATUS); + RegVal write_val = ((vsstatus & ~(NEMU_SSTATUS_WMASK)) | (val & NEMU_SSTATUS_WMASK)); + setMiscRegNoEffect(MISCREG_VSSTATUS, write_val); + } else if ((v == 1) && ((misc_reg == MISCREG_SATP))) { + if ((val & NEMU_SATP_SV39_MASK) >> NEMU_SATP_RIGHT_OFFSET == NEMU_SV39_SIGN0 || + (val & NEMU_SATP_SV39_MASK) >> NEMU_SATP_RIGHT_OFFSET == NEMU_SV39_SIGN1) { + setMiscRegNoEffect(MISCREG_VSATP, val & NEMU_SATP_MASK); + } + } else if ((v == 1) && (misc_reg == MISCREG_SEPC)) { + setMiscRegNoEffect(MISCREG_VSEPC, val); + } else if (misc_reg == MISCREG_HCOUNTEREN) { + auto hcounter = readMiscRegNoEffect(MISCREG_HCOUNTEREN); + RegVal write_val = ((hcounter & ~(NEMU_COUNTER_MASK)) | (val & NEMU_COUNTER_MASK)); + setMiscRegNoEffect(MISCREG_HCOUNTEREN, write_val); + } else if (misc_reg == MISCREG_SCOUNTEREN) { + auto scounter = readMiscRegNoEffect(MISCREG_SCOUNTEREN); + RegVal write_val = ((scounter & ~(NEMU_COUNTER_MASK)) | (val & NEMU_COUNTER_MASK)); + setMiscRegNoEffect(MISCREG_SCOUNTEREN, write_val); + } else if ((v == 1) && ((misc_reg == MISCREG_STVEC))) { + setMiscRegNoEffect(MISCREG_VSTVEC, val & ~(0x2UL)); } else { switch (misc_reg) { @@ -475,18 +627,42 @@ ISA::setMiscReg(int misc_reg, RegVal val) auto mmu = dynamic_cast (tc->getMMUPtr()); uint32_t pmp_index = misc_reg-MISCREG_PMPADDR00; + RegVal write_val = val & (((uint64_t)1 << (36 - 2)) - 1); + uint64_t csr_num = mmu->getPMP()->pmpcfg_from_index(pmp_index); + uint64_t cfg = miscRegFile[csr_num]; + mmu->getPMP()->pmpUpdateAddr(pmp_index, val); - setMiscRegNoEffect(misc_reg, val); + setMiscRegNoEffect(misc_reg, write_val); } break; + case MISCREG_HVIP: { + auto ic = + dynamic_cast(tc->getCpuPtr()->getInterruptController(tc->threadId())); + auto old = readMiscReg(MISCREG_IP); + RegVal writeVal = ((old & ~(NEMU_HVIP_MASK)) | (val & NEMU_HVIP_MASK)); + ic->setIP(writeVal); + } break; case MISCREG_IP: { auto ic = dynamic_cast( tc->getCpuPtr()->getInterruptController(tc->threadId())); DPRINTF(RiscvMisc, "Setting IP to %#lx.\n", val); - ic->setIP(val); + auto old = readMiscReg(MISCREG_IP); + RegVal writeVal = ((old & ~(NEMU_MIP_MASK)) | (val & NEMU_MIP_MASK)); + ic->setIP(writeVal); + } + break; + case MISCREG_HIE: + { + auto ic = dynamic_cast( + tc->getCpuPtr()->getInterruptController(tc->threadId())); + RegVal write_val =0; + RegVal old = readMiscReg(MISCREG_IE); + RegVal hip_Mask = NEMU_HIE_WMASK & (readMiscReg(MISCREG_MIDELEG) | NEMU_MIDELEG_FORCED_MASK); + write_val = ((old & ~(hip_Mask)) |(val & hip_Mask)); + ic->setIE(write_val); } break; case MISCREG_IE: @@ -494,6 +670,16 @@ ISA::setMiscReg(int misc_reg, RegVal val) auto ic = dynamic_cast( tc->getCpuPtr()->getInterruptController(tc->threadId())); DPRINTF(RiscvMisc, "Setting IE to %#lx.\n", val); + uint64_t sie_mask = 0x222 & readMiscReg(MISCREG_MIDELEG); + if ((v == 1) && ((misc_reg == MISCREG_IE)) && (readMiscRegNoEffect(MISCREG_PRV) == PRV_S)) { + RegVal old = readMiscReg(MISCREG_IE); + RegVal write_val = ((old & ~(NEMU_VS_MASK)) | ((val)&NEMU_VS_MASK)); + val = write_val; + } else if (readMiscRegNoEffect(MISCREG_PRV) == PRV_S) { + RegVal old = readMiscReg(MISCREG_IE); + RegVal write_val = ((old & ~(sie_mask)) | (val & sie_mask)); + val = write_val; + } ic->setIE(val); } break; @@ -503,13 +689,20 @@ ISA::setMiscReg(int misc_reg, RegVal val) // shall have no effect (see 4.1.12 in priv ISA manual) SATP cur_val = readMiscRegNoEffect(misc_reg); SATP new_val = val; - if (new_val.mode != AddrXlateMode::BARE && - new_val.mode != AddrXlateMode::SV39) - new_val.mode = cur_val.mode; - setMiscRegNoEffect(misc_reg, new_val); + //change the mode update , only support sv39 + //if (new_val.mode != AddrXlateMode::BARE && + // new_val.mode != AddrXlateMode::SV39) + // new_val.mode = cur_val.mode; + //setMiscRegNoEffect(misc_reg, new_val); if (cur_val != new_val) { tc->getCpuPtr()->flushTLBs(); } + if ((val & NEMU_SATP_SV39_MASK) >> NEMU_SATP_RIGHT_OFFSET == NEMU_SV39_SIGN0 || + (val & NEMU_SATP_SV39_MASK) >> NEMU_SATP_RIGHT_OFFSET == NEMU_SV39_SIGN1) { + RegVal writeVal = val & NEMU_SATP_MASK; + setMiscRegNoEffect(misc_reg, writeVal); + } + } break; case MISCREG_TSELECT: @@ -590,6 +783,19 @@ ISA::setMiscReg(int misc_reg, RegVal val) setMiscRegNoEffect(misc_reg, val); } break; + case MISCREG_MIDELEG: + { + RegVal writeVal = val|((1 << 12) | (1 << 10) | (1 << 6) | (1 << 2)); + setMiscRegNoEffect(misc_reg, writeVal); + } + break; + case MISCREG_HSTATUS: + { + RegVal oldVal = readMiscRegNoEffect(MISCREG_HSTATUS); + RegVal writeVal = (oldVal & ~HSTATUS_MASK)|(val & HSTATUS_MASK); + setMiscRegNoEffect(misc_reg, writeVal); + } + break; default: setMiscRegNoEffect(misc_reg, val); } @@ -694,6 +900,8 @@ operator<<(std::ostream &os, gem5::RiscvISA::PrivilegeMode pm) return os << "PRV_U"; case gem5::RiscvISA::PRV_S: return os << "PRV_S"; + case gem5::RiscvISA::PRV_HS: + return os << "PRV_HS"; case gem5::RiscvISA::PRV_M: return os << "PRV_M"; } diff --git a/src/arch/riscv/isa.hh b/src/arch/riscv/isa.hh index 3123914597..a754537c1f 100644 --- a/src/arch/riscv/isa.hh +++ b/src/arch/riscv/isa.hh @@ -54,6 +54,7 @@ enum PrivilegeMode { PRV_U = 0, PRV_S = 1, + PRV_HS = 2, PRV_M = 3 }; diff --git a/src/arch/riscv/isa/bitfields.isa b/src/arch/riscv/isa/bitfields.isa index 5a433ab3e1..953b12b59f 100644 --- a/src/arch/riscv/isa/bitfields.isa +++ b/src/arch/riscv/isa/bitfields.isa @@ -153,3 +153,6 @@ def bitfield BIT31 <31>; def bitfield BIT30 <30>; def bitfield SIMM5 <19:15>; def bitfield SIMM3 <17:15>; + +//h instructions +def bitfield HFUNCT2 <31:20>; diff --git a/src/arch/riscv/isa/decoder.isa b/src/arch/riscv/isa/decoder.isa index bf5ae299bb..e88ebd9d26 100644 --- a/src/arch/riscv/isa/decoder.isa +++ b/src/arch/riscv/isa/decoder.isa @@ -57,7 +57,7 @@ decode QUADRANT default Unknown::unknown() { machInst); } else { status.fs = 3; - xc->setMiscReg(MISCREG_STATUS,status); + //xc->setMiscReg(MISCREG_STATUS,status); } Fp2_bits = Mem; @@ -441,7 +441,7 @@ decode QUADRANT default Unknown::unknown() { "FPU is off", machInst); } else { status.fs = 3; - xc->setMiscReg(MISCREG_STATUS,status); + // xc->setMiscReg(MISCREG_STATUS,status); } freg_t fd; fd = freg(f64(Mem)); @@ -1390,7 +1390,7 @@ decode QUADRANT default Unknown::unknown() { Rd = (int32_t) ((Rs1_uw >> shamt) | (Rs1_uw << ((32 - shamt) & (32 - 1)))); }}); } - 0x6: decode FUNCT7 { + 0x6: decode FUNCT7 { 0x1: remw({{ if (Rs2_sw == 0) { Rd_sd = Rs1_sw; @@ -4293,6 +4293,13 @@ decode QUADRANT default Unknown::unknown() { }}, IsDirectControl, IsUncondControl); 0x1c: decode FUNCT3 { + format LoadH { + 0x04: decode HFUNCT2{ + 0x643: hlvx_hu({{ + Rd = Mem_uh; + }},inst_flags = IsHInst); + } + } format SystemOp { 0x0: decode FUNCT7 { 0x0: decode RS2 { @@ -4316,28 +4323,72 @@ decode QUADRANT default Unknown::unknown() { 0x8: decode RS2 { 0x2: sret({{ STATUS status = xc->readMiscReg(MISCREG_STATUS); + HSTATUS hstatus = xc->readMiscReg(MISCREG_HSTATUS); + VSSTATUS vsstatus = xc->readMiscReg(MISCREG_VSSTATUS); + VSSTATUS32 vsstatus32 = xc->readMiscReg(MISCREG_VSSTATUS); + int v = xc->readMiscReg(MISCREG_VIRMODE); + auto hstatus_spv = hstatus.spv; auto pm = (PrivilegeMode)xc->readMiscReg( MISCREG_PRV); - if (pm == PRV_U || - (pm == PRV_S && status.tsr == 1)) { - return std::make_shared( - "sret in user mode or TSR enabled", - machInst); - NPC = NPC; - } else { - xc->setMiscReg(MISCREG_PRV, status.spp); - status.sie = status.spie; - status.spie = 1; - status.spp = PRV_U; - xc->setMiscReg(MISCREG_STATUS, status); - NPC = xc->readMiscReg(MISCREG_SEPC); + bool write_sign = false; + if(v ==0){ + hstatus.spv =0; + xc->setMiscReg(MISCREG_HSTATUS, hstatus); + + } + else{ + write_sign = true; + if((pm == PRV_S && hstatus.vtsr) || pm < PRV_S){ + panic("todo EX_VI"); + } + if(hstatus.vsxl ){ + auto vsstatus_spp = vsstatus32.spp; + xc->setMiscReg(MISCREG_PRV, vsstatus32.spp); + vsstatus32.spp = PRV_U; + vsstatus32.sie = vsstatus32.spie; + vsstatus32.spie =1; + xc->setMiscReg(MISCREG_VSSTATUS, vsstatus32); + } + else{ + xc->setMiscReg(MISCREG_PRV, vsstatus.spp); + vsstatus.spp = PRV_U; + vsstatus.sie = vsstatus.spie; + vsstatus.spie =1; + xc->setMiscReg(MISCREG_VSSTATUS, vsstatus); + } + NPC = xc->readMiscReg(MISCREG_VSEPC); + } + if(!write_sign){ + if (pm == PRV_U || + (pm == PRV_S && status.tsr == 1)) { + return std::make_shared( + "sret in user mode or TSR enabled", + machInst); + NPC = NPC; + } else { + xc->setMiscReg(MISCREG_PRV, status.spp); + status.sie = status.spie; + status.spie = 1; + status.spp = PRV_U; + xc->setMiscReg(MISCREG_STATUS, status); + NPC = xc->readMiscReg(MISCREG_SEPC); + } + xc->setMiscReg(MISCREG_VIRMODE, hstatus_spv); } + + }}, IsSerializeAfter, IsNonSpeculative, IsReturn); 0x5: wfi({{ STATUS status = xc->readMiscReg(MISCREG_STATUS); auto pm = (PrivilegeMode)xc->readMiscReg( MISCREG_PRV); - if (pm == PRV_U || + HSTATUS hstatus = xc->readMiscReg(MISCREG_HSTATUS); + int v = xc->readMiscReg(MISCREG_VIRMODE); + if(( v && (pm == PRV_S) && (hstatus.vtw ==1) && (status.tw == 0)) + || (v && (pm == PRV_U) && (status.tw == 0))){ + return std::make_shared(); + } + else if (pm == PRV_U || (pm == PRV_S && status.tw == 1)) { return std::make_shared( "wfi in user mode or TW enabled", @@ -4368,43 +4419,143 @@ decode QUADRANT default Unknown::unknown() { status.mie = status.mpie; status.mpie = 1; status.mpp = PRV_U; + auto status_mpv = status.mpv; + + status.mpv = 0; xc->setMiscReg(MISCREG_STATUS, status); NPC = xc->readMiscReg(MISCREG_MEPC); + xc->setMiscReg(MISCREG_VIRMODE,status_mpv); } }}, IsSerializeAfter, IsNonSpeculative, IsReturn); + 0x31: priv({{ + auto pm = (PrivilegeMode)xc->readMiscReg( + MISCREG_PRV); + auto vir = (PrivilegeMode)xc->readMiscReg( + MISCREG_VIRMODE); + STATUS status = xc->readMiscReg(MISCREG_STATUS); + xc->tcBase()->getMMUPtr()->demapPage(Rs1, Rs2); + + if(vir){ + panic("add priv vir code\n"); + + } + if(pm == PRV_U){ + return std::make_shared( + "h-priv pm == prv_u", machInst); + + } + if(!(pm == PRV_M ||(pm == PRV_S && !vir && status.tvm ==0))){ + return std::make_shared( + "h-priv pm == prv_u", machInst); + + } + }},IsNonSpeculative, IsSerializeAfter, No_OpClass); } } format CSROp { - 0x1: csrrw({{ - Rd = data; - data = Rs1; - write = RS1 != 0; - }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); - 0x2: csrrs({{ - Rd = data; - data |= Rs1; - write = RS1 != 0; - }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); - 0x3: csrrc({{ - Rd = data; - data &= ~Rs1; - write = RS1 != 0; - }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); - 0x5: csrrwi({{ - Rd = data; - data = uimm; - write = uimm != 0; - }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); - 0x6: csrrsi({{ - Rd = data; - data |= uimm; - write = uimm != 0; - }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); - 0x7: csrrci({{ - Rd = data; - data &= ~uimm; - write = uimm != 0; - }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + 0x1:decode FUNCT12 { + 0x100:csrrw_s({{ + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if(v) + data = xc->readMiscReg(MISCREG_VSSTATUS) & NEMU_SSTATUS_RMASK; + Rd = data; + data = Rs1; + write = RS1 != 0; + }}, IsUpdateMstatusSd,IsUpdateMstatusSd,IsSerializeAfter, IsNonSpeculative, No_OpClass); + default:csrrw({{ + Rd = data; + data = Rs1; + write = RS1 != 0; + }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + } + 0x2: decode FUNCT12 { + 0x100:csrrs_s({{ + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if(v) + data = xc->readMiscReg(MISCREG_VSSTATUS) & NEMU_SSTATUS_RMASK; + Rd = data; + data |= Rs1; + write = RS1 != 0; + }}, IsUpdateVsstatusSd,IsUpdateMstatusSd,IsSerializeAfter, IsNonSpeculative, No_OpClass); + default:csrrs({{ + auto Rs1_use = Rs1; + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if((FUNCT12 == 0x104) && v) + Rs1_use = Rs1 <<1; + Rd = data; + data |= Rs1_use; + write = RS1 != 0; + }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + } + 0x3: decode FUNCT12 { + 0x100:csrrc_s({{ + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if(v) + data = xc->readMiscReg(MISCREG_VSSTATUS) & NEMU_SSTATUS_RMASK; + Rd = data; + data &= ~Rs1; + write = RS1 != 0; + }}, IsUpdateVsstatusSd,IsUpdateMstatusSd,IsSerializeAfter, IsNonSpeculative, No_OpClass); + default:csrrc({{ + auto Rs1_use = Rs1; + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if((FUNCT12 == 0x104) && v) + Rs1_use = Rs1 <<1; + Rd = data; + data &= ~Rs1_use; + write = RS1 != 0; + }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + } + 0x5: decode FUNCT12 { + 0x100:csrrwi_s({{ + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if(v) + data = xc->readMiscReg(MISCREG_VSSTATUS) & NEMU_SSTATUS_RMASK; + Rd = data; + data = uimm; + write = uimm != 0; + }}, IsUpdateVsstatusSd,IsSerializeAfter, IsNonSpeculative, No_OpClass); + default:csrrwi({{ + Rd = data; + data = uimm; + write = uimm != 0; + }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + } + 0x6: decode FUNCT12 { + 0x100:csrrsi_s({{ + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if(v) + data = xc->readMiscReg(MISCREG_VSSTATUS) & NEMU_SSTATUS_RMASK; + Rd = data; + data |= uimm; + write = uimm != 0; + }}, IsUpdateVsstatusSd,IsSerializeAfter, IsNonSpeculative, No_OpClass); + default:csrrsi({{ + auto uimm_use = uimm; + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if((FUNCT12 == 0x104) && v) + uimm_use =uimm<<1; + Rd = data; + data |= uimm_use; + write = uimm != 0; + }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + } + 0x7: decode FUNCT12 { + 0x100:csrrci_s({{ + auto v = xc->readMiscReg(MISCREG_VIRMODE); + if(v) + data = xc->readMiscReg(MISCREG_VSSTATUS) & NEMU_SSTATUS_RMASK; + Rd = data; + data &= ~uimm; + write = uimm != 0; + }}, IsUpdateVsstatusSd,IsSerializeAfter, IsNonSpeculative, No_OpClass); + default:csrrci({{ + Rd = data; + data &= ~uimm; + write = uimm != 0; + }}, IsSerializeAfter, IsNonSpeculative, No_OpClass); + } + } } diff --git a/src/arch/riscv/isa/formats/mem.isa b/src/arch/riscv/isa/formats/mem.isa index a1005b4a15..f79290ead2 100644 --- a/src/arch/riscv/isa/formats/mem.isa +++ b/src/arch/riscv/isa/formats/mem.isa @@ -231,3 +231,11 @@ def format Store(memacc_code, ea_code={{EA = Rs1 + offset;}}, LoadStoreBase(name, Name, offset_code, ea_code, memacc_code, mem_flags, inst_flags, 'Store', exec_template_base='Store') }}; + +def format LoadH(memacc_code, ea_code = {{EA = Rs1;}}, + offset_code={{offset = 0}}, + mem_flags=[], inst_flags=[]) {{ + (header_output, decoder_output, decode_block, exec_output) = \ + LoadStoreBase(name, Name, offset_code, ea_code, memacc_code, mem_flags, + inst_flags, 'Load', exec_template_base='Load') +}}; diff --git a/src/arch/riscv/isa/formats/standard.isa b/src/arch/riscv/isa/formats/standard.isa index 56f91187e1..b635a091e4 100644 --- a/src/arch/riscv/isa/formats/standard.isa +++ b/src/arch/riscv/isa/formats/standard.isa @@ -345,7 +345,9 @@ def template CSRExecute {{ RegVal data, olddata; auto lowestAllowedMode = (PrivilegeMode)bits(csr, 9, 8); auto pm = (PrivilegeMode)xc->readMiscReg(MISCREG_PRV); - if (pm < lowestAllowedMode) { + if ((pm < lowestAllowedMode) && ((csr CSR_HGATP)) && ((csr CSR_VSATP))) { return std::make_shared( csprintf("%s is not accessible in %s\n", csrName, pm), machInst); @@ -387,6 +389,9 @@ def template CSRExecute {{ %(code)s; data &= maskVal; + if(csr == CSR_HIE){ + xc->setMiscReg(midx, data); + } // NOTE: may has bug if (data != olddata || csr == CSR_FCSR || (csr == CSR_FRM && write) || csr == CSR_VCSR) { if (bits(csr, 11, 10) == 0x3) { @@ -409,6 +414,7 @@ def template CSRExecute {{ case CSR_SIP: case CSR_SIE: case CSR_UIP: case CSR_UIE: case CSR_MSTATUS: case CSR_SSTATUS: case CSR_USTATUS: + case CSR_HSTATUS: if (newdata_all != olddata_all) { xc->setMiscReg(midx, newdata_all); auto new_val = xc->readMiscReg(MISCREG_STATUS); diff --git a/src/arch/riscv/pagetable.hh b/src/arch/riscv/pagetable.hh index 1f70114484..aa026ff174 100644 --- a/src/arch/riscv/pagetable.hh +++ b/src/arch/riscv/pagetable.hh @@ -55,10 +55,18 @@ enum AddrXlateMode SV48 = 9, }; +const Addr H_VADDR_BITS = 41; // Sv39 paging const Addr VADDR_BITS = 39; const Addr LEVEL_BITS = 9; -const Addr LEVEL_MASK = (1 << LEVEL_BITS) - 1; +const Addr LEVEL_MASK = ((1 << LEVEL_BITS) - 1); +const Addr PGMASK = ((1 << 12) - 1); +const Addr TWO_STAGE_L2_LEVEL_MASK = 0x7ff; +const Addr VPN_MASK = 0x1ff; +const Addr PGSHFT = 12; +const Addr PTESIZE = 8; +const Addr L2PageTypeNum = 4; +const Addr L2PageStoreTypeNum = 5; const Addr L2TLB_BLK_OFFSET = 3; const Addr VADDR_CHOOSE_MASK = 7; @@ -89,6 +97,28 @@ enum l2TLBPage L_L2sp2 }; +enum HTLBHitState +{ + H_L1miss = 0, + h_l1AllstageHit, + h_l1VSstageHit, + h_l1GstageHit, + h_l2VSstageHitEnd, + h_l2VSstageHitContinue, + h_l2GstageHitEnd, + h_l2GstageHitContinue +}; + +enum TlbTranslateMode { direct = 0, vsstage, gstage, allstage }; + +enum TranslateMode +{ + defaultmode = 0, + twoStageMode = 1 + +}; + +enum MMUMode { MMU_DIRECT = 0, MMU_TRANSLATE = 1, MMU_DYNAMIC = 2 }; BitUnion64(PTESv39) Bitfield<53, 10> ppn; @@ -118,12 +148,19 @@ struct TlbEntry : public Serializable // The beginning of the virtual page this entry maps. Addr vaddr; + Addr gpaddr; // The size of the page this represents, in address bits. unsigned logBytes; - + //transalte mode + //0:direct 1:vsstage 2:gstage 3:allstage + uint8_t translateMode; + //vsatp.asid or satp.asid uint16_t asid; + // hgatp.vmid + uint16_t vmid; PTESv39 pte; + PTESv39 pteVS; TlbEntryTrie::Handle trieHandle; @@ -131,6 +168,7 @@ struct TlbEntry : public Serializable uint64_t lruSeq; uint64_t level; + uint64_t VSlevel; Addr index; @@ -145,10 +183,16 @@ struct TlbEntry : public Serializable TlbEntry() : paddr(0), vaddr(0), + gpaddr(0), logBytes(0), + translateMode(0), + asid(0), + vmid(0), pte(), + pteVS(), lruSeq(0), level(0), + VSlevel(0), index(0), isSquashed(false), used(false), diff --git a/src/arch/riscv/pagetable_walker.cc b/src/arch/riscv/pagetable_walker.cc index 196141b63d..e7677ce262 100644 --- a/src/arch/riscv/pagetable_walker.cc +++ b/src/arch/riscv/pagetable_walker.cc @@ -64,6 +64,7 @@ #include "debug/PageTableWalker.hh" #include "debug/PageTableWalker2.hh" #include "debug/PageTableWalker3.hh" +#include "debug/PageTableWalkerTwoStage.hh" #include "mem/packet_access.hh" #include "mem/request.hh" @@ -114,8 +115,8 @@ Walker::start(Addr ppn, ThreadContext *_tc, BaseMMU::Translation *_translation, tryCoalesce(_tc, _translation, _req, _mode, from_l2tlb, asid, from_forward_pre_req, from_back_pre_req); if (!coalesced) { // create state - WalkerState * newState = new WalkerState(this, _translation, _req); - newState->initState(_tc, _mode, sys->isTimingMode(), from_forward_pre_req, from_back_pre_req); + WalkerState *newState = new WalkerState(this, _translation, _req); + newState->initState(_tc, _req, _mode, sys->isTimingMode(), from_forward_pre_req, from_back_pre_req); assert(newState->isTiming()); // TODO: add to requestors DPRINTF(PageTableWalker, @@ -128,7 +129,7 @@ Walker::start(Addr ppn, ThreadContext *_tc, BaseMMU::Translation *_translation, if (!newState->isTiming()) { assert(0); } - return NoFault; + return fault; } else { DPRINTF(PageTableWalker, "Walks in progress: %d. Coalesce req pc: %#lx, addr: %#lx " @@ -138,7 +139,7 @@ Walker::start(Addr ppn, ThreadContext *_tc, BaseMMU::Translation *_translation, } } else { WalkerState *newState = new WalkerState(this, _translation, _req); - newState->initState(_tc, _mode, sys->isTimingMode(), from_forward_pre_req, from_back_pre_req); + newState->initState(_tc, _req, _mode, sys->isTimingMode(), from_forward_pre_req, from_back_pre_req); currStates.push_back(newState); Fault fault = newState->startWalk(ppn, f_level, from_l2tlb, openNextLine, autoOpenNextLine, from_forward_pre_req, from_back_pre_req); @@ -151,14 +152,15 @@ Walker::start(Addr ppn, ThreadContext *_tc, BaseMMU::Translation *_translation, } void -Walker::doL2TLBHitSchedule(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Translation *translation, - BaseMMU::Mode mode, Addr Paddr, - const TlbEntry &entry) +Walker::doL2TLBHitSchedule(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, + BaseMMU::Mode mode, Addr Paddr, TlbEntry *entry, TlbEntry *entryVsstage, + TlbEntry *entryGstage, int delaytick) { DPRINTF(PageTableWalker2, "schedule %d\n", curCycle()); - if (!doL2TLBHitEvent.scheduled()) - schedule(doL2TLBHitEvent, nextCycle()); + Tick hitdelay = curTick() + cyclesToTicks(Cycles(delaytick)); + if (!doL2TLBHitEvent.scheduled()) { + schedule(doL2TLBHitEvent, hitdelay); + } L2TlbState l2state; l2state.req = req; l2state.tc = tc; @@ -166,15 +168,16 @@ Walker::doL2TLBHitSchedule(const RequestPtr &req, ThreadContext *tc, l2state.mode = mode; l2state.Paddr = Paddr; l2state.entry = entry; + l2state.entryVsstage = entryVsstage; + l2state.entryGstage = entryGstage; L2TLBrequestors.push_back(l2state); - } Fault Walker::startFunctional(ThreadContext * _tc, Addr &addr, unsigned &logBytes, BaseMMU::Mode _mode) { - funcState.initState(_tc, _mode); + funcState.initState(_tc, nullptr, _mode); return funcState.startFunctional(addr, logBytes, openNextLine, autoOpenNextLine, false, false); } @@ -261,26 +264,62 @@ Walker::getPort(const std::string &if_name, PortID idx) } void -Walker::WalkerState::initState(ThreadContext *_tc, BaseMMU::Mode _mode, bool _isTiming, bool _from_forward_pre_req, - bool _from_back_pre_req) +Walker::WalkerState::initState(ThreadContext *_tc, const RequestPtr &_req, BaseMMU::Mode _mode, bool _isTiming, + bool _from_forward_pre_req, bool _from_back_pre_req) { - assert(state == Ready); - started = false; - assert(requestors.back().tc == nullptr); - requestors.back().tc = _tc; - requestors.back().fromForwardPreReq = _from_forward_pre_req; - requestors.back().fromBackPreReq = _from_back_pre_req; - mode = _mode; - timing = _isTiming; - // fetch these now in case they change during the walk - status = _tc->readMiscReg(MISCREG_STATUS); - pmode = walker->tlb->getMemPriv(_tc, mode); - satp = _tc->readMiscReg(MISCREG_SATP); - assert(satp.mode == AddrXlateMode::SV39); - fromPre = _from_forward_pre_req; - fromBackPre = _from_back_pre_req; + assert(_req != nullptr); + if (_req->get_two_stage_state()) { + assert(state == Ready); + started = false; + assert(requestors.back().tc == nullptr); + requestors.back().tc = _tc; + requestors.back().fromForwardPreReq = false; + requestors.back().fromBackPreReq = false; + mode = _mode; + timing = _isTiming; + status = _tc->readMiscReg(MISCREG_STATUS); + vsstatus = _tc->readMiscReg(MISCREG_VSSTATUS); + pmode = (PrivilegeMode)(RegVal)_req->get_twoStageTranslateMode(); + satp = 0; + vsatp = _tc->readMiscReg(MISCREG_VSATP); + fromPre = false; + fromBackPre = false; + translateMode = twoStageMode; + hgatp = _tc->readMiscReg(MISCREG_HGATP); + isHInst = _req->get_h_inst(); + isVsatp0Mode = _req->get_vsatp_0_mode(); + virt = _req->get_virt(); + GstageFault = false; + tlbHit = false; + } else { + assert(state == Ready); + started = false; + assert(requestors.back().tc == nullptr); + requestors.back().tc = _tc; + requestors.back().fromForwardPreReq = _from_forward_pre_req; + requestors.back().fromBackPreReq = _from_back_pre_req; + mode = _mode; + timing = _isTiming; + // fetch these now in case they change during the walk + status = _tc->readMiscReg(MISCREG_STATUS); + vsstatus = _tc->readMiscReg(MISCREG_VSSTATUS); + pmode = walker->tlb->getMemPriv(_tc, mode); + satp = _tc->readMiscReg(MISCREG_SATP); + vsatp = 0; + assert(satp.mode == AddrXlateMode::SV39); + fromPre = _from_forward_pre_req; + fromBackPre = _from_back_pre_req; + translateMode = defaultmode; + hgatp = _tc->readMiscReg(MISCREG_HGATP); + isHInst = false; + isVsatp0Mode = false; + GstageFault = false; + tlbHit = false; + assert(!_req->get_h_inst()); + } } + std::pair Walker::WalkerState::tryCoalesce(ThreadContext *_tc, BaseMMU::Translation *translation, const RequestPtr &req, BaseMMU::Mode _mode, bool from_l2tlb, Addr asid, bool from_forward_pre_req, @@ -288,7 +327,6 @@ Walker::WalkerState::tryCoalesce(ThreadContext *_tc, BaseMMU::Translation *trans { SATP _satp = _tc->readMiscReg(MISCREG_SATP); - assert(_satp.mode == AddrXlateMode::SV39); bool priv_match; if (from_l2tlb) { priv_match = mode == _mode && satp == _satp && @@ -305,6 +343,11 @@ Walker::WalkerState::tryCoalesce(ThreadContext *_tc, BaseMMU::Translation *trans bool addr_match; Addr addr_match_num; Addr pre_match_num; + bool model_match; + model_match = (mainReq->get_two_stage_state() == req->get_two_stage_state()) && + (mainReq->get_virt() == req->get_virt()) && + (mainReq->get_twoStageTranslateMode() == req->get_twoStageTranslateMode()) && + (mainReq->get_vsatp_0_mode() == req->get_vsatp_0_mode()); if (fromPre) { addr_match_num = mainReq->getForwardPreVaddr(); } else if (fromBackPre) { @@ -325,7 +368,7 @@ Walker::WalkerState::tryCoalesce(ThreadContext *_tc, BaseMMU::Translation *trans ((addr_match_num >> PageShift) << PageShift); - if (priv_match && addr_match && (!finishDefaultTranslate)) { + if (priv_match && addr_match && (!finishDefaultTranslate) && model_match) { // coalesce if (from_forward_pre_req || from_back_pre_req) { DPRINTF(PageTableWalker, "from_forward_pre_req be coalesced\n"); @@ -347,7 +390,7 @@ Walker::WalkerState::tryCoalesce(ThreadContext *_tc, BaseMMU::Translation *trans if (mainFault != NoFault) { // recreate fault for this txn, we don't have pmp yet // TODO: also consider pmp's addr fault - new_fault = pageFaultOnRequestor(r); + new_fault = pageFaultOnRequestor(r, false); } if (requestors.size() == 1) { // previous requestors are squashed DPRINTF(PageTableWalker, @@ -381,8 +424,15 @@ Walker::dol2TLBHit() pmodel2, dol2TLBHitrequestors.tc); //assert(l2tlbFault == NoFault); if (l2tlbFault == NoFault){ - tlb->insert(dol2TLBHitrequestors.entry.vaddr, - dol2TLBHitrequestors.entry, false); + if (dol2TLBHitrequestors.entry != nullptr) + tlb->insert(dol2TLBHitrequestors.entry->vaddr, *dol2TLBHitrequestors.entry, false, direct); + if (dol2TLBHitrequestors.entryVsstage != nullptr) + tlb->insert(dol2TLBHitrequestors.entryVsstage->vaddr, *dol2TLBHitrequestors.entryVsstage, false, + vsstage); + if (dol2TLBHitrequestors.entryGstage != nullptr) + tlb->insert(dol2TLBHitrequestors.entryGstage->gpaddr, *dol2TLBHitrequestors.entryGstage, false, + gstage); + dol2TLBHitrequestors.translation->finish( l2tlbFault, dol2TLBHitrequestors.req, dol2TLBHitrequestors.tc, dol2TLBHitrequestors.mode); @@ -435,15 +485,23 @@ Walker::WalkerState::startWalk(Addr ppn, int f_level, bool from_l2tlb, started = true; assert(!(from_forward_req && from_back_req)); - if (from_back_req) { - setupWalk(ppn, mainReq->getBackPreVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, - from_forward_req, from_back_req); - } else if (from_forward_req) { - setupWalk(ppn, mainReq->getForwardPreVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, - from_forward_req, from_back_req); + if (translateMode == twoStageMode) { + fault = setupWalk(ppn, mainReq->getVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, + from_forward_req, from_back_req); + if (fault != NoFault) + return fault; + } else { - setupWalk(ppn, mainReq->getVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, from_forward_req, - from_back_req); + if (from_back_req) { + setupWalk(ppn, mainReq->getBackPreVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, + from_forward_req, from_back_req); + } else if (from_forward_req) { + setupWalk(ppn, mainReq->getForwardPreVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, + from_forward_req, from_back_req); + } else { + setupWalk(ppn, mainReq->getVaddr(), f_level, from_l2tlb, open_nextline, auto_open_nextline, + from_forward_req, from_back_req); + } } if (timing) { nextState = state; @@ -451,9 +509,12 @@ Walker::WalkerState::startWalk(Addr ppn, int f_level, bool from_l2tlb, mainFault = NoFault; sendPackets(); } else { + if (translateMode == twoStageMode) + assert(0); do { walker->port.sendAtomic(read); PacketPtr write = NULL; + assert(translateMode == twoStageMode); fault = stepWalk(write); assert(fault == NoFault || read == NULL); state = nextState; @@ -495,6 +556,535 @@ Walker::WalkerState::startFunctional(Addr &addr, unsigned &logBytes, return fault; } +Fault +Walker::WalkerState::twoStageStepWalk(PacketPtr &write) +{ + assert(state != Ready && state != Waiting); + Fault fault = NoFault; + write = NULL; + uint64_t vaddr_choose; + PTESv39 pte; + bool doEndWalk = false; + bool doLLwalk = false; + Addr PgBase; + PTESv39 l2pte; + unsigned oldSize = 64; + Request::Flags flags = Request::PHYSICAL; + + vaddr_choose = (gPaddr >> (twoStageLevel * LEVEL_BITS + PageShift)) & VADDR_CHOOSE_MASK; + PacketPtr oldRead = read; + if (!tlbHit) { + pte = read->getLE_l2tlb(vaddr_choose); + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk pte %lx vaddr %lx gpaddr %lx\n", + read->getLE_l2tlb(vaddr_choose), entry.vaddr, gPaddr); + + flags = oldRead->req->getFlags(); + + walker->pma->check(read->req); + oldSize = oldRead->getSize(); + + // Effective privilege mode for pmp checks for page table + // walks is S mode according to specs + fault = walker->pmp->pmpCheck(read->req, BaseMMU::Read, RiscvISA::PrivilegeMode::PRV_S, requestors.front().tc, + nextlineEntry.vaddr); + } else { + pte = tlbHitPte; + flags = tlbflags; + } + + Addr nextRead = 0; + Addr nextcheck = 0; + + PgBase = pte.ppn << 12; + + if (fault == NoFault) { + if (pte.v && !pte.r && !pte.w && !pte.x) { + twoStageLevel--; + if (twoStageLevel < 0) { + endWalk(); + warn("pagefault in Gstage ptw twostagelevel <0\n"); + return endGstageWalk(); + } else { + nextRead = (pte.ppn << PageShift) + (getGVPNi(gPaddr, twoStageLevel) * PTESIZE); + nextcheck = nextRead; + nextRead = (nextRead >> 6) << 6; + nextState = Translate; + if ((!isVsatp0Mode) && (!tlbHit)) { + int l2_level = twoStageLevel + 1; + inl2Entry.gpaddr = gPaddr; + inl2Entry.pte = pte; + inl2Entry.logBytes = PageShift + (l2_level * LEVEL_BITS); + inl2Entry.level = l2_level; + for (int l2_i = 0; l2_i < l2tlbLineSize; l2_i++) { + inl2Entry.gpaddr = (((gPaddr >> ((l2_level * LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET))) + << L2TLB_BLK_OFFSET) + + l2_i) + << ((l2_level * LEVEL_BITS + PageShift)); + l2pte = read->getLE_l2tlb(l2_i); + inl2Entry.pte = l2pte; + inl2Entry.paddr = l2pte.ppn; + if (l2_level == 2) { + walker->tlb->L2TLBInsert(inl2Entry.gpaddr, inl2Entry, l2_level, L_L2L1, l2_i, false, + gstage); + } else if (l2_level == 1) { + inl2Entry.index = + (gPaddr >> (LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET)) & (L2TLB_L2_MASK); + walker->tlb->L2TLBInsert(inl2Entry.gpaddr, inl2Entry, l2_level, L_L2L2, l2_i, false, + gstage); + } + } + } + } + + } else if (!pte.v || (!pte.r && pte.w)) { + endWalk(); + return endGstageWalk(); + } else if (!pte.u) { + endWalk(); + return endGstageWalk(); + } else if (((mode == BaseMMU::Execute) || isHInst) && (!pte.x)) { + endWalk(); + return endGstageWalk(); + } else if ((mode == BaseMMU::Read) && (!pte.r && !(status.mxr && pte.x))) { + endWalk(); + return endGstageWalk(); + } else if ((mode == BaseMMU::Write) && !(pte.r && pte.w)) { + endWalk(); + GstageFault = true; + fault = pageFault(true, true); + return fault; + } else { + inGstage = false; + doEndWalk = true; + doLLwalk = true; + entry.gpaddr = gPaddr; + entry.pte = pte; + entry.logBytes = PageShift + (twoStageLevel * LEVEL_BITS); + entry.level = twoStageLevel; + + Addr pg_mask; + if (twoStageLevel > 0) { + pg_mask = ((1ULL << (12 + 9 * twoStageLevel)) - 1); + if (((pte.ppn << 12) & pg_mask) != 0) { + // missaligned superpage + warn("missaligned superpage vaddr %lx\n",entry.vaddr); + fault = pageFault(true, false); + endWalk(); + return fault; + } + PgBase = (PgBase & ~pg_mask) | (gPaddr & pg_mask & ~PGMASK); + } + PgBase = PgBase | (gPaddr & PGMASK); + vaddr_choose_flag = (PgBase & 0x3f) / 8; + nextcheck = PgBase; + nextRead = (PgBase >> 6) << 6; + gPaddr = nextRead; + entry.paddr = gPaddr; + if (finishGVA && (!isVsatp0Mode)) { + walker->tlb->insert(entry.gpaddr, entry, false, gstage); + int l2_level = twoStageLevel; + inl2Entry.gpaddr = gPaddr; + inl2Entry.pte = pte; + inl2Entry.logBytes = PageShift + (l2_level * LEVEL_BITS); + inl2Entry.level = l2_level; + if (!tlbHit) { + for (int l2_i = 0; l2_i < l2tlbLineSize; l2_i++) { + inl2Entry.gpaddr = + ((gPaddr >> ((l2_level * LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET)) << L2TLB_BLK_OFFSET) + + l2_i) + << ((l2_level * LEVEL_BITS + PageShift)); + l2pte = read->getLE_l2tlb(l2_i); + DPRINTF(PageTableWalker3, "final insert vaddr %#x ppn %#x pte %#x pre %d\n", inl2Entry.vaddr, + l2pte.ppn, l2pte, entry.fromForwardPreReq); + DPRINTF(PageTableWalker3, "level %d l2_level %d\n", level, l2_level); + inl2Entry.paddr = l2pte.ppn; + inl2Entry.pte = l2pte; + if (l2_level == 0) { + inl2Entry.index = (gPaddr >> (L2TLB_BLK_OFFSET + PageShift)) & L2TLB_L3_MASK; + walker->tlb->L2TLBInsert(inl2Entry.gpaddr, inl2Entry, l2_level, L_L2L3, l2_i, false, + gstage); + } + + else if (l2_level == 1) { + walker->tlb->L2TLBInsert(inl2Entry.gpaddr, inl2Entry, l2_level, L_L2sp2, l2_i, false, + gstage); + } // hit level =1 + + else if (l2_level == 2) { + walker->tlb->L2TLBInsert(inl2Entry.gpaddr, inl2Entry, l2_level, L_L2sp1, l2_i, false, + gstage); + } + } + } + } + + if ((gPaddr & ~(((int64_t)1 << 41) - 1)) != 0) { + // this is a excep + panic("address fault\n"); + } + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk gpaddr %lx vaddr %lx\n", gPaddr, entry.vaddr); + gpaddrMode =1; + mainReq->setgPaddr(gPaddr); + } + + if (doLLwalk && finishGVA) { + //entry.paddr = pte.ppn; + entry.paddr = gPaddr >> 12; + entry.pte = pte; + int put_level = 0; + put_level = std::min(twoStageLevel, level); + + entry.logBytes = PageShift + (put_level * LEVEL_BITS); + entry.level = put_level; + walker->tlb->insert(entry.vaddr, entry, false, allstage); + + + endWalk(); + return NoFault; + } else if ((!doEndWalk) || (doLLwalk)) { + if (isVsatp0Mode && doLLwalk) { + entry.paddr = gPaddr >> 12; + entry.pte = pte; + entry.logBytes = PageShift + (twoStageLevel * LEVEL_BITS); + entry.level = twoStageLevel; + entry.gpaddr = entry.vaddr; + walker->tlb->insert(entry.vaddr, entry, false, gstage); + endWalk(); + return NoFault; + } + if (nextRead == 0) + panic("nextread can't be 0\n"); + + TlbEntry *e[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + int hit_level = 2; + bool tlb_hit = false; + + if (walker->l2tlb == nullptr) + panic("walker->l2tlb is none\n"); + if (inGstage) { + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = walker->l2tlb->lookupL2TLB(nextcheck, hgatp.vmid, mode, false, i_e, true, gstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + } + } + if (e[0] && (twoStageLevel == e[0]->level)) { + tlbHit = true; + tlbHitPte = e[0]->pte; + hit_level = true; + tlbflags = flags; + return twoStageStepWalk(write); + } else { + tlbHit = false; + } + } + } else { + tlbHit = false; + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = walker->l2tlb->lookupL2TLB(nextcheck, vsatp.asid, mode, false, i_e, true, vsstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + } + } + if (e[0] && (level == e[0]->level)) { + tlbHit = true; + tlbHitPte = e[0]->pte; + hit_level = true; + tlbflags = flags; + return twoStageWalk(write); + } else { + tlbHit = false; + } + } + } + if (!tlbHit) { + delete oldRead; + oldRead = nullptr; + RequestPtr request = std::make_shared(nextRead, oldSize, flags, walker->requestorId); + DPRINTF(PageTableWalkerTwoStage, + "twoStageStepWalk nextRead %lx vaddr %lx gpaddr %lx level %d twolevel %d\n", nextRead, + entry.vaddr, gPaddr, level, twoStageLevel); + DPRINTF(PageTableWalker, "oldread size %d\n", oldSize); + + read = new Packet(request, MemCmd::ReadReq); + read->allocate(); + DPRINTF(PageTableWalker, "Loading level%d PTE from %#x vaddr %#x\n", level, nextRead, entry.vaddr); + } + } else { + panic("wrong in G ptw\n"); + } + } else { + panic("wrong in G ptw\n"); + } + + return fault; +} + +Fault +Walker::WalkerState::twoStageWalk(PacketPtr &write) +{ + Fault fault; + bool doEndWalk = false; + PTESv39 pte; + + PacketPtr oldRead = read; + Request::Flags flags; + + PTESv39 l2pte; + unsigned oldSize = 64; + + if (!tlbHit) { + pte = read->getLE_l2tlb(vaddr_choose_flag); + flags = oldRead->req->getFlags(); + walker->pma->check(read->req); + oldSize = oldRead->getSize(); + fault = walker->pmp->pmpCheck(read->req, BaseMMU::Read, RiscvISA::PrivilegeMode::PRV_S, requestors.front().tc, + gPaddr); + + } else { + pte = tlbHitPte; + flags = tlbflags; + } + TlbEntry *e[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + int hit_level = 2; + bool tlb_hit = false; + Addr shift = 0; + Addr idx_f = 0; + Addr idx = 0; + Addr nextcheck = 0; + + if (fault == NoFault) { + if (!pte.v || (!pte.r && pte.w)) { + doEndWalk = true; + DPRINTF(PageTableWalker3, "PTE invalid, raising PF\n"); + GstageFault = false; + fault = pageFault(pte.v, false); + endWalk(); + } else { + if (pte.r || pte.x) { + doEndWalk = true; + if (virt) { + fault = walker->tlb->checkPermissions(vsstatus, pmode, entry.vaddr, mode, pte, 0, false); + } else { + fault = walker->tlb->checkPermissions(status, pmode, entry.vaddr, mode, pte, 0, false); + } + + if (fault == NoFault) { + if ((!pte.a) || ((!pte.d) && (mode == BaseMMU::Write))) { + GstageFault = false; + fault = pageFault(true,false); + endWalk(); + } else { + finishGVA = true; + entry.gpaddr = gPaddr; + entry.pte = pte; + entry.pteVS = pte; + entry.logBytes = PageShift + (level * LEVEL_BITS); + entry.level = level; + + gPaddr = pte.ppn << 12; + if (level > 0) { + Addr pg_mask = (1ULL << (12 + 9 * level)) - 1; + if ((pg_mask & (pte.ppn << 12)) != 0) { + fault = pageFault(true, false); + endWalk(); + return fault; + } + gPaddr = ((pte.ppn << 12) & ~pg_mask) | (entry.vaddr & pg_mask & ~PGMASK); + } + gPaddr = gPaddr | (entry.vaddr & PGMASK); + + entry.paddr = (gPaddr >> 12) << 12; + walker->tlb->insert(entry.vaddr, entry, false, vsstage); + if (!tlbHit) { + int l2_level = level; + inl2Entry.gpaddr = gPaddr; + inl2Entry.pte = pte; + inl2Entry.logBytes = PageShift + (l2_level * LEVEL_BITS); + inl2Entry.level = l2_level; + + for (int l2_i = 0; l2_i < l2tlbLineSize; l2_i++) { + inl2Entry.vaddr = + (((entry.vaddr >> ((l2_level * LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET))) + << L2TLB_BLK_OFFSET) + + l2_i) + << ((l2_level * LEVEL_BITS + PageShift)); + l2pte = read->getLE_l2tlb(l2_i); + inl2Entry.pte = l2pte; + inl2Entry.paddr = l2pte.ppn; + if (l2_level == 0) { + inl2Entry.index = (gPaddr >> (L2TLB_BLK_OFFSET + PageShift)) & L2TLB_L3_MASK; + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L3, l2_i, false, + vsstage); + } else if (l2_level == 1) { + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2sp2, l2_i, + false, vsstage); + } else if (l2_level == 2) { + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2sp1, l2_i, + false, vsstage); + } + } + } + if ((gPaddr & ~(((int64_t)1 << 41) - 1)) != 0) { + // this is a excep + fault = pageFault(true, true); + endWalk(); + return fault; + } + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk gpaddr %lx vaddr %lx\n", gPaddr, + entry.vaddr); + gpaddrMode =3; + mainReq->setgPaddr(gPaddr); + nextState = Translate; + inGstage = true; + twoStageLevel = 2; + tlbHit = false; + nextcheck = 0; + shift = PageShift + LEVEL_BITS * twoStageLevel; + idx = ((gPaddr >> shift) & TWO_STAGE_L2_LEVEL_MASK); + nextcheck = (hgatp.ppn << PageShift) + (idx * sizeof(PTESv39)); + + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = walker->l2tlb->lookupL2TLB(nextcheck, hgatp.vmid, mode, false, i_e, true, gstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + } + } + if (e[0] && (level == e[0]->level)) { + tlbHit = true; + tlbHitPte = e[0]->pte; + hit_level = true; + tlbflags = flags; + inGstage = true; + return twoStageStepWalk(write); + } else { + tlbHit = false; + } + } + if (!tlbHit) { + delete oldRead; + oldRead = nullptr; + fault = startTwoStageWalk(gPaddr, entry.vaddr); + if (fault != NoFault) { + endWalk(); + return fault; + } + } + } + } else { + endWalk(); + } + + } else { + level--; + if (level < 0) { + doEndWalk = true; + GstageFault = false; + fault = pageFault(true, false); + endWalk(); + } else { + entry.gpaddr = gPaddr; + entry.pte = pte; + entry.logBytes = PageShift + (level * LEVEL_BITS); + entry.level = level; + + shift = (PageShift + LEVEL_BITS * level); + idx_f = (entry.vaddr >> shift) & LEVEL_MASK; + idx = (idx_f >> L2TLB_BLK_OFFSET) << L2TLB_BLK_OFFSET; + gPaddr = (pte.ppn << PageShift) + (idx_f * l2tlbLineSize); + entry.paddr = gPaddr; + if (!tlbHit) { + int l2_level = level + 1; + inl2Entry.gpaddr = gPaddr; + inl2Entry.pte = pte; + inl2Entry.logBytes = PageShift + (l2_level * LEVEL_BITS); + inl2Entry.level = l2_level; + + for (int l2_i = 0; l2_i < l2tlbLineSize; l2_i++) { + inl2Entry.vaddr = + (((entry.vaddr >> ((l2_level * LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET))) + << L2TLB_BLK_OFFSET) + + l2_i) + << ((l2_level * LEVEL_BITS + PageShift)); + l2pte = read->getLE_l2tlb(l2_i); + inl2Entry.pte = l2pte; + inl2Entry.paddr = l2pte.ppn; + if (l2_level == 2) { + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L1, l2_i, false, + vsstage); + } else if (l2_level == 1) { + inl2Entry.index = + (entry.vaddr >> (LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET)) & (L2TLB_L2_MASK); + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L2, l2_i, false, + vsstage); + } + } + } + + if ((gPaddr & ~(((int64_t)1 << 41) - 1)) != 0) { + // this is a excep + fault = pageFault(true, false); + endWalk(); + return fault; + } + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk gpaddr %lx vaddr %lx\n", gPaddr, entry.vaddr); + gpaddrMode =2; + mainReq->setgPaddr(gPaddr); + nextState = Translate; + inGstage = true; + twoStageLevel = 2; + tlbHit = false; + + + shift = PageShift + LEVEL_BITS * twoStageLevel; + idx = ((gPaddr >> shift) & TWO_STAGE_L2_LEVEL_MASK); + nextcheck = (hgatp.ppn << PageShift) + (idx * sizeof(PTESv39)); + + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = walker->l2tlb->lookupL2TLB(nextcheck, hgatp.vmid, mode, false, i_e, true, gstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + } + } + if (e[0] && (level == e[0]->level)) { + tlbHit = true; + tlbHitPte = e[0]->pte; + hit_level = true; + tlbflags = flags; + inGstage = true; + return twoStageStepWalk(write); + } else { + tlbHit = false; + } + } + if (!tlbHit) { + delete oldRead; + oldRead = nullptr; + fault = startTwoStageWalk(gPaddr, entry.vaddr); + if (fault != NoFault) { + endWalk(); + return fault; + } + } + } + } + } + } else { + panic("wrong in G ptw\n"); + } + + return fault; +} + + Fault Walker::WalkerState::stepWalk(PacketPtr &write) { @@ -549,25 +1139,24 @@ Walker::WalkerState::stepWalk(PacketPtr &write) if (!pte.v || (!pte.r && pte.w)) { doEndWalk = true; DPRINTF(PageTableWalker3, "PTE invalid, raising PF\n"); - fault = pageFault(pte.v); + fault = pageFault(pte.v, false); } else { // step 4: if (pte.r || pte.x) { // step 5: leaf PTE doEndWalk = true; - fault = walker->tlb->checkPermissions(status, pmode, - entry.vaddr, mode, pte); + fault = walker->tlb->checkPermissions(status, pmode, entry.vaddr, mode, pte, 0, false); // step 6 if (fault == NoFault) { if (level >= 1 && pte.ppn0 != 0) { DPRINTF(PageTableWalker3, "PTE has misaligned PPN, raising PF\n"); - fault = pageFault(true); + fault = pageFault(true,false); } else if (level == 2 && pte.ppn1 != 0) { DPRINTF(PageTableWalker3, "PTE has misaligned PPN, raising PF\n"); - fault = pageFault(true); + fault = pageFault(true,false); } } else { DPRINTF(PageTableWalker3, "checkpremission fault\n"); @@ -578,12 +1167,12 @@ Walker::WalkerState::stepWalk(PacketPtr &write) if (!pte.a) { DPRINTF(PageTableWalker3, "PTE needs to write pte.a,raising PF\n"); - fault = pageFault(true); + fault = pageFault(true,false); } if (!pte.d && mode == BaseMMU::Write) { DPRINTF(PageTableWalker3, "PTE needs to write pte.d,raising PF\n"); - fault = pageFault(true); + fault = pageFault(true,false); } // Performing PMA/PMP checks @@ -624,7 +1213,7 @@ Walker::WalkerState::stepWalk(PacketPtr &write) "No leaf PTE found," "raising PF\n"); doEndWalk = true; - fault = pageFault(true); + fault = pageFault(true,false); } else { inl2Entry.logBytes = PageShift + ((level + 1) * LEVEL_BITS); @@ -648,12 +1237,14 @@ Walker::WalkerState::stepWalk(PacketPtr &write) inl2Entry.paddr = l2pte.ppn; inl2Entry.pte = l2pte; if (l2_level == 2) { - walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L1, l2_i, false); + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L1, l2_i, false, + direct); } if (l2_level == 1) { inl2Entry.index = (entry.vaddr >> (LEVEL_BITS + PageShift + L2TLB_BLK_OFFSET)) & (L2TLB_L2_MASK); - walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L2, l2_i, false); + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L2, l2_i, false, + direct); } if (l2_level == 0) { @@ -711,7 +1302,7 @@ Walker::WalkerState::stepWalk(PacketPtr &write) if (doTLBInsert) { if (!functional) { if (((!entry.fromForwardPreReq) && (!entry.fromBackPreReq)) || (preHitInPtw)) { - walker->tlb->insert(entry.vaddr, entry, false); + walker->tlb->insert(entry.vaddr, entry, false, direct); } finishDefaultTranslate = true; @@ -734,13 +1325,13 @@ Walker::WalkerState::stepWalk(PacketPtr &write) inl2Entry.pte = l2pte; if (l2_level == 0) { inl2Entry.index = (entry.vaddr >> (L2TLB_BLK_OFFSET + PageShift)) & L2TLB_L3_MASK; - walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L3, l2_i, false); + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2L3, l2_i, false, direct); } else if (l2_level == 1) // hit level =1 - walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2sp2, l2_i, false); + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2sp2, l2_i, false, direct); else if (l2_level == 2) // - walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2sp1, l2_i, false); + walker->tlb->L2TLBInsert(inl2Entry.vaddr, inl2Entry, l2_level, L_L2sp1, l2_i, false, direct); } if (!doWrite) { nextlineVaddr = entry.vaddr + (l2tlbLineSize << (nextlineLevel * LEVEL_BITS + PageShift)); @@ -787,8 +1378,7 @@ Walker::WalkerState::stepWalk(PacketPtr &write) DPRINTF(PageTableWalker, "nextline level %d pte from %#x vaddr %#x " "nextline_vaddr %#x\n", - nextlineLevel, nextRead, entry.vaddr, - nextlineVaddr); + nextlineLevel, nextRead, entry.vaddr, nextlineVaddr); return fault; } else { DPRINTF(PageTableWalker,"no pre\n"); @@ -817,7 +1407,8 @@ Walker::WalkerState::stepWalk(PacketPtr &write) nextlineEntry.pte = l2pte; if (nextlineEntry.level == 0) { nextlineEntry.index = (nextlineEntry.vaddr >> (PageShift + L2TLB_BLK_OFFSET)) & (L2TLB_L3_MASK); - walker->tlb->L2TLBInsert(nextlineEntry.vaddr, nextlineEntry, nextlineLevel, L_L2L3, n_l2_i, false); + walker->tlb->L2TLBInsert(nextlineEntry.vaddr, nextlineEntry, nextlineLevel, L_L2L3, n_l2_i, false, + direct); } else if (nextlineEntry.level == 1) { panic("nextline level can't be 1\n"); } else if (nextlineEntry.level == 2) { @@ -859,93 +1450,260 @@ Walker::WalkerState::endWalk() delete read; read = NULL; } +Fault +Walker::WalkerState::endGstageWalk() +{ + endWalk(); + GstageFault = true; + return pageFault(true, true); +} +Fault +Walker::WalkerState::startTwoStageWalkFromTLBNotInG(Addr ppn, Addr vaddr) +{ + Addr PgBase = ppn << 12; + Addr pg_mask = 0; + Fault fault = NoFault; + Addr nextRead = 0; + inGstage = false; + if (twoStageLevel > 0) { + pg_mask = ((1ULL << (12 + 9 * twoStageLevel)) - 1); + if (((ppn << 12) & pg_mask) != 0) { + // missaligned superpage + warn("missaligned superpage vaddr %lx\n", entry.vaddr); + fault = pageFault(true, false); + endWalk(); + panic("address check wrong in from tlb ptw\n"); + return fault; + } + PgBase = (PgBase & ~pg_mask) | (gPaddr & pg_mask & ~PGMASK); + } + PgBase = PgBase | (gPaddr & PGMASK); + vaddr_choose_flag = (PgBase & 0x3f) / 8; + nextRead = (PgBase >> 6) << 6; + gPaddr = nextRead; + if ((gPaddr & ~(((int64_t)1 << 41) - 1)) != 0) { + // this is a excep + panic("address check wrong in from tlb ptw\n"); + } + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk gpaddr %lx vaddr %lx\n", gPaddr, entry.vaddr); + gpaddrMode = 1; + mainReq->setgPaddr(gPaddr); -void + if (nextRead == 0) + panic("nextread can't be 0\n"); + Request::Flags flags = Request::PHYSICAL; + RequestPtr request = std::make_shared(nextRead, 64, flags, walker->requestorId); + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk nextRead %lx vaddr %lx gpaddr %lx level %d twolevel %d\n", + nextRead, entry.vaddr, gPaddr, level, twoStageLevel); + read = new Packet(request, MemCmd::ReadReq); + read->allocate(); + DPRINTF(PageTableWalker, "Loading level%d PTE from %#x vaddr %#x\n", level, nextRead, entry.vaddr); + return NoFault; +} +Fault +Walker::WalkerState::startTwoStageWalkFromTLBInG(Addr ppn, Addr vaddr) +{ + // vaddr_choose = (gPaddr >> (twoStageLevel * LEVEL_BITS + PageShift)) & VADDR_CHOOSE_MASK; + Addr nextRead = (ppn << PageShift) + (getGVPNi(gPaddr, twoStageLevel) * PTESIZE); + Request::Flags flags = Request::PHYSICAL; + nextRead = (nextRead >> 6) << 6; + if (nextRead == 0) + panic("nextread can't be 0\n"); + RequestPtr request = std::make_shared(nextRead, 64, flags, walker->requestorId); + read = new Packet(request, MemCmd::ReadReq); + read->allocate(); + return NoFault; +} + +Fault +Walker::WalkerState::startTwoStageWalk(Addr ppn, Addr vaddr) +{ + Addr shift = PageShift + LEVEL_BITS * twoStageLevel; + Addr idx; + inGstage = true; + + idx = (((gPaddr >> shift) & TWO_STAGE_L2_LEVEL_MASK) >> 3) << 3; + if (hgatp.mode == 8) { + Addr TwoLevelTopAddr = 0; + if ((ppn & ~(((int64_t)1 << 41) - 1)) != 0) { + // this is a excep + panic("address check wrong in start ptw\n"); + } + TwoLevelTopAddr = (hgatp.ppn << PageShift) + (idx * sizeof(PTESv39)); + + Request::Flags flags = Request::PHYSICAL; + RequestPtr request = std::make_shared(TwoLevelTopAddr, 64, flags, walker->requestorId); + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk pte %lx vaddr %lx gpaddr %lx level %d twolevel %d\n", + TwoLevelTopAddr, entry.vaddr, gPaddr, level, twoStageLevel); + if (TwoLevelTopAddr == 0) + panic("topAddr can't be 0\n"); + DPRINTF(PageTableWalker, " sv39 size is %d\n", sizeof(PTESv39)); + + read = new Packet(request, MemCmd::ReadReq); + read->allocate(); + + } else { + panic("hgatp.mode != 8 \n"); + } + return NoFault; +} + +Fault Walker::WalkerState::setupWalk(Addr ppn, Addr vaddr, int f_level, bool from_l2tlb, bool open_nextline, bool auto_open_nextline, bool from_forward_pre_req, bool from_back_pre_req) { - vaddr = Addr(sext(vaddr)); Addr topAddr; if (from_l2tlb ){ level = f_level; } else { level = 2; + if (mainReq->get_level() != 2) + level = mainReq->get_level(); + if (isVsatp0Mode) + level = 0; } - nextline = false; - autoNextlineSign = auto_open_nextline; - preHitInPtw = false; - Addr shift = PageShift + LEVEL_BITS * level; Addr idx_f = (vaddr >> shift) & LEVEL_MASK; - Addr idx = (idx_f>>3)<<3; + Addr idx = (idx_f >> 3) << 3; + Fault fault = NoFault; + if (translateMode == twoStageMode) { + nextline = false; + autoNextlineSign = false; + preHitInPtw = false; + nextline = false; + topAddr = (vsatp.ppn << PageShift) + (idx * sizeof(PTESv39)); + gPaddr = (vsatp.ppn << PageShift) + (idx_f * sizeof(PTESv39)); + if ((mainReq->get_level() != 2) && (mainReq->getgPaddr() != 0)) { + gPaddr = mainReq->getgPaddr(); + } + if (isVsatp0Mode) { + gPaddr = vaddr; + } - if (from_l2tlb ){ - topAddr = (ppn << PageShift) + (idx * sizeof(PTESv39)); + DPRINTF(PageTableWalkerTwoStage, "twoStageStepWalk gpaddr %lx vaddr %lx level %d\n", gPaddr, vaddr, level); + mainReq->setgPaddr(gPaddr); + gpaddrMode = 0; nextlineLevelMask = LEVEL_MASK; nextlineShift = shift; tlbVaddr = vaddr; tlbppn = ppn; - nextlineRead = topAddr; - nextlineLevel = level; - } - else{ - topAddr = (satp.ppn << PageShift) + (idx * sizeof(PTESv39)); - nextlineLevelMask = LEVEL_MASK; - nextlineShift = shift; - tlbVaddr = vaddr; - tlbppn = satp.ppn; - nextlineRead = topAddr; + nextlineRead = 0; nextlineLevel = level; - } - - DPRINTF(PageTableWalker, - "Performing table walk for address %#x shift %d idx_f %#x ppn %#x " - "satp.ppn %#x\n", - vaddr, shift, idx_f, ppn, satp.ppn); - DPRINTF(PageTableWalker, - "Loading level%d PTE from %#x idx %#x idx_shift %#x vaddr %#x\n", - level, topAddr, idx, idx << shift, vaddr); - - state = Translate; - nextState = Ready; - entry.vaddr = vaddr; - entry.asid = satp.asid; - entry.isSquashed = false; - entry.used = false; - entry.isPre = false; - entry.fromForwardPreReq = from_forward_pre_req; - entry.fromBackPreReq = from_back_pre_req; - entry.preSign = false; - - - nextlineEntry.vaddr = vaddr; - nextlineEntry.asid = satp.asid; - nextlineEntry.isSquashed = false; - nextlineEntry.used = false; - nextlineEntry.isPre = true; - nextlineEntry.fromBackPreReq = from_back_pre_req; - nextlineEntry.preSign = false; - - inl2Entry.asid = satp.asid; - inl2Entry.isSquashed = false; - inl2Entry.used = false; - inl2Entry.isPre = false; - inl2Entry.fromBackPreReq = from_back_pre_req; - inl2Entry.preSign = false; - finishDefaultTranslate = false; + state = Translate; + nextState = Ready; + entry.vaddr = vaddr; + entry.asid = vsatp.asid; + entry.isSquashed = false; + entry.used = false; + entry.isPre = false; + entry.fromForwardPreReq = false; + entry.fromBackPreReq = false; + entry.preSign = false; + entry.vmid = hgatp.vmid; + + inl2Entry.vaddr = vaddr; + inl2Entry.asid = vsatp.asid; + inl2Entry.isSquashed = false; + inl2Entry.used = false; + inl2Entry.isPre = false; + inl2Entry.fromForwardPreReq = false; + inl2Entry.fromBackPreReq = false; + inl2Entry.preSign = false; + inl2Entry.vmid = hgatp.vmid; + inl2Entry.paddr = 0; + + finishGVA = mainReq->get_finish_gva(); + level = mainReq->get_level(); + twoStageLevel = mainReq->get_two_stage_level(); + if (finishGVA){ + entry.pteVS = mainReq->get_pte(); + inl2Entry.pteVS = mainReq->get_pte(); + } + if ((!isVsatp0Mode) && (mainReq->get_h_gstage()) && (mainReq->get_two_stage_level() != 2)) { + fault = startTwoStageWalkFromTLBInG(mainReq->get_ppn(), vaddr); + } else if ((!isVsatp0Mode) && (!mainReq->get_h_gstage()) && (mainReq->get_level() != 2)) { + fault = startTwoStageWalkFromTLBNotInG(mainReq->get_ppn(), vaddr); + } else if ((mainReq->get_level() == 2) || (isVsatp0Mode)) { + fault = startTwoStageWalk(gPaddr, vaddr); + } else { + fault = startTwoStageWalk(gPaddr, vaddr); + } + if (fault != NoFault) { + endWalk(); + return fault; + } + } else { + vaddr = Addr(sext(vaddr)); + twoStageLevel = 0; + + nextline = false; + autoNextlineSign = auto_open_nextline; + preHitInPtw = false; + + if (from_l2tlb) { + topAddr = (ppn << PageShift) + (idx * sizeof(PTESv39)); + nextlineLevelMask = LEVEL_MASK; + nextlineShift = shift; + tlbVaddr = vaddr; + tlbppn = ppn; + nextlineRead = topAddr; + nextlineLevel = level; + } else { + topAddr = (satp.ppn << PageShift) + (idx * sizeof(PTESv39)); + nextlineLevelMask = LEVEL_MASK; + nextlineShift = shift; + tlbVaddr = vaddr; + tlbppn = satp.ppn; + nextlineRead = topAddr; + nextlineLevel = level; + } + DPRINTF(PageTableWalker, + "Performing table walk for address %#x shift %d idx_f %#x ppn %#x " + "satp.ppn %#x\n", + vaddr, shift, idx_f, ppn, satp.ppn); + DPRINTF(PageTableWalker, "Loading level%d PTE from %#x idx %#x idx_shift %#x vaddr %#x\n", level, topAddr, idx, + idx << shift, vaddr); - Request::Flags flags = Request::PHYSICAL; - RequestPtr request = std::make_shared(topAddr, 64, flags, walker->requestorId); - if (topAddr == 0) - panic("topAddr can't be 0\n"); - DPRINTF(PageTableWalker," sv39 size is %d\n",sizeof(PTESv39)); + state = Translate; + nextState = Ready; + entry.vaddr = vaddr; + entry.asid = satp.asid; + entry.isSquashed = false; + entry.used = false; + entry.isPre = false; + entry.fromForwardPreReq = from_forward_pre_req; + entry.fromBackPreReq = from_back_pre_req; + entry.preSign = false; + + + nextlineEntry.vaddr = vaddr; + nextlineEntry.asid = satp.asid; + nextlineEntry.isSquashed = false; + nextlineEntry.used = false; + nextlineEntry.isPre = true; + nextlineEntry.fromBackPreReq = from_back_pre_req; + nextlineEntry.preSign = false; + + inl2Entry.asid = satp.asid; + inl2Entry.isSquashed = false; + inl2Entry.used = false; + inl2Entry.isPre = false; + inl2Entry.fromBackPreReq = from_back_pre_req; + inl2Entry.preSign = false; + finishDefaultTranslate = false; + Request::Flags flags = Request::PHYSICAL; + RequestPtr request = std::make_shared(topAddr, 64, flags, walker->requestorId); + if (topAddr == 0) + panic("topAddr can't be 0\n"); + DPRINTF(PageTableWalker, " sv39 size is %d\n", sizeof(PTESv39)); - //Addr - read = new Packet(request, MemCmd::ReadReq); - read->allocate(); + read = new Packet(request, MemCmd::ReadReq); + read->allocate(); + } + return NoFault; } bool @@ -979,7 +1737,13 @@ Walker::WalkerState::recvPacket(PacketPtr pkt) nextState = Ready; PacketPtr write = NULL; read = pkt; - mainFault = stepWalk(write); + if ((translateMode == twoStageMode) && (inGstage)) { + mainFault = twoStageStepWalk(write); + } else if ((translateMode == twoStageMode) && (!inGstage)) { + mainFault = twoStageWalk(write); + } else { + mainFault = stepWalk(write); + } state = Waiting; assert(mainFault == NoFault || read == NULL); if (write) { @@ -1002,6 +1766,33 @@ Walker::WalkerState::recvPacket(PacketPtr pkt) sendPackets(); } + if ((inflight == 0 && read == NULL && writes.size() == 0) && (translateMode == twoStageMode)) { + state = Ready; + nextState = Waiting; + for (auto &r : requestors) { + if (mainFault == NoFault) { + Addr vaddr = r.req->getVaddr(); + //Addr paddr = entry.paddr << PageShift | (vaddr & mask(entry.logBytes)); + Addr paddr = entry.paddr << PageShift | (vaddr & 0xfff); + r.req->setPaddr(paddr); + walker->pma->check(r.req); + mainFault = walker->pmp->pmpCheck(r.req, mode, pmode, r.tc); + if (mainFault != NoFault) { + warn("paddr overflow vaddr: %lx paddr: lx\n", vaddr, paddr); + r.translation->finish(mainFault, r.req, r.tc, mode); + panic("paddr overflow\n"); + return false; + } + r.translation->finish(mainFault, r.req, r.tc, mode); + } + else{ + r.fault = pageFaultOnRequestor(r, GstageFault); + r.translation->finish(r.fault, r.req, r.tc, mode); + DPRINTF(PageTableWalkerTwoStage, "translate fault vaddr %lx\n", mainReq->getVaddr()); + } + } + return true; + } if ((inflight == 0 && read == NULL && writes.size() == 0) && (!nextline)) { state = Ready; @@ -1033,8 +1824,7 @@ Walker::WalkerState::recvPacket(PacketPtr pkt) squashed_num++; } request_num++; - Addr paddr = - walker->tlb->translateWithTLB(vaddr, satp.asid, mode); + Addr paddr = walker->tlb->translateWithTLB(vaddr, satp.asid, mode, direct); r.req->setPaddr(paddr); walker->pma->check(r.req); @@ -1068,7 +1858,7 @@ Walker::WalkerState::recvPacket(PacketPtr pkt) "Finished fault walk for %#lx (pc=%#lx)\n", r.req->getVaddr(), r.req->getPC()); // recreate the fault to ensure that the faulting address matches - r.fault = pageFaultOnRequestor(r); + r.fault = pageFaultOnRequestor(r, false); r.translation->finish(r.fault, r.req, r.tc, mode); } @@ -1110,8 +1900,7 @@ Walker::WalkerState::recvPacket(PacketPtr pkt) squashed_num++; } request_num++; - Addr paddr = - walker->tlb->translateWithTLB(vaddr, satp.asid, mode); + Addr paddr = walker->tlb->translateWithTLB(vaddr, satp.asid, mode, direct); r.req->setPaddr(paddr); walker->pma->check(r.req); @@ -1136,7 +1925,7 @@ Walker::WalkerState::recvPacket(PacketPtr pkt) r.req->getVaddr(), r.req->getPC(), requestors.size()); // recreate the fault to ensure that the faulting address matches - r.fault = pageFaultOnRequestor(r); + r.fault = pageFaultOnRequestor(r, false); r.translation->finish(r.fault, r.req, r.tc, mode); } } @@ -1218,33 +2007,62 @@ Walker::WalkerState::retry() } Fault -Walker::WalkerState::pageFaultOnRequestor(RequestorState &r) +Walker::WalkerState::pageFaultOnRequestor(RequestorState &r, bool G) { + Addr gpaddr = mainReq->getgPaddr(); + Addr page_start = (entry.vaddr >> PageShift) << PageShift; + if (G && (gpaddrMode == 0) && isVsatp0Mode) { + Addr vaddr = 0; + if (r.req->isInstFetch()) { + if (r.req->getPC() < page_start) { + vaddr = page_start; + } else { + vaddr = r.req->getPC(); + } + + } else { + vaddr = r.req->getVaddr(); + } + gpaddr = ((mainReq->getgPaddr() >> 12) << 12) | (vaddr & 0xfff); + } if (r.req->isInstFetch()) { - Addr page_start = (entry.vaddr >> PageShift) << PageShift; if (r.req->getPC() < page_start) { // expected: instruction crosses the page boundary if (!r.req->getPC() + 4 >= entry.vaddr) { - warn("Unexepected fetch page fault: PC: %#x, Page: %#x\n", - r.req->getPC(), entry.vaddr); + warn("Unexepected fetch page fault: PC: %#x, Page: %#x\n", r.req->getPC(), entry.vaddr); } - return walker->tlb->createPagefault(page_start, mode); + return walker->tlb->createPagefault(page_start, gpaddr, mode, G); } else { - return walker->tlb->createPagefault(r.req->getPC(), mode); + return walker->tlb->createPagefault(r.req->getPC(), gpaddr, mode, G); } } else { - return walker->tlb->createPagefault(r.req->getVaddr(), mode); + return walker->tlb->createPagefault(r.req->getVaddr(), gpaddr, mode, G); } } +Addr +Walker::WalkerState::getGVPNi(Addr vaddr, int level) +{ + if (level == 2) + return vaddr >> VpniShift(level) & TWO_STAGE_L2_LEVEL_MASK; + else + return vaddr >> VpniShift(level) & VPN_MASK; +} + +Addr +Walker::WalkerState::VpniShift(int level) +{ + return PGSHFT + LEVEL_BITS * level; +} + Fault -Walker::WalkerState::pageFault(bool present) +Walker::WalkerState::pageFault(bool present,bool G) { bool found_main = false; for (auto &r: requestors) { DPRINTF(PageTableWalker, "Mark page fault for req %#lx (pc=%#lx).\n", r.req->getVaddr(), r.req->getPC()); - auto _fault = pageFaultOnRequestor(r); + auto _fault = pageFaultOnRequestor(r, G); if (r.req->getVaddr() == mainReq->getVaddr()) { mainFault = _fault; found_main = true; diff --git a/src/arch/riscv/pagetable_walker.hh b/src/arch/riscv/pagetable_walker.hh index 74eb1b96ec..f04a5c999b 100644 --- a/src/arch/riscv/pagetable_walker.hh +++ b/src/arch/riscv/pagetable_walker.hh @@ -136,6 +136,7 @@ namespace RiscvISA State state; State nextState; int level; + int twoStageLevel; unsigned inflight; TlbEntry entry; TlbEntry inl2Entry; @@ -143,9 +144,14 @@ namespace RiscvISA std::vector writes; Fault mainFault; BaseMMU::Mode mode; + bool isHInst; + bool isVsatp0Mode; SATP satp; + SATP vsatp; STATUS status; + STATUS vsstatus; PrivilegeMode pmode; + HGATP hgatp; bool functional; bool timing; bool retrying; @@ -161,6 +167,8 @@ namespace RiscvISA Addr nextlineShift; Addr tlbVaddr; Addr tlbppn; + Addr gPaddr; + int vaddr_choose_flag; Addr tlbSizePte; bool openNextline; bool autoNextlineSign; @@ -168,24 +176,37 @@ namespace RiscvISA bool preHitInPtw; bool fromPre; bool fromBackPre; + bool virt; + int translateMode; + bool inGstage; + bool finishGVA; + int gpaddrMode; + bool finishGPA; + bool GstageFault; + bool tlbHit; + PTESv39 tlbHitPte; + Request::Flags tlbflags; public: WalkerState(Walker * _walker, BaseMMU::Translation *_translation, const RequestPtr &_req, bool _isFunctional = false) : walker(_walker), mainReq(_req), state(Ready), - nextState(Ready), level(0), inflight(0), + nextState(Ready), level(0), twoStageLevel(2),inflight(0), functional(_isFunctional), timing(false), retrying(false), started(false), squashed(false), nextline(false), nextlineRead(0), nextlineLevel(0), nextlineVaddr(0), nextlineLevelMask(0), nextlineShift(0), tlbVaddr(0), tlbppn(0), + gPaddr(0),vaddr_choose_flag(0), tlbSizePte(0), openNextline(false), autoNextlineSign(false), finishDefaultTranslate(false), preHitInPtw(false), fromPre(false), - fromBackPre(false) + fromBackPre(false),virt(0),translateMode(0),inGstage(false),finishGVA(false), + gpaddrMode(0),finishGPA(false),GstageFault(false), + tlbHit(false),tlbHitPte(0),tlbflags(Request::PHYSICAL) { requestors.emplace_back(nullptr, _req, _translation); } - void initState(ThreadContext *_tc, BaseMMU::Mode _mode, + void initState(ThreadContext *_tc, const RequestPtr &_req,BaseMMU::Mode _mode, bool _isTiming = false, bool _from_forward_pre_req = false, bool _from_back_pre_req = false); @@ -211,16 +232,25 @@ namespace RiscvISA bool anyRequestorSquashed() const; bool allRequestorSquashed() const; - - private: - void setupWalk(Addr ppn, Addr vaddr, int f_level, bool from_l2tlb, + Fault setupWalk(Addr ppn, Addr vaddr, int f_level, bool from_l2tlb, bool open_nextline, bool auto_openNextline, bool from_forward_pre_req, bool from_back_pre_req); + + private: + Fault startTwoStageWalk(Addr ppn, Addr vaddr); + Fault startTwoStageWalkFromTLBNotInG(Addr ppn, Addr vaddr); + Fault startTwoStageWalkFromTLBInG(Addr ppn, Addr vaddr); + + Fault twoStageStepWalk(PacketPtr &write); + Fault twoStageWalk(PacketPtr &write); Fault stepWalk(PacketPtr &write); void sendPackets(); void endWalk(); - Fault pageFault(bool present); - Fault pageFaultOnRequestor(RequestorState &requestor); + Fault endGstageWalk(); + Fault pageFault(bool present, bool G); + Fault pageFaultOnRequestor(RequestorState &requestor, bool G); + Addr getGVPNi(Addr vaddr, int level); + Addr VpniShift(int level); }; struct L2TlbState @@ -231,8 +261,9 @@ namespace RiscvISA BaseMMU::Translation *translation; BaseMMU::Mode mode; Addr Paddr; - TlbEntry entry; - + TlbEntry *entry; + TlbEntry *entryVsstage; + TlbEntry *entryGstage; }; std::list L2TLBrequestors; @@ -261,7 +292,8 @@ namespace RiscvISA void doL2TLBHitSchedule(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode, Addr Paddr, - const TlbEntry &entry); + TlbEntry *entry,TlbEntry *entryVsstage, + TlbEntry *entryGstage,int delaytick); @@ -281,6 +313,7 @@ namespace RiscvISA protected: // The TLB we're supposed to load. TLB * tlb; + TLB * l2tlb; System * sys; PMAChecker * pma; PMP * pmp; @@ -322,6 +355,10 @@ namespace RiscvISA { tlb = _tlb; } + void setL2TLB(TLB * _l2tlb) + { + l2tlb = _l2tlb; + } using Params = RiscvPagetableWalkerParams; diff --git a/src/arch/riscv/pmp.cc b/src/arch/riscv/pmp.cc index 3239d8a69d..db900cf805 100644 --- a/src/arch/riscv/pmp.cc +++ b/src/arch/riscv/pmp.cc @@ -138,7 +138,7 @@ PMP::createAddrfault(Addr vaddr, BaseMMU::Mode mode) code = RiscvISA::ExceptionCode::INST_ACCESS; } warn("pmp access fault.mode %d vaddr %lx \n",mode , vaddr); - return std::make_shared(vaddr, code); + return std::make_shared(vaddr, 0, code); } inline uint8_t @@ -163,6 +163,12 @@ PMP::pmpUpdateCfg(uint32_t pmp_index, uint8_t this_cfg) } +uint64_t +PMP::pmpTorMask() +{ + return -((uint64_t)1 << (CONFIG_PMP_GRANULARITY - PMP_SHIFT)); +} + void PMP::pmpUpdateRule(uint32_t pmp_index) { @@ -211,7 +217,31 @@ PMP::pmpUpdateRule(uint32_t pmp_index) } } } - +bool +PMP::pmp_read_config(uint64_t cfg) +{ + return (cfg & PMP_A) >= NEMU_PMP_NAPOT; +} +uint64_t +PMP::pmpcfg_from_index(uint32_t pmp_index){ + int xlen =64; + assert(pmp_index <16); + int cfgPerCSR = xlen / 8; + int cfg_csr_addr; + switch (pmp_index / cfgPerCSR) { + case 0: + cfg_csr_addr = 0; + break; // MISCREG_PMPCFG0 + case 1: + cfg_csr_addr = 8; + break; // MISCREG_PMPCFG1 + default: + assert(0); + } + uint32_t now_pmp_index = (pmp_index % cfgPerCSR) + cfg_csr_addr; + DPRINTF(PMP, "return num %x yushu %d\n", pmpTable[now_pmp_index].pmpCfg, pmp_index % cfgPerCSR); + return pmpTable[now_pmp_index].pmpCfg; +} void PMP::pmpUpdateAddr(uint32_t pmp_index, Addr this_addr) { diff --git a/src/arch/riscv/pmp.hh b/src/arch/riscv/pmp.hh index b802c56315..f135c5ea39 100644 --- a/src/arch/riscv/pmp.hh +++ b/src/arch/riscv/pmp.hh @@ -85,9 +85,16 @@ class PMP : public SimObject /** pmpcfg address range execute permission mask */ const uint8_t PMP_EXEC = 1 << 2; + const uint8_t PMP_A = 0x18; + + const uint8_t NEMU_PMP_NAPOT = 0x18; + /** pmpcfg address range locked mask */ const uint8_t PMP_LOCK = 1 << 7; + const uint64_t CONFIG_PMP_GRANULARITY = 12; + const uint64_t PMP_SHIFT = 2; + /** variable to keep track of active number of rules any time */ int numRules; @@ -130,6 +137,11 @@ class PMP : public SimObject */ void pmpUpdateCfg(uint32_t pmp_index, uint8_t this_cfg); + uint64_t pmpTorMask(); + bool pmp_read_config(uint64_t cfg); + + uint64_t pmpcfg_from_index(uint32_t pmp_inde); + /** * pmpUpdateAddr updates the pmpaddr for a pmp * entry and calls pmpUpdateRule to update the diff --git a/src/arch/riscv/regs/misc.hh b/src/arch/riscv/regs/misc.hh index 41492fa831..dc054f7393 100644 --- a/src/arch/riscv/regs/misc.hh +++ b/src/arch/riscv/regs/misc.hh @@ -63,6 +63,7 @@ namespace RiscvISA enum MiscRegIndex { MISCREG_PRV = 0, + MISCREG_VIRMODE, MISCREG_ISA, MISCREG_VENDORID, MISCREG_ARCHID, @@ -74,6 +75,35 @@ enum MiscRegIndex MISCREG_CYCLE, MISCREG_TIME, MISCREG_INSTRET, + MISCREG_MHPMCOUNTER3, + MISCREG_MHPMCOUNTER4, + MISCREG_MHPMCOUNTER5, + MISCREG_MHPMCOUNTER6, + MISCREG_MHPMCOUNTER7, + MISCREG_MHPMCOUNTER8, + MISCREG_MHPMCOUNTER9, + MISCREG_MHPMCOUNTER10, + MISCREG_MHPMCOUNTER11, + MISCREG_MHPMCOUNTER12, + MISCREG_MHPMCOUNTER13, + MISCREG_MHPMCOUNTER14, + MISCREG_MHPMCOUNTER15, + MISCREG_MHPMCOUNTER16, + MISCREG_MHPMCOUNTER17, + MISCREG_MHPMCOUNTER18, + MISCREG_MHPMCOUNTER19, + MISCREG_MHPMCOUNTER20, + MISCREG_MHPMCOUNTER21, + MISCREG_MHPMCOUNTER22, + MISCREG_MHPMCOUNTER23, + MISCREG_MHPMCOUNTER24, + MISCREG_MHPMCOUNTER25, + MISCREG_MHPMCOUNTER26, + MISCREG_MHPMCOUNTER27, + MISCREG_MHPMCOUNTER28, + MISCREG_MHPMCOUNTER29, + MISCREG_MHPMCOUNTER30, + MISCREG_MHPMCOUNTER31, MISCREG_HPMCOUNTER03, MISCREG_HPMCOUNTER04, MISCREG_HPMCOUNTER05, @@ -103,6 +133,7 @@ enum MiscRegIndex MISCREG_HPMCOUNTER29, MISCREG_HPMCOUNTER30, MISCREG_HPMCOUNTER31, + MISCREG_MCOUNTINHIBIT, MISCREG_HPMEVENT03, MISCREG_HPMEVENT04, MISCREG_HPMEVENT05, @@ -133,8 +164,6 @@ enum MiscRegIndex MISCREG_HPMEVENT30, MISCREG_HPMEVENT31, - MISCREG_MTINST, - MISCREG_MTVAL2, MISCREG_TSELECT, MISCREG_TDATA1, @@ -148,6 +177,7 @@ enum MiscRegIndex MISCREG_MIDELEG, MISCREG_MTVEC, MISCREG_MCOUNTEREN, + MISCREG_MENVCFG, MISCREG_MSCRATCH, MISCREG_MEPC, MISCREG_MCAUSE, @@ -199,7 +229,32 @@ enum MiscRegIndex MISCREG_VTYPE, MISCREG_VLENB, - // These registers are not in the standard, hence does not exist in the + MISCREG_HSTATUS, + MISCREG_HEDELEG, + MISCREG_HIDELEG, + MISCREG_HIE, + MISCREG_HCOUNTEREN, + MISCREG_HGEIE, + MISCREG_HTVAL, + MISCREG_HIP, + MISCREG_HVIP, + MISCREG_HTINST, + MISCREG_HGEIP, + MISCREG_HENVCFG, + MISCREG_HGATP, + MISCREG_HTIMEDELTA, + MISCREG_VSSTATUS, + MISCREG_VSIE, + MISCREG_VSTVEC, + MISCREG_VSSCRATCH, + MISCREG_VSEPC, + MISCREG_VSCAUSE, + MISCREG_VSTVAL, + MISCREG_VSIP, + MISCREG_VSATP, + MISCREG_MTINST, + MISCREG_MTVAL2, + // CSRData map. These are mainly used to provide a minimal implementation // for non-maskable-interrupt in our simple cpu. // non-maskable-interrupt-vector-base-address: NMI version of xTVEC @@ -283,6 +338,7 @@ enum CSRIndex CSR_MIE = 0x304, CSR_MTVEC = 0x305, CSR_MCOUNTEREN = 0x306, + CSR_MENVCFG = 0x30A, CSR_MSCRATCH = 0x340, CSR_MEPC = 0x341, CSR_MCAUSE = 0x342, @@ -310,6 +366,35 @@ enum CSRIndex CSR_PMPADDR15 = 0x3BF, CSR_MCYCLE = 0xB00, CSR_MINSTRET = 0xB02, + CSR_MMHPMCOUNTER3 = 0xB03, + CSR_MMHPMCOUNTER4 = 0xB04, + CSR_MMHPMCOUNTER5 = 0xB05, + CSR_MMHPMCOUNTER6 = 0xB06, + CSR_MMHPMCOUNTER7 = 0xB07, + CSR_MMHPMCOUNTER8 = 0xB08, + CSR_MMHPMCOUNTER9 = 0xB09, + CSR_MMHPMCOUNTER10 = 0xB0A, + CSR_MMHPMCOUNTER11 = 0xB0B, + CSR_MMHPMCOUNTER12 = 0xB0C, + CSR_MMHPMCOUNTER13 = 0xB0D, + CSR_MMHPMCOUNTER14 = 0xB0E, + CSR_MMHPMCOUNTER15 = 0xB0F, + CSR_MMHPMCOUNTER16 = 0xB10, + CSR_MMHPMCOUNTER17 = 0xB11, + CSR_MMHPMCOUNTER18 = 0xB12, + CSR_MMHPMCOUNTER19 = 0xB13, + CSR_MMHPMCOUNTER20 = 0xB14, + CSR_MMHPMCOUNTER21 = 0xB15, + CSR_MMHPMCOUNTER22 = 0xB16, + CSR_MMHPMCOUNTER23 = 0xB17, + CSR_MMHPMCOUNTER24 = 0xB18, + CSR_MMHPMCOUNTER25 = 0xB19, + CSR_MMHPMCOUNTER26 = 0xB1A, + CSR_MMHPMCOUNTER27 = 0xB1B, + CSR_MMHPMCOUNTER28 = 0xB1C, + CSR_MMHPMCOUNTER29 = 0xB1D, + CSR_MMHPMCOUNTER30 = 0xB1E, + CSR_MMHPMCOUNTER31 = 0xB1F, CSR_MHPMCOUNTER03 = 0xC03, CSR_MHPMCOUNTER04 = 0xC04, CSR_MHPMCOUNTER05 = 0xC05, @@ -340,6 +425,7 @@ enum CSRIndex CSR_MHPMCOUNTER30 = 0xC1E, CSR_MHPMCOUNTER31 = 0xC1F, // MHPMCOUNTERH rv32 only + CSR_MMCOUNTINHIBIT = 0x320, CSR_MHPMEVENT03 = 0x323, CSR_MHPMEVENT04 = 0x324, CSR_MHPMEVENT05 = 0x325, @@ -370,9 +456,6 @@ enum CSRIndex CSR_MHPMEVENT30 = 0x33E, CSR_MHPMEVENT31 = 0x33F, - CSR_MTINST = 0x34A, - CSR_MTVAL2 = 0x34B, - CSR_TSELECT = 0x7A0, CSR_TDATA1 = 0x7A1, CSR_TDATA2 = 0x7A2, @@ -387,7 +470,33 @@ enum CSRIndex CSR_VCSR = 0x00F, CSR_VL = 0xC20, CSR_VTYPE = 0xC21, - CSR_VLENB = 0xC22 + CSR_VLENB = 0xC22, + + CSR_HSTATUS = 0x600, + CSR_HEDELEG = 0x602, + CSR_HIDELEG = 0x603, + CSR_HIE = 0x604, + CSR_HCOUNTEREN = 0x606, + CSR_HGEIE = 0x607, + CSR_HTVAL = 0x643, + CSR_HIP = 0x644, + CSR_HVIP = 0x645, + CSR_HTINST = 0x64A, + CSR_HGEIP = 0xE12, + CSR_HENVCFG = 0x60A, + CSR_HGATP = 0x680, + CSR_HTIMEDELTA = 0x605, + CSR_VSSTATUS = 0x200, + CSR_VSIE = 0x204, + CSR_VSTVEC = 0x205, + CSR_VSSCRATCH = 0x240, + CSR_VSEPC = 0x241, + CSR_VSCAUSE = 0x242, + CSR_VSTVAL = 0x243, + CSR_VSIP = 0x244, + CSR_VSATP = 0x280, + CSR_MTINST = 0x34A, + CSR_MTVAL2 = 0x34B, }; struct CSRMetadata @@ -411,6 +520,35 @@ const std::map CSRData = { {CSR_CYCLE, {"cycle", MISCREG_CYCLE}}, {CSR_TIME, {"time", MISCREG_TIME}}, {CSR_INSTRET, {"instret", MISCREG_INSTRET}}, + {CSR_MMHPMCOUNTER3, {"mmhpmcounter03", MISCREG_MHPMCOUNTER3}}, + {CSR_MMHPMCOUNTER4, {"mmhpmcounter04", MISCREG_MHPMCOUNTER4}}, + {CSR_MMHPMCOUNTER5, {"mmhpmcounter05", MISCREG_MHPMCOUNTER5}}, + {CSR_MMHPMCOUNTER6, {"mmhpmcounter06", MISCREG_MHPMCOUNTER6}}, + {CSR_MMHPMCOUNTER7, {"mmhpmcounter07", MISCREG_MHPMCOUNTER7}}, + {CSR_MMHPMCOUNTER8, {"mmhpmcounter08", MISCREG_MHPMCOUNTER8}}, + {CSR_MMHPMCOUNTER9, {"mmhpmcounter09", MISCREG_MHPMCOUNTER9}}, + {CSR_MMHPMCOUNTER10, {"mmhpmcounter10", MISCREG_MHPMCOUNTER10}}, + {CSR_MMHPMCOUNTER11, {"mmhpmcounter11", MISCREG_MHPMCOUNTER11}}, + {CSR_MMHPMCOUNTER12, {"mmhpmcounter12", MISCREG_MHPMCOUNTER12}}, + {CSR_MMHPMCOUNTER13, {"mmhpmcounter13", MISCREG_MHPMCOUNTER13}}, + {CSR_MMHPMCOUNTER14, {"mmhpmcounter14", MISCREG_MHPMCOUNTER14}}, + {CSR_MMHPMCOUNTER15, {"mmhpmcounter15", MISCREG_MHPMCOUNTER15}}, + {CSR_MMHPMCOUNTER16, {"mmhpmcounter16", MISCREG_MHPMCOUNTER16}}, + {CSR_MMHPMCOUNTER17, {"mmhpmcounter17", MISCREG_MHPMCOUNTER17}}, + {CSR_MMHPMCOUNTER18, {"mmhpmcounter18", MISCREG_MHPMCOUNTER18}}, + {CSR_MMHPMCOUNTER19, {"mmhpmcounter19", MISCREG_MHPMCOUNTER19}}, + {CSR_MMHPMCOUNTER20, {"mmhpmcounter20", MISCREG_MHPMCOUNTER20}}, + {CSR_MMHPMCOUNTER21, {"mmhpmcounter21", MISCREG_MHPMCOUNTER21}}, + {CSR_MMHPMCOUNTER22, {"mmhpmcounter22", MISCREG_MHPMCOUNTER22}}, + {CSR_MMHPMCOUNTER23, {"mmhpmcounter23", MISCREG_MHPMCOUNTER23}}, + {CSR_MMHPMCOUNTER24, {"mmhpmcounter24", MISCREG_MHPMCOUNTER24}}, + {CSR_MMHPMCOUNTER25, {"mmhpmcounter25", MISCREG_MHPMCOUNTER25}}, + {CSR_MMHPMCOUNTER26, {"mmhpmcounter26", MISCREG_MHPMCOUNTER26}}, + {CSR_MMHPMCOUNTER27, {"mmhpmcounter27", MISCREG_MHPMCOUNTER27}}, + {CSR_MMHPMCOUNTER28, {"mmhpmcounter28", MISCREG_MHPMCOUNTER28}}, + {CSR_MMHPMCOUNTER29, {"mmhpmcounter29", MISCREG_MHPMCOUNTER29}}, + {CSR_MMHPMCOUNTER30, {"mmhpmcounter30", MISCREG_MHPMCOUNTER30}}, + {CSR_MMHPMCOUNTER31, {"mmhpmcounter31", MISCREG_MHPMCOUNTER31}}, {CSR_HPMCOUNTER03, {"hpmcounter03", MISCREG_HPMCOUNTER03}}, {CSR_HPMCOUNTER04, {"hpmcounter04", MISCREG_HPMCOUNTER04}}, {CSR_HPMCOUNTER05, {"hpmcounter05", MISCREG_HPMCOUNTER05}}, @@ -465,6 +603,7 @@ const std::map CSRData = { {CSR_MIE, {"mie", MISCREG_IE}}, {CSR_MTVEC, {"mtvec", MISCREG_MTVEC}}, {CSR_MCOUNTEREN, {"mcounteren", MISCREG_MCOUNTEREN}}, + {CSR_MENVCFG, {"menvcfg", MISCREG_MENVCFG}}, {CSR_MSCRATCH, {"mscratch", MISCREG_MSCRATCH}}, {CSR_MEPC, {"mepc", MISCREG_MEPC}}, {CSR_MCAUSE, {"mcause", MISCREG_MCAUSE}}, @@ -492,6 +631,7 @@ const std::map CSRData = { {CSR_PMPADDR15, {"pmpaddr15", MISCREG_PMPADDR15}}, {CSR_MCYCLE, {"mcycle", MISCREG_CYCLE}}, {CSR_MINSTRET, {"minstret", MISCREG_INSTRET}}, + {CSR_MHPMCOUNTER03, {"mhpmcounter03", MISCREG_HPMCOUNTER03}}, {CSR_MHPMCOUNTER04, {"mhpmcounter04", MISCREG_HPMCOUNTER04}}, {CSR_MHPMCOUNTER05, {"mhpmcounter05", MISCREG_HPMCOUNTER05}}, @@ -521,6 +661,7 @@ const std::map CSRData = { {CSR_MHPMCOUNTER29, {"mhpmcounter29", MISCREG_HPMCOUNTER29}}, {CSR_MHPMCOUNTER30, {"mhpmcounter30", MISCREG_HPMCOUNTER30}}, {CSR_MHPMCOUNTER31, {"mhpmcounter31", MISCREG_HPMCOUNTER31}}, + {CSR_MMCOUNTINHIBIT,{"mcountinhibit", MISCREG_MCOUNTINHIBIT}}, {CSR_MHPMEVENT03, {"mhpmevent03", MISCREG_HPMEVENT03}}, {CSR_MHPMEVENT04, {"mhpmevent04", MISCREG_HPMEVENT04}}, {CSR_MHPMEVENT05, {"mhpmevent05", MISCREG_HPMEVENT05}}, @@ -551,9 +692,6 @@ const std::map CSRData = { {CSR_MHPMEVENT30, {"mhpmevent30", MISCREG_HPMEVENT30}}, {CSR_MHPMEVENT31, {"mhpmevent31", MISCREG_HPMEVENT31}}, - {CSR_MTINST, {"mtisnt", MISCREG_MTINST}}, - {CSR_MTVAL2, {"mtval2", MISCREG_MTVAL2}}, - {CSR_TSELECT, {"tselect", MISCREG_TSELECT}}, {CSR_TDATA1, {"tdata1", MISCREG_TDATA1}}, {CSR_TDATA2, {"tdata2", MISCREG_TDATA2}}, @@ -568,7 +706,33 @@ const std::map CSRData = { {CSR_VCSR, {"vcsr" , MISCREG_VCSR}}, {CSR_VL, {"vl" , MISCREG_VL}}, {CSR_VTYPE, {"vtype" , MISCREG_VTYPE}}, - {CSR_VLENB, {"VLENB" , MISCREG_VLENB}} + {CSR_VLENB, {"VLENB" , MISCREG_VLENB}}, + + {CSR_HSTATUS, {"hstatus", MISCREG_HSTATUS}}, + {CSR_HEDELEG, {"hedeleg", MISCREG_HEDELEG}}, + {CSR_HIDELEG, {"hideleg", MISCREG_HIDELEG}}, + {CSR_HIE, {"hie", MISCREG_HIE}}, + {CSR_HCOUNTEREN, {"hcounteren", MISCREG_HCOUNTEREN}}, + {CSR_HGEIE, {"hgeie", MISCREG_HGEIE}}, + {CSR_HTVAL, {"htval", MISCREG_HTVAL}}, + {CSR_HIP, {"hip", MISCREG_HIP}}, + {CSR_HVIP, {"hvip", MISCREG_HVIP}}, + {CSR_HTINST, {"htinst", MISCREG_HTINST}}, + {CSR_HGEIP, {"hgeip", MISCREG_HGEIP}}, + {CSR_HENVCFG, {"henvcfg", MISCREG_HENVCFG}}, + {CSR_HGATP, {"hgatp", MISCREG_HGATP}}, + {CSR_HTIMEDELTA, {"htimedalta", MISCREG_HTIMEDELTA}}, + {CSR_VSSTATUS, {"vsstatus", MISCREG_VSSTATUS}}, + {CSR_VSIE, {"vsie", MISCREG_VSIE}}, + {CSR_VSTVEC, {"vstvec", MISCREG_VSTVEC}}, + {CSR_VSSCRATCH, {"vsscratch", MISCREG_VSSCRATCH}}, + {CSR_VSEPC, {"vsepc", MISCREG_VSEPC}}, + {CSR_VSCAUSE, {"vscause", MISCREG_VSCAUSE}}, + {CSR_VSTVAL, {"vstval", MISCREG_VSTVAL}}, + {CSR_VSIP, {"vsip", MISCREG_VSIP}}, + {CSR_VSATP, {"vsatp", MISCREG_VSATP}}, + {CSR_MTINST, {"mtinst", MISCREG_MTINST}}, + {CSR_MTVAL2, {"mtval2", MISCREG_MTVAL2}}, }; /** @@ -580,6 +744,11 @@ const std::map CSRData = { */ BitUnion64(STATUS) Bitfield<63> sd; + Bitfield<62,40> wpri; + Bitfield<39> mpv; + Bitfield<38> gva; + Bitfield<37> mbe; + Bitfield<36> sbe; Bitfield<35, 34> sxl; Bitfield<33, 32> uxl; Bitfield<22> tsr; @@ -601,6 +770,71 @@ BitUnion64(STATUS) Bitfield<0> uie; EndBitUnion(STATUS) +BitUnion64(HSTATUS) + Bitfield<63,34> pad4; + Bitfield<33,32> vsxl; + Bitfield<31,23> pad3; + Bitfield<22> vtsr; + Bitfield<21> vtw; + Bitfield<20> vtvm; + Bitfield<19, 18> pad2; + Bitfield<17,12> vgein; + Bitfield<11,10> pad1; + Bitfield<9> hu; + Bitfield<8> spvp; + Bitfield<7> spv; + Bitfield<6> gva; + Bitfield<5> vsbe; + Bitfield<4,0> pad0; +EndBitUnion(HSTATUS) + +BitUnion64(VSSTATUS) + Bitfield<63> sd; + Bitfield<62,34> pad6; + Bitfield<33,32> uxl; + Bitfield<31,20> pad5; + Bitfield<19> mxr; + Bitfield<18> sum; + Bitfield<17> pad4; + Bitfield<16,15> xs; + Bitfield<14,13> fs; + Bitfield<12,11> pad3; + Bitfield<10,9> vs; + Bitfield<8> spp; + Bitfield<7> pad2; + Bitfield<6> ube; + Bitfield<5> spie; + Bitfield<4,2> pad1; + Bitfield<1> sie; + Bitfield<0> pad0; +EndBitUnion(VSSTATUS) + +BitUnion32(VSSTATUS32) + Bitfield<31> sd; + Bitfield<30,20> pad5; + Bitfield<19> mxr; + Bitfield<18> sum; + Bitfield<17> pad4; + Bitfield<16,15> xs; + Bitfield<14,13> fs; + Bitfield<12,11> pad3; + Bitfield<10,9> vs; + Bitfield<8> spp; + Bitfield<7> pad2; + Bitfield<6> ube; + Bitfield<5> spie; + Bitfield<4,2> pad1; + Bitfield<1> sie; + Bitfield<0> pad0; +EndBitUnion(VSSTATUS32) + +BitUnion64(HGATP) + Bitfield<63,60> mode; + Bitfield<59,58> pad0; + Bitfield<57,44> vmid; + Bitfield<43,0> ppn; +EndBitUnion(HGATP) + /** * These fields are specified in the RISC-V Instruction Set Manual, Volume II, * v1.10 in Figures 3.11 and 3.12, accessible at www.riscv.org. Both the MIP @@ -632,6 +866,30 @@ const RegVal ISA_EXT_C_MASK = 1UL << ('c' - 'a'); const RegVal ISA_EXT_H_MASK = 1UL << ('h' - 'a'); const RegVal MISA_MASK = ISA_MXL_MASK | ISA_EXT_MASK; +const uint64_t NEMU_SV39_SIGN0 = 0; +const uint64_t NEMU_SV39_SIGN1 = 8; +const uint64_t NEMU_SATP_SV39_MASK = 0xf000000000000000; +const uint64_t NEMU_SATP_ASID_LEN = 16; +const uint64_t NEMU_SATP_PADDR_LEN = 24; +const uint64_t NEMU_SATP_PADDR_MAX_LEN = 44; +const uint64_t NEMU_SATP_RIGHT_OFFSET = 60; +const uint64_t NEMU_SATP_MODE_MASK = (uint64_t)8<<(NEMU_SATP_ASID_LEN+NEMU_SATP_PADDR_MAX_LEN); +const uint64_t NEMU_SATP_ASID_MASK = ((((uint64_t)1 <<(NEMU_SATP_ASID_LEN))-1)< CSRMasks = { {CSR_FRM, FRM_MASK}, {CSR_FCSR, FFLAGS_MASK | (FRM_MASK << FRM_OFFSET)}, {CSR_SSTATUS, SSTATUS_MASK}, - {CSR_SIE, SI_MASK}, {CSR_SIP, SI_MASK}, - {CSR_MSTATUS, MSTATUS_MASK}, - {CSR_MISA, MISA_MASK}, - {CSR_MIE, MI_MASK}, - {CSR_MIP, MI_MASK} + {CSR_MISA, MISA_MASK} }; +#define concat_temp(x, y) x ## y +#define concat(x, y) concat_temp(x, y) + +#define CSR_STRUCT_START(name) \ + typedef union { \ + struct { + +#define CSR_STRUCT_END(name) \ + }; \ + uint64_t val; \ + } concat(name, _t); + + + +CSR_STRUCT_START(hstatus) + uint64_t pad0 : 5; + uint64_t vsbe : 1; + uint64_t gva : 1; + uint64_t spv : 1; + uint64_t spvp : 1; + uint64_t hu : 1; + uint64_t pad1 : 2; + uint64_t vgein : 6; + uint64_t pad2 : 2; + uint64_t vtvm : 1; + uint64_t vtw : 1; + uint64_t vtsr : 1; + uint64_t pad3 : 9; + uint64_t vsxl : 2; +CSR_STRUCT_END(hstatus) + +CSR_STRUCT_START(hie) + uint64_t pad0 : 2; + uint64_t vssie : 1; + uint64_t pad1 : 3; + uint64_t vstie : 1; + uint64_t pad2 : 3; + uint64_t vseie : 1; + uint64_t pad3 : 1; + uint64_t sgeie : 1; +CSR_STRUCT_END(hie) + +CSR_STRUCT_START(hip) + uint64_t pad0 : 2; + uint64_t vssip : 1; + uint64_t pad1 : 3; + uint64_t vstip : 1; + uint64_t pad2 : 3; + uint64_t vseip : 1; + uint64_t pad3 : 1; + uint64_t sgeip : 1; +CSR_STRUCT_END(hip) + +CSR_STRUCT_START(hvip) + uint64_t pad0 : 2; + uint64_t vssip : 1; + uint64_t pad1 : 3; + uint64_t vstip : 1; + uint64_t pad2 : 3; + uint64_t vseip : 1; +CSR_STRUCT_END(hvip) + +CSR_STRUCT_START(henvcfg) + uint64_t fiom : 1; + uint64_t pad0 : 3; + uint64_t cbie : 2; + uint64_t cbcfe : 1; + uint64_t cbze : 1; + uint64_t pad1 :54; + uint64_t pbmte : 1; + uint64_t vstce : 1; +CSR_STRUCT_END(henvcfg) + +CSR_STRUCT_START(hgatp) + uint64_t ppn : 44; + uint64_t vmid : 14; + uint64_t pad0 : 2; + uint64_t mode : 4; +CSR_STRUCT_END(hgatp) + +CSR_STRUCT_START(vsstatus) + union{ + struct{ + uint64_t pad0: 1; + uint64_t sie : 1; + uint64_t pad1: 3; + uint64_t spie: 1; + uint64_t ube : 1; + uint64_t pad2: 1; + uint64_t spp : 1; + uint64_t vs : 2; + uint64_t pad3: 2; + uint64_t fs : 2; + uint64_t xs : 2; + uint64_t pad4: 1; + uint64_t sum : 1; + uint64_t mxr : 1; + uint64_t pad5:11; + uint64_t sd : 1; + }_32; + struct{ + uint64_t pad0: 1; + uint64_t sie : 1; + uint64_t pad1: 3; + uint64_t spie: 1; + uint64_t ube : 1; + uint64_t pad2: 1; + uint64_t spp : 1; + uint64_t vs : 2; + uint64_t pad3: 2; + uint64_t fs : 2; + uint64_t xs : 2; + uint64_t pad4: 1; + uint64_t sum : 1; + uint64_t mxr : 1; + uint64_t pad5:12; + uint64_t uxl : 2; + uint64_t pad6:29; + uint64_t sd : 1; + }_64; + }; +CSR_STRUCT_END(vsstatus) + +CSR_STRUCT_START(vsie) + uint64_t pad0 : 1; + uint64_t ssie : 1; + uint64_t pad1 : 3; + uint64_t stie : 1; + uint64_t pad2 : 3; + uint64_t seie : 1; +CSR_STRUCT_END(vsie) + +CSR_STRUCT_START(vstvec) + uint64_t mode : 2; + uint64_t base :62; +CSR_STRUCT_END(vstvec) + +CSR_STRUCT_START(vscause) + union{ + struct{ + uint64_t code:31; + uint64_t intr: 1; + }_32; + struct{ + uint64_t code:63; + uint64_t intr: 1; + }_64; + }; +CSR_STRUCT_END(vscause) + +CSR_STRUCT_START(vsip) + uint64_t pad0 : 1; + uint64_t ssip : 1; + uint64_t pad1 : 3; + uint64_t stip : 1; + uint64_t pad2 : 3; + uint64_t seip : 1; +CSR_STRUCT_END(vsip) + +CSR_STRUCT_START(vsatp) + union{ + struct{ + uint64_t ppn :22; + uint64_t asid : 9; + uint64_t mode : 1; + }_32; + struct{ + uint64_t ppn :44; + uint64_t asid :16; + uint64_t mode : 4; + }_64; + }; +CSR_STRUCT_END(vsatp) + } // namespace RiscvISA } // namespace gem5 diff --git a/src/arch/riscv/tlb.cc b/src/arch/riscv/tlb.cc index cd4f428647..73d6897931 100644 --- a/src/arch/riscv/tlb.cc +++ b/src/arch/riscv/tlb.cc @@ -70,9 +70,10 @@ using namespace RiscvISA; // static Addr -buildKey(Addr vpn, uint16_t asid) +buildKey(Addr vpn, uint16_t asid, uint8_t translateMode) { - return (static_cast(asid) << 48) | vpn; + return (static_cast(asid) << 48) | (static_cast(translateMode & 0x3) << 46) | + (vpn & (((uint64_t)1 << 46) - 1)); } TLB::TLB(const Params &p) : @@ -108,6 +109,13 @@ TLB::TLB(const Params &p) : } walker = p.walker; walker->setTLB(this); + TLB *l2tlb; + if (isStage2) { + l2tlb = this; + } else { + l2tlb = static_cast(nextLevel()); + } + walker->setL2TLB(l2tlb); DPRINTF(TLBVerbose, "tlb11 tlb_size %d size() %d\n", size, tlb.size()); } @@ -271,9 +279,9 @@ TLB::l2TLBEvictLRU(int l2TLBlevel, Addr vaddr) TlbEntry * TLB::lookup(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, - bool sign_used) + bool sign_used,uint8_t translateMode) { - TlbEntry *entry = trie.lookup(buildKey(vpn, asid)); + TlbEntry *entry = trie.lookup(buildKey(vpn, asid, translateMode)); if (!hidden) { if (entry) @@ -320,7 +328,7 @@ TLB::lookup(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, TlbEntry * TLB::lookupForwardPre(Addr vpn, uint64_t asid, bool hidden) { - TlbEntry *entry = trieForwardPre.lookup(buildKey(vpn, asid)); + TlbEntry *entry = trieForwardPre.lookup(buildKey(vpn, asid, 0)); if (!hidden) { if (entry) { entry->lruSeq = nextSeq(); @@ -333,7 +341,7 @@ TLB::lookupForwardPre(Addr vpn, uint64_t asid, bool hidden) TlbEntry * TLB::lookupBackPre(Addr vpn, uint64_t asid, bool hidden) { - TlbEntry *entry = trieBackPre.lookup(buildKey(vpn, asid)); + TlbEntry *entry = trieBackPre.lookup(buildKey(vpn, asid, 0)); if (!hidden) { if (entry) { entry->lruSeq = nextSeq(); @@ -349,41 +357,33 @@ TLB::autoOpenNextline() { TLB *l2tlb; - if (isStage2) + if (isStage2) { l2tlb = this; - else + } else { l2tlb = static_cast(nextLevel()); + } int pre_num_c = l2tlb->AllPre; int removePreUnused_c = l2tlb->RemovePreUnused; - bool auto_nextline; - double precision = - (double)((pre_num_c - removePreUnused_c) / (pre_num_c + 1)); + bool auto_nextline = true; + double precision = (double)((pre_num_c - removePreUnused_c) / (pre_num_c + 1)); if (isOpenAutoNextLine) { if (removePreUnused_c > regulationNum) { if (precision < nextlinePrecision) { - DPRINTF(autoNextline, - "pre_num %d removePreUnused %d precision %f\n", - pre_num_c, removePreUnused_c, precision); + DPRINTF(autoNextline, "pre_num %d removePreUnused %d precision %f\n", pre_num_c, removePreUnused_c, + precision); auto_nextline = false; } - - else - auto_nextline = true; - } else - auto_nextline = true; - } else - auto_nextline = true; - - + } + } return auto_nextline; } void -TLB::updateL2TLBSeq(TlbEntryTrie *Trie_l2, Addr vpn, Addr step, uint16_t asid) +TLB::updateL2TLBSeq(TlbEntryTrie *Trie_l2, Addr vpn, Addr step, uint16_t asid, uint8_t translateMode) { for (int i = 0; i < l2tlbLineSize; i++) { - TlbEntry *m_entry = (*Trie_l2).lookup(buildKey(vpn + step * i, asid)); + TlbEntry *m_entry = (*Trie_l2).lookup(buildKey(vpn + step * i, asid, translateMode)); if (m_entry == nullptr) { DPRINTF(TLB, "l2sp1 vaddr basic %#x vaddr %#x \n", vpn, vpn + step * i); panic("l2 TLB link num is empty\n"); @@ -392,7 +392,8 @@ TLB::updateL2TLBSeq(TlbEntryTrie *Trie_l2, Addr vpn, Addr step, uint16_t asid) } } TlbEntry * -TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f_level, bool sign_used) +TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f_level, bool sign_used, + uint8_t translateMode) { Addr f_vpnl2l1 = (vpn >> (PageShift + 2 * LEVEL_BITS)) << (PageShift + 2 * LEVEL_BITS); @@ -416,24 +417,24 @@ TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f panic("wrong in tlb config\n"); if (f_level == L_L2L1) { - DPRINTF(TLB, "look up l2tlb in l2l1 key %#x\n", buildKey(f_vpnl2l1, asid)); - TlbEntry *entry_l2l1 = trieL2L1.lookup(buildKey(f_vpnl2l1, asid)); + DPRINTF(TLB, "look up l2tlb in l2l1 key %#x\n", buildKey(f_vpnl2l1, asid, translateMode)); + TlbEntry *entry_l2l1 = trieL2L1.lookup(buildKey(f_vpnl2l1, asid, translateMode)); entry_l2 = entry_l2l1; step = 0x1 << (PageShift + 2 * LEVEL_BITS); if ((!hidden) && (entry_l2l1)) - updateL2TLBSeq(&trieL2L1, vpnl2l1, step, asid); + updateL2TLBSeq(&trieL2L1, vpnl2l1, step, asid, translateMode); } if (f_level == L_L2L2) { DPRINTF(TLB, "look up l2tlb in l2l2\n"); - TlbEntry *entry_l2l2 = trieL2L2.lookup(buildKey(f_vpnl2l2, asid)); + TlbEntry *entry_l2l2 = trieL2L2.lookup(buildKey(f_vpnl2l2, asid, translateMode)); entry_l2 = entry_l2l2; step = 0x1 << (PageShift + LEVEL_BITS); if ((!hidden) && (entry_l2l2)) - updateL2TLBSeq(&trieL2L2, vpnl2l2, step, asid); + updateL2TLBSeq(&trieL2L2, vpnl2l2, step, asid, translateMode); } if (f_level == L_L2L3) { DPRINTF(TLB, "look up l2tlb in l2l3\n"); - TlbEntry *entry_l2l3 = trieL2L3.lookup(buildKey(vpn, asid)); + TlbEntry *entry_l2l3 = trieL2L3.lookup(buildKey(vpn, asid, translateMode)); entry_l2 = entry_l2l3; step = 0x1000; bool write_sign = false; @@ -450,7 +451,7 @@ TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f } } for (i = 0; i < l2tlbLineSize; i++) { - TlbEntry *m_entry_l2l3 = trieL2L3.lookup(buildKey((vpnl2l3 + step * i), asid)); + TlbEntry *m_entry_l2l3 = trieL2L3.lookup(buildKey((vpnl2l3 + step * i), asid, translateMode)); if (m_entry_l2l3 == nullptr) { DPRINTF(TLB, "l2l3 vaddr basic %#x vaddr %#x\n", vpnl2l3, vpnl2l3 + step * i); panic("l2l3 TLB link num is empty\n"); @@ -478,7 +479,7 @@ TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f } if (f_level == L_L2sp1) { DPRINTF(TLB, "look up l2tlb in l2sp1\n"); - TlbEntry *entry_l2sp1 = trieL2sp.lookup(buildKey(f_vpnl2l1, asid)); + TlbEntry *entry_l2sp1 = trieL2sp.lookup(buildKey(f_vpnl2l1, asid, translateMode)); entry_l2 = entry_l2sp1; step = 0x1 << (PageShift + 2 * LEVEL_BITS); if (entry_l2sp1) { @@ -487,12 +488,12 @@ TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f return nullptr; } if (!hidden) - updateL2TLBSeq(&trieL2sp, vpnl2sp1, step, asid); + updateL2TLBSeq(&trieL2sp, vpnl2sp1, step, asid, translateMode); } } if (f_level == L_L2sp2) { DPRINTF(TLB, "look up l2tlb in l2sp2\n"); - TlbEntry *entry_l2sp2 = trieL2sp.lookup(buildKey(f_vpnl2l2, asid)); + TlbEntry *entry_l2sp2 = trieL2sp.lookup(buildKey(f_vpnl2l2, asid, translateMode)); entry_l2 = entry_l2sp2; step = 0x1 << (PageShift + LEVEL_BITS); if (entry_l2sp2) { @@ -501,7 +502,7 @@ TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f return nullptr; } if (!hidden) - updateL2TLBSeq(&trieL2sp, vpnl2sp2, step, asid); + updateL2TLBSeq(&trieL2sp, vpnl2sp2, step, asid, translateMode); } } @@ -515,13 +516,18 @@ TLB::lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f } TlbEntry * -TLB::insert(Addr vpn, const TlbEntry &entry,bool squashed_update) +TLB::insert(Addr vpn, const TlbEntry &entry,bool squashed_update,uint8_t translateMode) { DPRINTF(TLBGPre, "insert(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n", vpn, entry.asid, entry.paddr, entry.pte, entry.size()); // If somebody beat us to it, just use that existing entry. - TlbEntry *newEntry = lookup(vpn, entry.asid, BaseMMU::Read, true, false); + TlbEntry *newEntry = nullptr; + if (translateMode == gstage) + newEntry = lookup(vpn, entry.vmid, BaseMMU::Read, true, false, translateMode); + else + newEntry = lookup(vpn, entry.asid, BaseMMU::Read, true, false, translateMode); + if (squashed_update) { if (newEntry) { if (newEntry->isSquashed) { @@ -539,8 +545,8 @@ TLB::insert(Addr vpn, const TlbEntry &entry,bool squashed_update) if (newEntry) { // update PTE flags (maybe we set the dirty/writable flag) newEntry->pte = entry.pte; - Addr newEntryAddr = buildKey(newEntry->vaddr,newEntry->asid); - Addr vpnAddr = buildKey(entry.vaddr,entry.asid); + Addr newEntryAddr = buildKey(newEntry->vaddr, newEntry->asid, translateMode); + Addr vpnAddr = buildKey(entry.vaddr, entry.asid, translateMode); if (newEntry->vaddr != vpn) { DPRINTF(TLBGPre, "tlb in newEntryAddr %#x vpnAddr %#x\n", newEntryAddr, vpnAddr); DPRINTF(TLBGPre, "l1 tlb insert(vpn=%#x, vpn2 %#x asid=%#x): ppn=%#x pte=%#x size=%#x\n", vpn, entry.vaddr, @@ -550,7 +556,6 @@ TLB::insert(Addr vpn, const TlbEntry &entry,bool squashed_update) DPRINTF(TLBGPre, "l1 newEntry->vaddr %#x vpn %#x \n", newEntry->vaddr, vpn); } - assert(newEntry->vaddr == vpn); return newEntry; } @@ -560,7 +565,9 @@ TLB::insert(Addr vpn, const TlbEntry &entry,bool squashed_update) newEntry = freeList.front(); freeList.pop_front(); - Addr key = buildKey(vpn, entry.asid); + Addr key = buildKey(vpn, entry.asid, translateMode); + if (translateMode == gstage) + key = buildKey(vpn, entry.vmid, translateMode); *newEntry = entry; newEntry->lruSeq = nextSeq(); newEntry->vaddr = vpn; @@ -586,7 +593,7 @@ TLB::insertForwardPre(Addr vpn, const TlbEntry &entry) newEntry = freeListForwardPre.front(); freeListForwardPre.pop_front(); - Addr key = buildKey(vpn, entry.asid); + Addr key = buildKey(vpn, entry.asid, 0); *newEntry = entry; newEntry->lruSeq = nextSeq(); newEntry->vaddr = vpn; @@ -608,7 +615,7 @@ TLB::insertBackPre(Addr vpn, const TlbEntry &entry) newEntry = freeListBackPre.front(); freeListBackPre.pop_front(); - Addr key = buildKey(vpn, entry.asid); + Addr key = buildKey(vpn, entry.asid, 0); *newEntry = entry; newEntry->lruSeq = nextSeq(); newEntry->vaddr = vpn; @@ -619,7 +626,7 @@ TLB::insertBackPre(Addr vpn, const TlbEntry &entry) TlbEntry * TLB::L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, TlbEntryTrie *Trie_l2, int sign, - bool squashed_update) + bool squashed_update, uint8_t translateMode) { DPRINTF(TLB, "l2tlb insert(vpn=%#x, vpn2 %#x asid=%#x): ppn=%#x pte=%#x " @@ -628,8 +635,10 @@ TLB::L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, choose); TlbEntry *newEntry; Addr key; - newEntry = - lookupL2TLB(vpn, entry.asid, BaseMMU::Read, true, choose, false); + if (translateMode == gstage) + newEntry = lookupL2TLB(vpn, entry.vmid, BaseMMU::Read, true, choose, false, translateMode); + else + newEntry = lookupL2TLB(vpn, entry.asid, BaseMMU::Read, true, choose, false, translateMode); Addr step = 0; if ((choose == L_L2L1) || (choose == L_L2sp1)) { step = 0x1 << 30; @@ -647,7 +656,13 @@ TLB::L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, newEntry->isSquashed = true; stats.squashedInsertL2++; for (int i = 1; i < l2tlbLineSize; i++) { - newEntry = lookupL2TLB(vpn + step * i, entry.asid, BaseMMU::Read, true, choose, false); + if (translateMode == gstage) { + newEntry = + lookupL2TLB(vpn + step * i, entry.vmid, BaseMMU::Read, true, choose, false, translateMode); + } else { + newEntry = + lookupL2TLB(vpn + step * i, entry.asid, BaseMMU::Read, true, choose, false, translateMode); + } stats.squashedInsertL2++; if (newEntry) { newEntry->isSquashed = true; @@ -659,19 +674,20 @@ TLB::L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, if (newEntry) { newEntry->pte = entry.pte; if (newEntry->vaddr != vpn) { - Addr newEntryAddr = buildKey(newEntry->vaddr,newEntry->asid); - Addr vpnAddr = buildKey(entry.vaddr,entry.asid); + Addr newEntryAddr = ((buildKey(newEntry->vaddr, newEntry->asid, translateMode) >> 12) << 12); + Addr vpnAddr = ((buildKey(entry.vaddr, entry.asid, translateMode) >> 12) << 12); + Addr vpngpaddr = ((buildKey(entry.gpaddr, entry.vmid, translateMode) >> 12) << 12); + warn("vaddr newEntry vaddr %lx vpn %lx key newEntry vaddr %lx vpn %lx vpngpaddr %lx\n", newEntry->vaddr, + vpn, newEntryAddr, vpnAddr, vpngpaddr); + DPRINTF(TLBVerbosel2, "newEntryAddr %#x vpnAddr %#x\n", newEntryAddr, vpnAddr); - DPRINTF(TLBVerbosel2, - "l2tlb insert(vpn=%#x, vpn2 %#x asid=%#x): ppn=%#x " - "pte=%#x size=%#x level %d\n", - vpn, entry.vaddr, entry.asid, entry.paddr, entry.pte, entry.size(), choose); - DPRINTF(TLBVerbosel2, - "newentry(vpn=%#x, vpn2 %#x asid=%#x): ppn=%#x pte=%#x " - "size=%#x level %d\n", - vpn, newEntry->vaddr, newEntry->asid, newEntry->paddr, newEntry->pte, newEntry->size(), choose); + DPRINTF(TLBVerbosel2, "l2tlb insert(vpn=%#x, vpn2 %#x asid=%#x): ppn=%#x pte=%#x size=%#x level %d\n", vpn, + entry.vaddr, entry.asid, entry.paddr, entry.pte, entry.size(), choose); + DPRINTF(TLBVerbosel2, "newentry(vpn=%#x, vpn2 %#x asid=%#x): ppn=%#x pte=%#x size=%#x level %d\n", vpn, + newEntry->vaddr, newEntry->asid, newEntry->paddr, newEntry->pte, newEntry->size(), choose); DPRINTF(TLBVerbosel2, "newEntry->vaddr %#x vpn %#x choose %d\n", newEntry->vaddr, vpn, choose); - assert(0); + if ((newEntryAddr != vpnAddr) && (newEntryAddr != vpngpaddr)) + panic("tlb match but key is wrong\n"); } return newEntry; } @@ -688,7 +704,9 @@ TLB::L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, (*List).pop_front(); - key = buildKey(vpn, entry.asid); + key = buildKey(vpn, entry.asid, translateMode); + if (translateMode == gstage) + key = buildKey(vpn, entry.vmid, translateMode); *newEntry = entry; newEntry->lruSeq = nextSeq(); @@ -717,31 +735,33 @@ TLB::L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, } TlbEntry * -TLB::L2TLBInsert(Addr vpn, const TlbEntry &entry, int level, int choose, int sign, bool squashed_update) +TLB::L2TLBInsert(Addr vpn, const TlbEntry &entry, int level, int choose, int sign, bool squashed_update, + uint8_t translateMode) { TLB *l2tlb; - if (isStage2) + if (isStage2) { l2tlb = this; - else + } else { l2tlb = static_cast(nextLevel()); + } TlbEntry *newEntry = nullptr; DPRINTF(TLB, "choose %d vpn %#x entry->vaddr %#x\n", choose, vpn, entry.vaddr); if (choose == 1) - newEntry = - l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2L1, &l2tlb->trieL2L1, sign, squashed_update); + newEntry = l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2L1, &l2tlb->trieL2L1, sign, + squashed_update, translateMode); else if (choose == 2) - newEntry = - l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2L2, &l2tlb->trieL2L2, sign, squashed_update); + newEntry = l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2L2, &l2tlb->trieL2L2, sign, + squashed_update, translateMode); else if (choose == 3) - newEntry = - l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2L3, &l2tlb->trieL2L3, sign, squashed_update); + newEntry = l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2L3, &l2tlb->trieL2L3, sign, + squashed_update, translateMode); else if (choose == 4) - newEntry = - l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2sp, &l2tlb->trieL2sp, sign, squashed_update); + newEntry = l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2sp, &l2tlb->trieL2sp, sign, + squashed_update, translateMode); else if (choose == 5) - newEntry = - l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2sp, &l2tlb->trieL2sp, sign, squashed_update); + newEntry = l2tlb->L2TLBInsertIn(vpn, entry, choose, &l2tlb->freeListL2sp, &l2tlb->trieL2sp, sign, + squashed_update, translateMode); if (!squashed_update) { assert(newEntry != nullptr); @@ -761,13 +781,14 @@ TLB::demapPage(Addr vpn, uint64_t asid) size_t i; TLB *l2tlb; - if (isStage2) + if (isStage2) { l2tlb = this; - else + } else { l2tlb = static_cast(nextLevel()); + } if ((l2tlb == nullptr) && (!isStage2)) - assert(0); + panic("l2tlb is fault\n"); if (vpn == 0 && asid == 0) { flushAll(); @@ -781,18 +802,22 @@ TLB::demapPage(Addr vpn, uint64_t asid) DPRINTF(TLB, "flush(vpn=%#x, asid=%#x)\n", vpn, asid); DPRINTF(TLB, "l1tlb flush(vpn=%#x, asid=%#x)\n", vpn, asid); if (vpn != 0 && asid != 0) { - - TlbEntry *newEntry = lookup(vpn, asid, BaseMMU::Read, true, false); - if (newEntry) - remove(newEntry - tlb.data()); - l2tlb->demapPageL2(vpn, asid); - + for (uint8_t i = 0; i < 4; i++) { + TlbEntry *newEntry = lookup(vpn, asid, BaseMMU::Read, true, false, i); + if (newEntry) + remove(newEntry - tlb.data()); + l2tlb->demapPageL2(vpn, asid); + } } else { for (i = 0; i < size; i++) { if (tlb[i].trieHandle) { Addr mask = ~(tlb[i].size() - 1); - if ((vpn == 0 || (vpn & mask) == tlb[i].vaddr) && - (asid == 0 || tlb[i].asid == asid)) + if ((vpn == 0 || (vpn & mask) == (tlb[i].vaddr & mask)) && (asid == 0 || tlb[i].asid == asid)) + remove(i); + } + if (tlb[i].trieHandle) { + Addr mask = ~(tlb[i].size() - 1); + if ((vpn == 0 || (vpn & mask) == (tlb[i].gpaddr & mask)) && (asid == 0 || tlb[i].vmid == asid)) remove(i); } } @@ -807,36 +832,61 @@ TLB::demapPageL2(Addr vpn, uint64_t asid) asid &= 0xFFFF; std::vector vpn_vec; std::vector tlb_lists; - vpn_vec.push_back(0); + std::vector tlb_size; Addr vpnl2l1 = (vpn >> (PageShift + 2 * LEVEL_BITS + L2TLB_BLK_OFFSET)) << (PageShift + 2 * LEVEL_BITS + L2TLB_BLK_OFFSET); vpn_vec.push_back(vpnl2l1); tlb_lists.push_back(tlbL2L1.data()); + tlb_size.push_back(l2TlbL1Size); Addr vpnl2l2 = (vpn >> (PageShift + LEVEL_BITS + L2TLB_BLK_OFFSET)) << (PageShift + LEVEL_BITS + L2TLB_BLK_OFFSET); vpn_vec.push_back(vpnl2l2); tlb_lists.push_back(tlbL2L2.data()); + tlb_size.push_back(l2TlbL2Size); Addr vpnl2l3 = (vpn >> (PageShift + L2TLB_BLK_OFFSET)) << (PageShift + L2TLB_BLK_OFFSET); vpn_vec.push_back(vpnl2l3); tlb_lists.push_back(tlbL2L3.data()); + tlb_size.push_back(l2TlbL3Size); Addr vpnl2sp1 = (vpn >> (PageShift + 2 * LEVEL_BITS + L2TLB_BLK_OFFSET)) << (PageShift + 2 * LEVEL_BITS + L2TLB_BLK_OFFSET); vpn_vec.push_back(vpnl2sp1); tlb_lists.push_back(tlbL2Sp.data()); + tlb_size.push_back(l2TlbSpSize); Addr vpnl2sp2 = (vpn >> (PageShift + LEVEL_BITS + L2TLB_BLK_OFFSET)) << (PageShift + LEVEL_BITS + L2TLB_BLK_OFFSET); vpn_vec.push_back(vpnl2sp2); + tlb_lists.push_back(tlbL2Sp.data()); + tlb_size.push_back(l2TlbSpSize); int i; DPRINTF(TLB, "l2 flush(vpn=%#x, asid=%#x)\n", vpn, asid); DPRINTF(TLBVerbose3, "l2tlb flush(vpn=%#x, asid=%#x)\n", vpn, asid); TlbEntry *l2_newEntry[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + TlbEntry *l2_newEntry1[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + TlbEntry *l2_newEntry2[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + + for (int ii = 1; ii < 6; ii++) { + l2_newEntry[ii] = lookupL2TLB(vpn_vec[ii - 1], asid, BaseMMU::Read, true, ii, false, direct); + } for (int ii = 1; ii < 6; ii++) { - l2_newEntry[ii] = lookupL2TLB(vpn_vec[ii], asid, BaseMMU::Read, true, ii, false); + l2_newEntry1[ii] = lookupL2TLB(vpn_vec[ii - 1], asid, BaseMMU::Read, true, ii, true, gstage); + } + for (int ii = 1; ii < 6; ii++) { + l2_newEntry2[ii] = lookupL2TLB(vpn_vec[ii - 1], asid, BaseMMU::Read, true, ii, true, vsstage); } + if (vpn != 0 && asid != 0) { + if (isStage2 || isTheSharedL2) { + for (int i_type = 0; i_type < L2PageTypeNum; i_type++) { + for (i = 0; i < tlb_size[i_type] * l2tlbLineSize; i = i + l2tlbLineSize) { + if ((tlb_lists[i_type] + i)->trieHandle) { + l2TLBRemove(i, i_type + 1); + } + } + } + } for (i = 1; i < 6; i++) { int tlb_i = 0; if (i - 1 > 3) @@ -844,53 +894,46 @@ TLB::demapPageL2(Addr vpn, uint64_t asid) else tlb_i = i - 1; if (l2_newEntry[i]) { - TlbEntry *m_newEntry = lookupL2TLB(vpn_vec[i], asid, BaseMMU::Read, true, i, false); + TlbEntry *m_newEntry = lookupL2TLB(vpn_vec[i - 1], asid, BaseMMU::Read, true, i, false, direct); assert(m_newEntry != nullptr); l2TLBRemove(m_newEntry - tlb_lists[tlb_i], i); } - } - } else { - for (i = 0; i < l2TlbL1Size * l2tlbLineSize; i = i + l2tlbLineSize) { - if (tlbL2L1[i].trieHandle) { - Addr l2l1_mask = ~(tlbL2L1[i].size() - 1); - if ((vpnl2l1 == 0 || (vpnl2l1 & l2l1_mask) == tlbL2L1[i].vaddr) && - (asid == 0 || tlbL2L1[i].asid == asid)) { - l2TLBRemove(i, L_L2L1); - } + if (l2_newEntry1[i]) { + TlbEntry *m_newEntry = lookupL2TLB(vpn_vec[i - 1], asid, BaseMMU::Read, true, i, true, gstage); + assert(m_newEntry != nullptr); + l2TLBRemove(m_newEntry - tlb_lists[tlb_i], i); } - } - for (i = 0; i < l2TlbL2Size * l2tlbLineSize; i = i + l2tlbLineSize) { - if (tlbL2L2[i].trieHandle) { - Addr l2l2_mask = ~(tlbL2L2[i].size() - 1); - if ((vpnl2l2 == 0 || (vpnl2l2 & l2l2_mask) == tlbL2L2[i].vaddr) && - (asid == 0 || tlbL2L2[i].asid == asid)) { - l2TLBRemove(i, L_L2L2); - DPRINTF(TLBVerbose3, "l2l2 remove vaddr %#x vpn %#x vpnl2l3\n", tlbL2L2[i].vaddr, vpn, vpnl2l2); - } + if (l2_newEntry2[i]) { + TlbEntry *m_newEntry = lookupL2TLB(vpn_vec[i - 1], asid, BaseMMU::Read, true, i, true, vsstage); + assert(m_newEntry != nullptr); + l2TLBRemove(m_newEntry - tlb_lists[tlb_i], i); } } - for (i = 0; i < l2TlbL3Size * l2tlbLineSize; i = i + l2tlbLineSize) { - if (tlbL2L3[i].trieHandle) { - Addr l2l3_mask = ~(tlbL2L3[i].size() - 1); - if ((vpnl2l3 == 0 || (vpnl2l3 & l2l3_mask) == tlbL2L3[i].vaddr) && - (asid == 0 || tlbL2L3[i].asid == asid)) { - l2TLBRemove(i, L_L2L3); - DPRINTF(TLBVerbose3, "l2l3 remove vaddr %#x vpn %#x vpnl2l3\n", tlbL2L3[i].vaddr, vpn, vpnl2l3); + } else { + if (isStage2 || isTheSharedL2) { + for (int i_type = 0; i_type < L2PageTypeNum; i_type++) { + for (i = 0; i < tlb_size[i_type] * l2tlbLineSize; i = i + l2tlbLineSize) { + if ((tlb_lists[i_type] + i)->trieHandle) { + l2TLBRemove(i, i_type + 1); + } } } } - for (i = 0; i < l2TlbSpSize * l2tlbLineSize; i++) { - Addr l2sp_mask = ~(tlbL2Sp[i].size() - 1); - if (tlbL2Sp[i].trieHandle) { - if ((vpnl2l1 == 0 || (vpnl2l1 & l2sp_mask) == tlbL2Sp[i].vaddr) && - (asid == 0 || tlbL2Sp[i].asid == asid)) { - l2TLBRemove(i, L_L2sp1); + for (int i_type = 0; i_type < L2PageTypeNum; i_type++) { + for (i = 0; i < tlb_size[i_type] * l2tlbLineSize; i = i + l2tlbLineSize) { + Addr mask = ~((tlb_lists[i_type] + i)->size() - 1); + if ((tlb_lists[i_type] + i)->trieHandle) { + if ((vpn_vec[i_type] == 0 || (vpn_vec[i_type] & mask) == (tlb_lists[i_type] + i)->vaddr) && + (asid == 0 || (tlb_lists[i_type] + i)->asid == asid)) { + l2TLBRemove(i, i_type + 1); + } } - } - if (tlbL2Sp[i].trieHandle) { - if ((vpnl2l2 == 0 || (vpnl2l2 & l2sp_mask) == tlbL2Sp[i].vaddr) && - (asid == 0 || tlbL2Sp[i].asid == asid)) { - l2TLBRemove(i, L_L2sp2); + if ((tlb_lists[i_type] + i)->trieHandle) { + if ((vpn_vec[i_type] == 0 || + (vpn_vec[i_type] & mask) == ((tlb_lists[i_type] + i)->gpaddr & mask)) && + (asid == 0 || (tlb_lists[i_type] + i)->vmid == asid)) { + l2TLBRemove(i, i_type + 1); + } } } } @@ -1049,48 +1092,78 @@ TLB::l2TLBRemove(size_t idx, int choose) Fault TLB::checkPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr, - BaseMMU::Mode mode, PTESv39 pte) + BaseMMU::Mode mode, PTESv39 pte,Addr gpaddr,bool G) { Fault fault = NoFault; if (mode == BaseMMU::Read && !pte.r) { - fault = createPagefault(vaddr, mode); + fault = createPagefault(vaddr, gpaddr, mode, G); } else if (mode == BaseMMU::Write && !pte.w) { - fault = createPagefault(vaddr, mode); + fault = createPagefault(vaddr, gpaddr, mode, G); } else if (mode == BaseMMU::Execute && !pte.x) { - fault = createPagefault(vaddr, mode); + fault = createPagefault(vaddr, gpaddr, mode, G); } if (fault == NoFault) { if (pmode == PrivilegeMode::PRV_U && !pte.u) { - fault = createPagefault(vaddr, mode); - } else if (pmode == PrivilegeMode::PRV_S && pte.u && - (status.sum == 0)) { - fault = createPagefault(vaddr, mode); + fault = createPagefault(vaddr, gpaddr, mode, G); + } else if (pmode == PrivilegeMode::PRV_S && pte.u && (status.sum == 0)) { + fault = createPagefault(vaddr, gpaddr, mode, G); } } return fault; } +std::pair +TLB::checkGPermissions(STATUS status,Addr vaddr,Addr gpaddr,BaseMMU::Mode mode, PTESv39 pte,bool h_inst){ + bool continuePtw = false; + if (pte.v && !pte.r && !pte.w && !pte.x) { + panic("checkGpremission hit in no leaf node\n"); + } else if (!pte.v || (!pte.r && pte.w)) { + return std::make_pair(continuePtw,createPagefault(vaddr, gpaddr, mode, true)); + } else if (!pte.u) { + return std::make_pair(continuePtw,createPagefault(vaddr, gpaddr, mode, true)); + } else if (((mode == BaseMMU::Execute) || (h_inst)) && (!pte.x)) { + return std::make_pair(continuePtw,createPagefault(vaddr, gpaddr, mode, true)); + } else if ((mode == BaseMMU::Read) && (!pte.r && !(status.mxr && pte.x))) { + return std::make_pair(continuePtw,createPagefault(vaddr, gpaddr, mode, true)); + } else if ((mode == BaseMMU::Write) && !(pte.r && pte.w)) { + return std::make_pair(continuePtw,createPagefault(vaddr, gpaddr, mode, true)); + } + return std::make_pair(continuePtw,NoFault); +} + Fault -TLB::createPagefault(Addr vaddr, BaseMMU::Mode mode) +TLB::createPagefault(Addr vaddr, Addr gPaddr,BaseMMU::Mode mode,bool G) { ExceptionCode code; - if (mode == BaseMMU::Read) - code = ExceptionCode::LOAD_PAGE; - else if (mode == BaseMMU::Write) - code = ExceptionCode::STORE_PAGE; - else - code = ExceptionCode::INST_PAGE; + if (G) { + if (mode == BaseMMU::Read) { + code = ExceptionCode::LOAD_G_PAGE; + } else if (mode == BaseMMU::Write) { + code = ExceptionCode::STORE_G_PAGE; + } else { + code = ExceptionCode::INST_G_PAGE; + } + } else { + if (mode == BaseMMU::Read) { + code = ExceptionCode::LOAD_PAGE; + } else if (mode == BaseMMU::Write) { + code = ExceptionCode::STORE_PAGE; + } else { + code = ExceptionCode::INST_PAGE; + } + } + DPRINTF(TLB, "Create page fault #%i on %#lx\n", code, vaddr); - return std::make_shared(vaddr, code); + return std::make_shared(vaddr, gPaddr, code); } Addr -TLB::translateWithTLB(Addr vaddr, uint16_t asid, BaseMMU::Mode mode) +TLB::translateWithTLB(Addr vaddr, uint16_t asid, BaseMMU::Mode mode, uint8_t translateMode) { - TlbEntry *e = lookup(vaddr, asid, mode, false, false); + TlbEntry *e = lookup(vaddr, asid, mode, false, false, translateMode); DPRINTF(TLB, "translateWithTLB vaddr %#x \n", vaddr); assert(e != nullptr); DPRINTF(TLBGPre, "translateWithTLB vaddr %#x paddr %#x\n", vaddr, @@ -1107,18 +1180,17 @@ TLB::L2TLBPagefault(Addr vaddr, BaseMMU::Mode mode, const RequestPtr &req, bool if (req->getPC() < page_l2_start) { DPRINTF(TLBVerbosel2, "vaddr %#x,req_pc %#x,page_l2_start %#x\n", vaddr, req->getPC(), page_l2_start); - return createPagefault(page_l2_start, mode); + return createPagefault(page_l2_start, 0, mode, false); } - return createPagefault(req->getPC(), mode); + return createPagefault(req->getPC(), 0, mode, false); } else { - DPRINTF(TLBVerbosel2, "vaddr 2 %#x,req_pc %#x,get vaddr %#x\n", vaddr, - req->getPC(), req->getVaddr()); + DPRINTF(TLBVerbosel2, "vaddr 2 %#x,req_pc %#x,get vaddr %#x\n", vaddr, req->getPC(), req->getVaddr()); if (is_back_pre) - return createPagefault(req->getBackPreVaddr(), mode); + return createPagefault(req->getBackPreVaddr(), 0, mode, false); else if (isPre) - return createPagefault(req->getForwardPreVaddr(), mode); + return createPagefault(req->getForwardPreVaddr(), 0, mode, false); else - return createPagefault(req->getVaddr(), mode); + return createPagefault(req->getVaddr(), 0, mode, false); } } @@ -1140,7 +1212,7 @@ TLB::L2TLBCheck(PTESv39 pte, int level, STATUS status, PrivilegeMode pmode, Addr } else { if (pte.r || pte.x) { hitInSp = true; - fault = checkPermissions(status, pmode, vaddr, mode, pte); + fault = checkPermissions(status, pmode, vaddr, mode, pte, 0, false); if (fault == NoFault) { if (level >= 1 && pte.ppn0 != 0) { fault = L2TLBPagefault(vaddr, mode, req, isPre, is_back_pre); @@ -1203,10 +1275,11 @@ TLB::sendPreHitOnHitRequest(TlbEntry *e_pre_1, TlbEntry *e_pre_2, const RequestP TlbEntry *e_pre; double pre_precision = 0; TLB *l2tlb; - if (isStage2) + if (isStage2) { l2tlb = this; - else + } else { l2tlb = static_cast(nextLevel()); + } assert(l2tlb != nullptr); pre_entry.vaddr = pre_block; pre_entry.asid = asid; @@ -1238,10 +1311,13 @@ TLB::L2TLBSendRequest(Fault fault, TlbEntry *e_l2tlb, const RequestPtr &req, Thr BaseMMU::Translation *translation, BaseMMU::Mode mode, Addr vaddr, bool &delayed, int level) { Addr paddr; + TlbEntry *e_l2tlbVsstage = nullptr; + TlbEntry *e_l2tlbGstage = nullptr; + if (hitInSp) { if (fault == NoFault) { paddr = e_l2tlb->paddr << PageShift | (vaddr & mask(e_l2tlb->logBytes)); - walker->doL2TLBHitSchedule(req, tc, translation, mode, paddr, *e_l2tlb); + walker->doL2TLBHitSchedule(req, tc, translation, mode, paddr, e_l2tlb, e_l2tlbVsstage, e_l2tlbGstage, 1); delayed = true; return std::make_pair(true, fault); } @@ -1255,6 +1331,402 @@ TLB::L2TLBSendRequest(Fault fault, TlbEntry *e_l2tlb, const RequestPtr &req, Thr return std::make_pair(false, fault); } +std::pair +TLB::checkHL1Tlb(const RequestPtr &req, ThreadContext *tc, + BaseMMU::Translation *translation, BaseMMU::Mode mode) +{ + Addr vaddr = Addr(sext(req->getVaddr())); + SATP vsatp = tc->readMiscReg(MISCREG_VSATP); + HGATP hgatp = tc->readMiscReg(MISCREG_HGATP); + STATUS status = tc->readMiscReg(MISCREG_STATUS); + Fault fault = NoFault; + PrivilegeMode pmode = getMemPriv(tc, mode); + bool continuePtw =false; + Addr ppn = 0; + uint64_t pte = 0; + TlbEntry *e[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + int hit_type = H_L1miss; + TlbEntry *e_l2tlb = nullptr; + TlbEntry *e_l2tlbVsstage = nullptr; + TlbEntry *e_l2tlbGstage = nullptr; + if (vsatp.mode != 0) + e[0] = lookup(vaddr, vsatp.asid, mode, false, true, allstage); + else + e[0] = lookup(vaddr, hgatp.vmid, mode, false, true, gstage); + + if (e[0]) { + hit_type = h_l1AllstageHit; + assert(hgatp.vmid == e[0]->vmid); + if (vsatp.mode != 0) { + if ((mode == BaseMMU::Write && !e[0]->pteVS.d) || (!e[0]->pteVS.a)) + fault = createPagefault(vaddr, 0, mode, false); + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } + fault = checkPermissions(status, pmode, vaddr, mode, e[0]->pteVS, 0, false); + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } + } + Addr fault_gpaddr = ((e[0]->gpaddr >> 12) << 12) | (vaddr & 0xfff); + + std::pair(continuePtw,fault) = checkGPermissions(status,vaddr,fault_gpaddr,mode,e[0]->pte,req->get_h_inst()); + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } + + Addr paddr = e[0]->paddr << PageShift | (vaddr & mask(e[0]->logBytes)); + if (e[0]->level > 0) { + paddr = (paddr >> (PageShift + e[0]->level * 9)) << (PageShift + e[0]->level * 9) | + (vaddr & mask(e[0]->logBytes)); + } + req->setPaddr(paddr); + return std::make_pair(hit_type,NoFault); + } else { + Addr pgBase = vsatp.ppn << PageShift; + Addr gPaddr = 0; + + Addr paddrBase = 0; + Addr pg_mask = 0; + + + e[0] = lookup(vaddr, vsatp.asid, mode, false, true, vsstage); + if (e[0]){ + req->setPte(e[0]->pte); + hit_type = h_l1VSstageHit; + gPaddr = e[0]->pte.ppn <<12; + if (e[0]->level >0){ + pg_mask = (1ULL << (12 + 9 * e[0]->level)) - 1; + gPaddr = ((e[0]->pte.ppn << 12) & ~pg_mask) | (vaddr & pg_mask & ~PGMASK); + } + gPaddr = gPaddr | (vaddr & PGMASK); + if (mode == BaseMMU::Write && !e[0]->pte.d) { + fault = createPagefault(vaddr, 0, mode, false); + //return fault; + return std::make_pair(hit_type,fault); + } else { + fault = checkPermissions(status, pmode, vaddr, mode, e[0]->pte, 0, false); + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } + } + req->setPaddr(gPaddr); + ppn = e[0]->pte.ppn; + pte = e[0]->pte; + + e[0] = lookup(gPaddr, hgatp.vmid, mode, false, true, gstage); + if (e[0]) { + hit_type = h_l1GstageHit; + std::pair(continuePtw, fault) = + checkGPermissions(status, vaddr, gPaddr, mode, e[0]->pte, req->get_h_inst()); + if (e[0]->level >0){ + pg_mask = (1ULL << (12 + 9 * e[0]->level)) - 1; + pgBase = ((e[0]->pte.ppn << 12) & ~pg_mask) | (gPaddr & pg_mask & ~PGMASK); + } + gPaddr = pgBase |(gPaddr & PGMASK); + + + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } + + + walker->doL2TLBHitSchedule(req, tc, translation, mode, gPaddr, e_l2tlb, e_l2tlbVsstage, e_l2tlbGstage, + 3); + return std::make_pair(hit_type,NoFault); + } else { + req->setTwoPtwWalk(true, 0, 2, ppn, true); + req->setgPaddr(gPaddr); + return std::make_pair(hit_type, NoFault); + } + } else { + req->setTwoPtwWalk(false, 2, 2, ppn, false); + req->setgPaddr(gPaddr); + return std::make_pair(hit_type, NoFault); + } + } +} + +std::pair +TLB::checkHL2Tlb(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode, + int l1tlbtype) +{ + Addr vaddr = Addr(sext(req->getVaddr())); + SATP vsatp = tc->readMiscReg(MISCREG_VSATP); + HGATP hgatp = tc->readMiscReg(MISCREG_HGATP); + STATUS status = tc->readMiscReg(MISCREG_STATUS); + STATUS vstatus = tc->readMiscReg(MISCREG_VSSTATUS); + Fault fault = NoFault; + PrivilegeMode pmode = getMemPriv(tc, mode); + bool continuePtw = false; + TlbEntry *e[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + int hit_type = H_L1miss; + int hit_level = 2; + int hit_flag = 0; + Addr gPaddr = 0; + Addr pg_mask = 0; + int level = 2; + int twoStageLevel = 2; + bool finishgva = false; + hitInSp = false; + + Addr pgBase = vsatp.ppn << PageShift; + + TLB *l2tlb; + if (isStage2) { + l2tlb = this; + } else { + l2tlb = static_cast(nextLevel()); + } + + assert(l2tlb != nullptr); + // first check + TlbEntry *e_l2tlb = nullptr; + TlbEntry *e_l2tlbVsstage = nullptr; + TlbEntry *e_l2tlbGstage = nullptr; + + if ((!e[0]) && (l1tlbtype == h_l1VSstageHit)) { + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = l2tlb->lookupL2TLB(req->getgPaddr(), hgatp.vmid, mode, false, i_e, true, gstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + hit_flag = i_e; + } + } + } + if (e[0]) { + e_l2tlbGstage = e[0]; + gPaddr = req->getgPaddr(); + std::pair(continuePtw, fault) = + checkGPermissions(status, vaddr, gPaddr, mode, e[0]->pte, req->get_h_inst()); + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } else if (continuePtw) { + hit_type = h_l2GstageHitContinue; + req->setTwoPtwWalk(true, level, twoStageLevel--, e[0]->pte.ppn, hitInSp); + req->setgPaddr(gPaddr); + return std::make_pair(hit_type, fault); + } else { + hit_type = h_l2GstageHitEnd; + if (e[0]->level > 0) { + pg_mask = (1ULL << (12 + 9 * e[0]->level)) - 1; + pgBase = ((e[0]->pte.ppn << 12) & ~pg_mask) | (gPaddr & pg_mask & ~PGMASK); + } + gPaddr = pgBase | (gPaddr & PGMASK); + walker->doL2TLBHitSchedule(req, tc, translation, mode, gPaddr, e_l2tlb, e_l2tlbVsstage, e_l2tlbGstage, + 4); + return std::make_pair(hit_type, NoFault); + } + } else { + hit_type = h_l2VSstageHitEnd; + return std::make_pair(hit_type, NoFault); + } + } + + if (!e[0]) { + e[0] = lookup(vaddr, vsatp.asid, mode, false, true, vsstage); + if (!e[0]) { + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = l2tlb->lookupL2TLB(vaddr, vsatp.asid, mode, false, i_e, true, vsstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + hit_flag = i_e; + } + } + } + } + if (e[0]) { + e_l2tlbVsstage = e[0]; + hit_type = h_l2VSstageHitContinue; + level = e[0]->level; + fault = L2TLBCheck(e[0]->pte, e[0]->level, vstatus, pmode, vaddr, mode, req, false, false); + finishgva = hitInSp; + req->setPte(e[0]->pte); + uint64_t hit_vaddr = e[0]->vaddr; + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } else { + hit_type = h_l2VSstageHitEnd; + gPaddr = e[0]->pte.ppn << 12; + if (finishgva) { + if (e[0]->level > 0) { + pg_mask = (1ULL << (12 + 9 * e[0]->level)) - 1; + gPaddr = ((e[0]->pte.ppn << 12) & ~pg_mask) | (vaddr & pg_mask & ~PGMASK); + } + gPaddr = gPaddr | (vaddr & PGMASK); + } else { + level--; + Addr shift = (PageShift + LEVEL_BITS * level); + Addr idx_f = (vaddr >> shift) & LEVEL_MASK; + Addr idx = (idx_f >> L2TLB_BLK_OFFSET) << L2TLB_BLK_OFFSET; + gPaddr = (e[0]->pte.ppn << PageShift) + (idx_f * l2tlbLineSize); + } + + e[0] = nullptr; + e[0] = lookup(gPaddr, hgatp.vmid, mode, false, true, gstage); + if (!e[0]) { + for (int i_e = 1; i_e < 6; i_e++) { + e[i_e] = l2tlb->lookupL2TLB(gPaddr, hgatp.vmid, mode, false, i_e, true, gstage); + if (e[i_e]) { + if (e[i_e]->level < hit_level) { + e[0] = e[i_e]; + hit_level = e[i_e]->level; + hit_flag = i_e; + } + } + } + } + if (e[0]) { + e_l2tlbGstage = e[0]; + twoStageLevel = e[0]->level; + req->setgPaddr(gPaddr); + std::pair(continuePtw, fault) = + checkGPermissions(status, vaddr, gPaddr, mode, e[0]->pte, req->get_h_inst()); + if (fault != NoFault) { + return std::make_pair(hit_type, fault); + } else if (continuePtw) { + hit_type = h_l2GstageHitContinue; + req->setTwoPtwWalk(true, level, twoStageLevel--, e[0]->pte.ppn, hitInSp); + req->setgPaddr(gPaddr); + return std::make_pair(hit_type, fault); + } else { + hit_type = h_l2GstageHitEnd; + if (finishgva) { + if (e[0]->level > 0) { + pg_mask = (1ULL << (12 + 9 * e[0]->level)) - 1; + pgBase = ((e[0]->pte.ppn << 12) & ~pg_mask) | (gPaddr & pg_mask & ~PGMASK); + } + gPaddr = pgBase | (gPaddr & PGMASK); + walker->doL2TLBHitSchedule(req, tc, translation, mode, gPaddr, e_l2tlb, e_l2tlbVsstage, + e_l2tlbGstage, 4); + } else { + uint64_t ppppn = e[0]->pte.ppn; + hit_type = h_l2VSstageHitContinue; + req->setTwoPtwWalk(false, level, e[0]->level, e[0]->pte.ppn, hitInSp); + } + return std::make_pair(hit_type, NoFault); + } + + } else { + hit_type = h_l2VSstageHitEnd; + // req->setTwoPtwWalk(false,level,twoStageLevel,0,hitInSp); + req->setTwoPtwWalk(true, level, twoStageLevel, 0, hitInSp); + req->setgPaddr(gPaddr); + if ((gPaddr & ~(((int64_t)1 << 41) - 1)) != 0) { + // this is a excep + fault = createPagefault(vaddr, gPaddr, mode, false); + return std::make_pair(hit_type, fault); + } + return std::make_pair(hit_type, NoFault); + } + } + + } else { + hit_type = H_L1miss; + req->setTwoPtwWalk(false, level, twoStageLevel, 0, hitInSp); + req->setgPaddr(gPaddr); + return std::make_pair(hit_type,NoFault); + } + } + return std::make_pair(hit_type,NoFault); +} + +Fault +TLB::doTwoStageTranslate(const RequestPtr &req, ThreadContext *tc, + BaseMMU::Translation *translation, BaseMMU::Mode mode, + bool &delayed) +{ + Addr vaddr = Addr(sext(req->getVaddr())); + SATP vsatp = tc->readMiscReg(MISCREG_VSATP); + HGATP hgatp = tc->readMiscReg(MISCREG_HGATP); + int virt = tc->readMiscReg(MISCREG_VIRMODE); + STATUS status = tc->readMiscReg(MISCREG_STATUS); + HSTATUS hstatus = tc->readMiscReg(MISCREG_HSTATUS); + int two_stage_pmode = (int)getMemPriv(tc, mode); + Fault fault = NoFault; + PrivilegeMode pmode = getMemPriv(tc, mode); + TlbEntry *e[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; + bool continuePtw = false; + int l1tlbtype = H_L1miss; + + TLB *l2tlb; + if (isStage2) { + l2tlb = this; + } else { + l2tlb = static_cast(nextLevel()); + } + + assert(l2tlb != nullptr); + + int level = 2; + if (mode != BaseMMU::Execute) { + if (status.mprv) { + two_stage_pmode = status.mpp; + virt = status.mpv && (two_stage_pmode != PrivilegeMode::PRV_M); + } + + if (req->get_h_inst()) { + virt = 1; + two_stage_pmode = (PrivilegeMode)(RegVal)hstatus.spvp; + } + } + + if (virt != 0) { + if (vsatp.mode == 0) { + req->setVsatp0Mode(true); + req->setTwoStageState(true, virt, two_stage_pmode); + if ((vaddr & ~(((int64_t)1 << 41) - 1)) != 0 ){ + return createPagefault(vaddr,vaddr,mode,true); + } + } else { + req->setVsatp0Mode(false); + req->setTwoStageState(true, virt, two_stage_pmode); + } + std::pair result = checkHL1Tlb(req, tc, translation, mode); + l1tlbtype = result.first; + fault = result.second; + + if (fault != NoFault) { + return fault; + } else if ((l1tlbtype == h_l1VSstageHit) || (l1tlbtype == H_L1miss)) { + /*fault = walker->start(0, tc, translation, req, mode, false, false, 0, false, 0); + if (translation != nullptr || fault != NoFault) { + delayed = true; + return fault; + } + return fault;*/ + std::pair result = checkHL2Tlb(req, tc, translation, mode, l1tlbtype); + if (result.second != NoFault) { + return result.second; + } + + if ((result.first == h_l2GstageHitContinue) || (result.first == h_l2VSstageHitEnd) || + (result.first == H_L1miss) || (result.first == h_l2VSstageHitContinue)) { + fault = walker->start(0, tc, translation, req, mode, false, false, 0, false, 0); + if (translation != nullptr || fault != NoFault) { + delayed = true; + return fault; + } + return fault; + } else if (result.first == h_l2GstageHitEnd) { + delayed = true; + return fault; + } else { + req->getPaddr(); + } + } else if (l1tlbtype == h_l1GstageHit) { + delayed = true; + return fault; + } else { + req->getPaddr(); + } + } + return fault; +} Fault TLB::doTranslate(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode, @@ -1276,7 +1748,7 @@ TLB::doTranslate(const RequestPtr &req, ThreadContext *tc, TlbEntry *e[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; TlbEntry *forward_pre[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; TlbEntry *back_pre[6] = {nullptr, nullptr, nullptr, nullptr, nullptr, nullptr}; - e[0] = lookup(vaddr, satp.asid, mode, false, true); + e[0] = lookup(vaddr, satp.asid, mode, false, true, direct); Addr paddr = 0; Fault fault = NoFault; Fault fault_return = NoFault; @@ -1286,10 +1758,11 @@ TLB::doTranslate(const RequestPtr &req, ThreadContext *tc, PrivilegeMode pmode = getMemPriv(tc, mode); TLB *l2tlb; - if (isStage2) + if (isStage2) { l2tlb = this; - else + } else { l2tlb = static_cast(nextLevel()); + } assert(l2tlb != nullptr); @@ -1319,10 +1792,10 @@ TLB::doTranslate(const RequestPtr &req, ThreadContext *tc, for (int i_e = 1; i_e < 6; i_e++) { if (!e[0]) - e[i_e] = l2tlb->lookupL2TLB(vaddr, satp.asid, mode, false, i_e, true); + e[i_e] = l2tlb->lookupL2TLB(vaddr, satp.asid, mode, false, i_e, true, direct); - forward_pre[i_e] = l2tlb->lookupL2TLB(forward_pre_block, satp.asid, mode, true, i_e, true); - back_pre[i_e] = l2tlb->lookupL2TLB(back_pre_block, satp.asid, mode, true, i_e, true); + forward_pre[i_e] = l2tlb->lookupL2TLB(forward_pre_block, satp.asid, mode, true, i_e, true, direct); + back_pre[i_e] = l2tlb->lookupL2TLB(back_pre_block, satp.asid, mode, true, i_e, true, direct); } bool return_flag = false; @@ -1336,7 +1809,10 @@ TLB::doTranslate(const RequestPtr &req, ThreadContext *tc, if (fault == NoFault) { paddr = e[0]->paddr << PageShift | (vaddr & mask(e[0]->logBytes)); DPRINTF(TLBVerbosel2, "vaddr %#x,paddr %#x,pc %#x\n", vaddr, paddr, req->getPC()); - walker->doL2TLBHitSchedule(req, tc, translation, mode, paddr, *e[3]); + TlbEntry *e_l2tlbVsstage = nullptr; + TlbEntry *e_l2tlbGstage = nullptr; + walker->doL2TLBHitSchedule(req, tc, translation, mode, paddr, e[3], e_l2tlbVsstage, e_l2tlbGstage, + 1); DPRINTF(TLBVerbosel2, "finish Schedule\n"); delayed = true; if ((forward_pre_block != vaddr_block) && (!forward_pre[3]) && openForwardPre && (!pre_forward)) { @@ -1415,25 +1891,24 @@ TLB::doTranslate(const RequestPtr &req, ThreadContext *tc, delayed = true; return fault; } - e[0] = lookup(vaddr, satp.asid, mode, false, true); + e[0] = lookup(vaddr, satp.asid, mode, false, true, direct); assert(e[0] != nullptr); } } if (!e[0]) - e[0] = lookup(vaddr, satp.asid, mode, false, true); + e[0] = lookup(vaddr, satp.asid, mode, false, true, direct); assert(e[0] != nullptr); status = tc->readMiscReg(MISCREG_STATUS); - pmode = getMemPriv(tc, mode); if (mode == BaseMMU::Write && !e[0]->pte.d) { - fault = createPagefault(vaddr, mode); + fault = createPagefault(vaddr, 0, mode, false); } if (fault == NoFault) { DPRINTF(TLB, "final checkpermission\n"); DPRINTF(TLB, "translate(vpn=%#x, asid=%#x): %#x pc %#x mode %i pte.d %\n", vaddr, satp.asid, paddr, req->getPC(), mode, e[0]->pte.d); - fault = checkPermissions(status, pmode, vaddr, mode, e[0]->pte); + fault = checkPermissions(status, pmode, vaddr, mode, e[0]->pte, 0, false); } @@ -1492,6 +1967,60 @@ TLB::getMemPriv(ThreadContext *tc, BaseMMU::Mode mode) pmode = (PrivilegeMode)(RegVal)status.mpp; return pmode; } +bool +TLB::hasTwoStageTranslation(ThreadContext *tc, const RequestPtr &req, BaseMMU::Mode mode) +{ + STATUS status = (STATUS)tc->readMiscReg(MISCREG_STATUS); + int v_mode = tc->readMiscReg(MISCREG_VIRMODE); + return (req->get_h_inst()) || (status.mprv && status.mpv) || v_mode; +} +Fault +TLB::misalignDataAddrCheck(const RequestPtr &req, BaseMMU::Mode mode) +{ + Addr vaddr = req->getVaddr(); + auto size = req->getSize(); + if (mode != BaseMMU::Execute) { + if (size <= 8) { + if ((vaddr & (size - 1)) != 0) { + if (mode == BaseMMU::Write) + return std::make_shared(req->getVaddr() & 0x7FFFFFFFFF, 0, + ExceptionCode::STORE_ADDR_MISALIGNED); + else + return std::make_shared(req->getVaddr() & 0x7FFFFFFFFF, 0, + ExceptionCode::LOAD_ADDR_MISALIGNED); + } + } + } + + return NoFault; +} + +MMUMode +TLB::isaMMUCheck(ThreadContext *tc, Addr vaddr, BaseMMU::Mode mode) +{ + STATUS status = tc->readMiscReg(MISCREG_STATUS); + PrivilegeMode pp = (PrivilegeMode)tc->readMiscReg(MISCREG_PRV); + SATP satp = tc->readMiscReg(MISCREG_SATP); + SATP vsatp = tc->readMiscReg(MISCREG_VSATP); + int v_mode = tc->readMiscReg(MISCREG_VIRMODE); + HGATP hgatp = tc->readMiscReg(MISCREG_HGATP); + bool vm_enable = (status.mprv && (mode == BaseMMU::Execute) ? status.mpp : pp) < PRV_M && + (satp.mode == 8 || (v_mode && (vsatp.mode == 8 || hgatp.mode == 8))); + Addr vaMask = ((((Addr)1) << (63 - 38 + 1)) - 1); + Addr vaMsbs = vaddr >> 38; + bool vaMsbsOk = (vaMsbs == vaMask) || vaMsbs == 0 || !vm_enable; + bool gpf = false; + if ((v_mode == 1) && (vsatp.mode == 0)) { + Addr maxgpa = ((((Addr)1) << (41)) - 1); + if ((vaddr & ~maxgpa) == 0) { + vaMsbsOk = 1; + } else { + gpf = true; + } + } + assert(vaMsbsOk); + return MMU_DIRECT; +} Fault TLB::translate(const RequestPtr &req, ThreadContext *tc, @@ -1503,18 +2032,44 @@ TLB::translate(const RequestPtr &req, ThreadContext *tc, if (FullSystem) { PrivilegeMode pmode = getMemPriv(tc, mode); SATP satp = tc->readMiscReg(MISCREG_SATP); - if (pmode == PrivilegeMode::PRV_M || satp.mode == AddrXlateMode::BARE) + SATP vsatp = tc->readMiscReg(MISCREG_VSATP); + HGATP hgatp = tc->readMiscReg(MISCREG_HGATP); + int v_mode = tc->readMiscReg(MISCREG_VIRMODE); + bool two_stage_translation = false; + STATUS status = tc->readMiscReg(MISCREG_STATUS); + + if ((pmode == PrivilegeMode::PRV_M || satp.mode == AddrXlateMode::BARE)) req->setFlags(Request::PHYSICAL); Fault fault; if (req->getFlags() & Request::PHYSICAL) { + req->setTwoStageState(false, 0, 0); /** * we simply set the virtual address to physical address */ - req->setPaddr(req->getVaddr()); - fault = NoFault; + + if ((hgatp.mode == 8 || vsatp.mode == 8) && (pmode < PrivilegeMode::PRV_M)) { + fault = doTwoStageTranslate(req, tc, translation, mode, delayed); + } else { + if (req->getVaddr() == 0) + warn("vaddr ==0 pc %lx \n", req->getPC()); + req->setPaddr(req->getVaddr()); + fault = NoFault; + assert(!req->get_h_inst()); + } } else { - fault = doTranslate(req, tc, translation, mode, delayed); + two_stage_translation = hasTwoStageTranslation(tc, req, mode); + if (two_stage_translation) { + fault = misalignDataAddrCheck(req, mode); + if (fault != NoFault) { + return fault; + } + assert((vsatp.mode == 8) || (hgatp.mode == 8)); + fault = doTwoStageTranslate(req, tc, translation, mode, delayed); + } else { + req->setTwoStageState(false, 0, 0); + fault = doTranslate(req, tc, translation, mode, delayed); + } } // according to the RISC-V tests, negative physical addresses trigger @@ -1524,11 +2079,12 @@ TLB::translate(const RequestPtr &req, ThreadContext *tc, ExceptionCode code; if (mode == BaseMMU::Read) code = ExceptionCode::LOAD_ACCESS; - else if (mode == BaseMMU::Write) + else if (mode == BaseMMU::Write) { code = ExceptionCode::STORE_ACCESS; + } else code = ExceptionCode::INST_ACCESS; - fault = std::make_shared(req->getVaddr(), code); + fault = std::make_shared(req->getVaddr(), 0, code); } if (!delayed && fault == NoFault) { @@ -1674,7 +2230,7 @@ TLB::unserialize(CheckpointIn &cp) freeList.pop_front(); newEntry->unserializeSection(cp, csprintf("Entry%d", x)); - Addr key = buildKey(newEntry->vaddr, newEntry->asid); + Addr key = buildKey(newEntry->vaddr, newEntry->asid,0); newEntry->trieHandle = trie.insert(key, TlbEntryTrie::MaxBits - newEntry->logBytes, newEntry); } diff --git a/src/arch/riscv/tlb.hh b/src/arch/riscv/tlb.hh index 6a20613202..6386d859d8 100644 --- a/src/arch/riscv/tlb.hh +++ b/src/arch/riscv/tlb.hh @@ -180,13 +180,14 @@ class TLB : public BaseTLB void takeOverFrom(BaseTLB *old) override {} - TlbEntry *insert(Addr vpn, const TlbEntry &entry, bool suqashed_update); + TlbEntry *insert(Addr vpn, const TlbEntry &entry, bool suqashed_update, uint8_t translateMode); TlbEntry *insertForwardPre(Addr vpn, const TlbEntry &entry); TlbEntry *insertBackPre(Addr vpn, const TlbEntry &entry); - TlbEntry *L2TLBInsert(Addr vpn, const TlbEntry &entry, int level, int choose, int sign, bool squashed_update); + TlbEntry *L2TLBInsert(Addr vpn, const TlbEntry &entry, int level, int choose, int sign, bool squashed_update, + uint8_t translateMode); TlbEntry *L2TLBInsertIn(Addr vpn, const TlbEntry &entry, int choose, EntryList *List, TlbEntryTrie *Trie_l2, - int sign, bool squashed_update); + int sign, bool squashed_update, uint8_t translateMode); // TlbEntry *L2TLB_insert_in(Addr vpn,const TlbEntry &entry,int level); @@ -194,9 +195,13 @@ class TLB : public BaseTLB void demapPage(Addr vaddr, uint64_t asn) override; void demapPageL2(Addr vaddr,uint64_t asn); - Fault checkPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr, - BaseMMU::Mode mode, PTESv39 pte); - Fault createPagefault(Addr vaddr, BaseMMU::Mode mode); + Fault checkPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr, BaseMMU::Mode mode, PTESv39 pte, + Addr gpaddr, bool G); + Fault checkGuestPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr, + BaseMMU::Mode mode, PTESv39 pte); + std::pair checkGPermissions(STATUS status, Addr vaddr, Addr gpaddr, BaseMMU::Mode mode, PTESv39 pte, + bool h_inst); + Fault createPagefault(Addr vaddr, Addr gPaddr,BaseMMU::Mode mode,bool G); PrivilegeMode getMemPriv(ThreadContext *tc, BaseMMU::Mode mode); @@ -216,7 +221,7 @@ class TLB : public BaseTLB */ Port *getTableWalkerPort() override; - Addr translateWithTLB(Addr vaddr, uint16_t asid, BaseMMU::Mode mode); + Addr translateWithTLB(Addr vaddr, uint16_t asid, BaseMMU::Mode mode, uint8_t translateMode); Fault L2TLBPagefault(Addr vaddr, BaseMMU::Mode mode, const RequestPtr &req, bool is_pre, bool is_back_pre); @@ -239,11 +244,12 @@ class TLB : public BaseTLB BaseMMU::Mode mode) override; Fault finalizePhysical(const RequestPtr &req, ThreadContext *tc, BaseMMU::Mode mode) const override; - TlbEntry *lookup(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, - bool sign_used); + TlbEntry *lookup(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, bool sign_used, uint8_t translateMode); TlbEntry *lookupForwardPre(Addr vpn, uint64_t asid, bool hidden); TlbEntry *lookupBackPre(Addr vpn, uint64_t asid, bool hidden); bool autoOpenNextline(); + TlbEntry *lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f_level, bool sign_used, + uint8_t translateMode); @@ -274,8 +280,8 @@ class TLB : public BaseTLB private: uint64_t nextSeq() { return ++lruSeq; } - void updateL2TLBSeq(TlbEntryTrie *Trie_l2,Addr vpn,Addr step, uint16_t asid); - TlbEntry *lookupL2TLB(Addr vpn, uint16_t asid, BaseMMU::Mode mode, bool hidden, int f_level, bool sign_used); + void updateL2TLBSeq(TlbEntryTrie *Trie_l2,Addr vpn,Addr step, uint16_t asid,uint8_t translateMode); + void evictLRU(); void evictForwardPre(); @@ -288,11 +294,21 @@ class TLB : public BaseTLB void removeBackPre(size_t idx); void l2tlbRemoveIn(EntryList *List, TlbEntryTrie *Trie_l2,std::vector&tlb,size_t idx, int choose); void l2TLBRemove(size_t idx, int choose); + bool hasTwoStageTranslation(ThreadContext *tc, const RequestPtr &req, BaseMMU::Mode mode); + Fault misalignDataAddrCheck(const RequestPtr &req, BaseMMU::Mode mode); + MMUMode isaMMUCheck(ThreadContext *tc, Addr vaddr, BaseMMU::Mode mode); Fault translate(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode, bool &delayed); + Fault doTwoStageTranslate(const RequestPtr &req, ThreadContext *tc, + BaseMMU::Translation *translation, BaseMMU::Mode mode, + bool &delayed); + std::pair checkHL1Tlb(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, + BaseMMU::Mode mode); + std::pair checkHL2Tlb(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, + BaseMMU::Mode mode, int l1tlbtype); Fault doTranslate(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode, bool &delayed); diff --git a/src/cpu/BaseCPU.py b/src/cpu/BaseCPU.py index bfa873a08f..5c01db1b2d 100644 --- a/src/cpu/BaseCPU.py +++ b/src/cpu/BaseCPU.py @@ -159,6 +159,8 @@ def takeOverFrom(self, old_cpu): arch_db = Param.ArchDBer(Parent.any, "Arch DB") enable_riscv_vector = Param.Bool(False, "Enable riscv vector extension") + enable_riscv_h = Param.Bool(True, "Enable riscv vector extension") + enable_difftest_inst_trace = Param.Bool(False, "Enable difftest inst trace") enable_mem_dedup = Param.Bool(False, "Enable memory deduplication for difftest and golden memory") def createInterruptController(self): diff --git a/src/cpu/SConscript b/src/cpu/SConscript index e0b1434856..556b8ab545 100644 --- a/src/cpu/SConscript +++ b/src/cpu/SConscript @@ -71,6 +71,7 @@ DebugFlag('Mwait') DebugFlag('Diff') DebugFlag('DumpCommit') DebugFlag('Diff2') +DebugFlag('DiffValue') CompoundFlag('ExecAll', [ 'ExecEnable', 'ExecCPSeq', 'ExecEffAddr', 'ExecFaulting', 'ExecFetchSeq', 'ExecOpClass', 'ExecRegDelta', diff --git a/src/cpu/StaticInstFlags.py b/src/cpu/StaticInstFlags.py index a86c3aac9b..72d2c94009 100644 --- a/src/cpu/StaticInstFlags.py +++ b/src/cpu/StaticInstFlags.py @@ -49,6 +49,7 @@ class StaticInstFlags(Enum): 'IsOper16', 'IsOper32', 'IsOper64', + 'IsHInst', 'IsNop', # Is a no-op (no effect at all). 'IsMov', @@ -96,6 +97,11 @@ class StaticInstFlags(Enum): 'IsSquashAfter', # Squash all uncommitted state after executed + #update vsstatus + 'IsUpdateVsstatusSd', + #update mstatussd + 'IsUpdateMstatusSd', + # hardware transactional memory 'IsHtmStart', # Starts a HTM transaction 'IsHtmStop', # Stops (commits) a HTM transaction diff --git a/src/cpu/base.cc b/src/cpu/base.cc index 43ecfbecbf..2e3aaf1d15 100644 --- a/src/cpu/base.cc +++ b/src/cpu/base.cc @@ -48,8 +48,8 @@ #include #include "arch/generic/tlb.hh" -#include "arch/riscv/regs/misc.hh" #include "arch/riscv/insts/static_inst.hh" +#include "arch/riscv/regs/misc.hh" #include "base/cprintf.hh" #include "base/loader/symtab.hh" #include "base/logging.hh" @@ -60,10 +60,11 @@ #include "cpu/thread_context.hh" #include "debug/Diff.hh" #include "debug/Diff2.hh" +#include "debug/DiffValue.hh" +#include "debug/DumpCommit.hh" #include "debug/Mwait.hh" #include "debug/SyscallVerbose.hh" #include "debug/Thread.hh" -#include "debug/DumpCommit.hh" #include "mem/page_table.hh" #include "params/BaseCPU.hh" #include "sim/clocked_object.hh" @@ -161,6 +162,8 @@ BaseCPU::BaseCPU(const Params &p, bool is_checker) dumpCommitFlag(p.dump_commit), dumpStartNum(p.dump_start), enableRVV(p.enable_riscv_vector), + enableRVHDIFF(p.enable_riscv_h), + enabledifftesInstTrace(p.enable_difftest_inst_trace), noHypeMode(false), enableMemDedup(p.enable_mem_dedup) { @@ -855,6 +858,18 @@ BaseCPU::GlobalStats::GlobalStats(statistics::Group *parent) hostOpRate = simOps / hostSeconds; } +void +BaseCPU::csrDiffMessage(uint64_t gem5_val, uint64_t ref_val, int error_num, uint64_t &error_reg, InstSeqNum seq, + std::string error_csr_name, int &diff_at) +{ + DPRINTF(DiffValue, "Inst [sn:%lli] pc: %#lx\n", seq, diffInfo.pc->instAddr()); + DPRINTF(DiffValue, "Diff at \033[31m%s\033[0m Ref value: \033[31m%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", + error_csr_name, ref_val, gem5_val); + diffInfo.errorCsrsValue[error_num] = 1; + error_reg = gem5_val; + if (!diff_at) + diff_at = ValueDiff; +} @@ -980,7 +995,7 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq) diffAllStates->gem5RegFile.vtype = gem5_val; uint64_t ref_val = diffAllStates->referenceRegFile.vtype; if (gem5_val != ref_val) { - warn("Diff at \033[31m%s\033[0m Ref value: \033[31m" + DPRINTF(DiffValue,"Diff at \033[31m%s\033[0m Ref value: \033[31m" "%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", "vtype", ref_val, gem5_val); if (!diff_at) { @@ -1015,7 +1030,7 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq) diffAllStates->gem5RegFile.vl = gem5_val; ref_val = diffAllStates->referenceRegFile.vl; if (gem5_val != ref_val) { - warn("Diff at \033[31m%s\033[0m Ref value: \033[31m" + DPRINTF(DiffValue,"Diff at \033[31m%s\033[0m Ref value: \033[31m" "%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", "vl", ref_val, gem5_val); if (!diff_at) { @@ -1026,94 +1041,215 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq) // always check some CSR regs { - DPRINTF(Diff, "Check critical CSR regs for Inst [sn:%lli] pc: %#lx\n", seq, diffInfo.pc->instAddr()); // mstatus auto gem5_val = readMiscRegNoEffect( RiscvISA::MiscRegIndex::MISCREG_STATUS, tid); // readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_STATUS, 0); auto ref_val = diffAllStates->referenceRegFile.mstatus; - DPRINTF(Diff, "mstatus:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::mstatus, diffAllStates->gem5RegFile.mstatus, seq, "mstatus", + diff_at); diffMsg += csprintf("Diff at \033[31m%s\033[0m Ref value: \033[31m%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", - "mstatus", ref_val, gem5_val); - diffInfo.errorCsrsValue[CsrRegIndex::mstatus] = 1; - diffAllStates->gem5RegFile.mstatus = gem5_val; - if (!diff_at) - diff_at = ValueDiff; + "mstatus", ref_val, gem5_val); } //stval gem5_val = readMiscRegNoEffect( RiscvISA::MiscRegIndex::MISCREG_STVAL, tid); ref_val = diffAllStates->referenceRegFile.stval; - DPRINTF(Diff, "stval:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::stval, diffAllStates->gem5RegFile.stval, seq, "stval", + diff_at); diffMsg += csprintf("Diff at \033[31m%s\033[0m Ref value: \033[31m%#lx\033" "[0m, GEM5 value: \033[31m%#lx\033[0m\n", "stval", ref_val, gem5_val); - diffInfo.errorCsrsValue[CsrRegIndex::stval] = 1; - diffAllStates->gem5RegFile.stval = gem5_val; - if (!diff_at) - diff_at = ValueDiff; } //DIFFTEST_STVDIFFTEST_STVAL // mcause gem5_val = readMiscRegNoEffect( RiscvISA::MiscRegIndex::MISCREG_MCAUSE, tid); - // readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_MCAUSE, 0); ref_val = diffAllStates->referenceRegFile.mcause; - DPRINTF(Diff, "mcause:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::mcause, diffAllStates->gem5RegFile.mcause, seq, "mcause", + diff_at); diffMsg += csprintf("Diff at \033[31m%s\033[0m Ref value: \033[31m%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", "mcause", ref_val, gem5_val); - diffInfo.errorCsrsValue[CsrRegIndex::mcause] = 1; - diffAllStates->gem5RegFile.mcause = gem5_val; - if (!diff_at) - diff_at = ValueDiff; } // satp gem5_val = readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_SATP, tid); - // readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_SATP, 0); ref_val = diffAllStates->referenceRegFile.satp; - DPRINTF(Diff, "satp:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::satp, diffAllStates->gem5RegFile.satp, seq, "satp", + diff_at); diffMsg += csprintf("CPU%i Diff at \033[31m%s\033[0m Ref value: \033[31m%#lx\033" "[0m, GEM5 value: \033[31m%#lx\033[0m\n", cpuId(), "satp", ref_val, gem5_val); - diffInfo.errorCsrsValue[CsrRegIndex::satp] = 1; - diffAllStates->gem5RegFile.satp = gem5_val; - if (!diff_at) - diff_at = ValueDiff; } // mie gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_IE, tid); - // readMiscReg(RiscvISA::MiscRegIndex::MISCREG_IE, 0); ref_val = diffAllStates->referenceRegFile.mie; - DPRINTF(Diff, "mie:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::mie, diffAllStates->gem5RegFile.mie, seq, "mie", diff_at); diffMsg += csprintf("Diff at \033[31m%s\033[0m Ref value: \033[31m" "%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", "mie", ref_val, gem5_val); - diffInfo.errorCsrsValue[CsrRegIndex::mie] = 1; - diffAllStates->gem5RegFile.mie = gem5_val; - if (!diff_at) - diff_at = ValueDiff; } // mip gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_IP, tid); - // readMiscReg(RiscvISA::MiscRegIndex::MISCREG_IP, 0); ref_val = diffAllStates->referenceRegFile.mip; - DPRINTF(Diff, "mip:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); if (gem5_val != ref_val) { + DPRINTF(DiffValue, "mip:\tGEM5: %#lx,\tREF: %#lx\n", gem5_val, ref_val); diffMsg += csprintf("%s at \033[31m%s\033[0m Ref value: \033[31m%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", gem5_val == ref_val ? "match" : "diff", "mip", ref_val, gem5_val); diffInfo.errorCsrsValue[CsrRegIndex::mip] = 1; diffAllStates->gem5RegFile.mip = gem5_val; } + //mepc + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_MEPC, tid); + ref_val = diffAllStates->referenceRegFile.mepc; + if (gem5_val != ref_val) { + DPRINTF(DiffValue, "Inst [sn:%lli] pc: %#lx\n", seq, diffInfo.pc->instAddr()); + DPRINTF(DiffValue, "Diff at \033[31m%s\033[0m Ref value: \033[31m" + "%#lx\033[0m, GEM5 value: \033[31m%#lx\033[0m\n", + "mepc", ref_val, gem5_val); + diffInfo.errorCsrsValue[CsrRegIndex::mepc] = 1; + diffAllStates->gem5RegFile.mepc = gem5_val; + } + } + + if (enableRVHDIFF){ + //h difftest + //mtval2 + auto gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_MTVAL2, tid); + auto ref_val = diffAllStates->referenceRegFile.mtval2; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::mtval2, diffAllStates->gem5RegFile.mtval2, seq, "mtval2", + diff_at); + } + //mtinst + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_MTINST, tid); + ref_val = diffAllStates->referenceRegFile.mtinst; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::mtinst, diffAllStates->gem5RegFile.mtinst, seq, "mtinst", + diff_at); + } + //hstatus + gem5_val = readMiscRegNoEffect( + RiscvISA::MiscRegIndex::MISCREG_HSTATUS, tid); + ref_val = diffAllStates->referenceRegFile.hstatus; + if ((gem5_val != ref_val)) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::hstatus, diffAllStates->gem5RegFile.hstatus, seq, "hstatus", + diff_at); + } + //hideleg + gem5_val = readMiscRegNoEffect( + RiscvISA::MiscRegIndex::MISCREG_HIDELEG, tid); + ref_val = diffAllStates->referenceRegFile.hideleg; + if ((gem5_val != ref_val)) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::hideleg, diffAllStates->gem5RegFile.hideleg, seq, "hideleg", + diff_at); + } + //hedeleg + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_HEDELEG, tid); + ref_val = diffAllStates->referenceRegFile.hedeleg; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::hedeleg, diffAllStates->gem5RegFile.hedeleg, seq, "hedeleg", + diff_at); + } + //hcounteren + gem5_val = readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_HCOUNTEREN, tid); + ref_val = diffAllStates->referenceRegFile.hcounteren; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::hcounteren, diffAllStates->gem5RegFile.hcounteren, seq, + "hcounteren", diff_at); + } + + //htval + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_HTVAL, tid); + ref_val = diffAllStates->referenceRegFile.htval; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::htval, diffAllStates->gem5RegFile.htval, seq, "htval", + diff_at); + } + + // htinst + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_HTINST, tid); + ref_val = diffAllStates->referenceRegFile.htinst; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::htinst, diffAllStates->gem5RegFile.htinst, seq, "htinst", + diff_at); + } + // hgatp + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_HGATP, tid); + ref_val = diffAllStates->referenceRegFile.hgatp; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::hgatp, diffAllStates->gem5RegFile.hgatp, seq, "hgatp", + diff_at); + } + // vsstatus + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, tid); + ref_val = diffAllStates->referenceRegFile.vsstatus; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vsstatus, diffAllStates->gem5RegFile.vsstatus, seq, + "vsstatus", diff_at); + } + // vstvec + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSTVEC, tid); + ref_val = diffAllStates->referenceRegFile.vstvec; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vstvec, diffAllStates->gem5RegFile.vstvec, seq, "vstvec", + diff_at); + } + // vsepc + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSEPC, tid); + ref_val = diffAllStates->referenceRegFile.vsepc; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vsepc, diffAllStates->gem5RegFile.vsepc, seq, "vsepc", + diff_at); + } + // vscause + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSCAUSE, tid); + ref_val = diffAllStates->referenceRegFile.vscause; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vscause, diffAllStates->gem5RegFile.vscause, seq, "vscause", + diff_at); + } + // vstval + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSTVAL, tid); + ref_val = diffAllStates->referenceRegFile.vstval; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vstval, diffAllStates->gem5RegFile.vstval, seq, "vstval", + diff_at); + } + + // vsatp + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSATP, tid); + ref_val = diffAllStates->referenceRegFile.vsatp; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vsatp, diffAllStates->gem5RegFile.vsatp, seq, "vsatp", + diff_at); + } + + // vsscratch + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSSCRATCH, tid); + ref_val = diffAllStates->referenceRegFile.vsscratch; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::vsscratch, diffAllStates->gem5RegFile.vsscratch, seq, + "vsscratch", diff_at); + } + // cpu.v diff + gem5_val = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VIRMODE, tid); + ref_val = diffAllStates->referenceRegFile.v; + if (gem5_val != ref_val) { + csrDiffMessage(gem5_val, ref_val, CsrRegIndex::v, diffAllStates->gem5RegFile.v, seq, "v", diff_at); + } + } + + if (diff_at != NoneDiff) { DPRINTF(Diff, "Inst [sn:%llu] @ \033[31m%#lx\033[0m in GEM5 is \033[31m%s\033[0m\n", seq, @@ -1123,7 +1259,7 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq) DPRINTF(Diff, "Load addr: %#lx\n", diffInfo.physEffAddr); } } - } + for (int dest_idx = 0; dest_idx < diffInfo.inst->numDestRegs(); dest_idx++) { @@ -1224,7 +1360,7 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq) } } } - if (diff_at) { + if (diff_at && (enabledifftesInstTrace)) { diffMsg += csprintf("In CPU%d: NEMU PC: %#10lx, GEM5 PC: %#10lx, inst: %s\n", cpuId(), nemu_pc, gem5_pc, diffInfo.inst->disassemble(diffInfo.pc->instAddr()).c_str()); @@ -1234,7 +1370,6 @@ BaseCPU::diffWithNEMU(ThreadID tid, InstSeqNum seq) void BaseCPU::clearDiffMismatch(ThreadID tid, InstSeqNum seq) { - DPRINTF(Diff, "Clearing diff mismatch\n"); diffMsg = ""; memset(diffInfo.errorRegsValue, 0, sizeof(diffInfo.errorRegsValue)); memset(diffInfo.errorCsrsValue, 0, sizeof(diffInfo.errorCsrsValue)); @@ -1300,8 +1435,10 @@ BaseCPU::difftestStep(ThreadID tid, InstSeqNum seq) "NEMU seems to commit the failed mem instruction\n"); } } else { - reportDiffMismatch(tid, seq); + if (enabledifftesInstTrace) + reportDiffMismatch(tid, seq); panic("Difftest failed!\n"); + } } else { clearDiffMismatch(tid, seq); @@ -1313,7 +1450,9 @@ BaseCPU::difftestStep(ThreadID tid, InstSeqNum seq) diffInfo.pc->instAddr(), diffInfo.inst->disassemble(diffInfo.pc->instAddr()).c_str())); } - DPRINTF(Diff, "commit_pc: %s, committedInstNum: %d\n", diffInfo.pc, committedInstNum); + if (enabledifftesInstTrace) { + DPRINTF(Diff, "commit_pc: %s, committedInstNum: %d\n", diffInfo.pc, committedInstNum); + } } void @@ -1453,15 +1592,17 @@ void BaseCPU::setSCSuccess(bool success, paddr_t addr) } void -BaseCPU::setExceptionGuideExecInfo(uint64_t exception_num, uint64_t mtval, - uint64_t stval, bool force_set_jump_target, - uint64_t jump_target) +BaseCPU::setExceptionGuideExecInfo(uint64_t exception_num, uint64_t mtval, uint64_t stval, bool force_set_jump_target, + uint64_t jump_target, ThreadID tid) { auto &gd = diffAllStates->diff.guide; gd.force_raise_exception = true; gd.exception_num = exception_num; gd.mtval = mtval; gd.stval = stval; + gd.mtval2 = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_MTVAL2, tid); + gd.htval = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_HTVAL, tid); + gd.vstval = readMiscReg(RiscvISA::MiscRegIndex::MISCREG_VSTVAL, tid); gd.force_set_jump_target = force_set_jump_target; gd.jump_target = jump_target; diff --git a/src/cpu/base.hh b/src/cpu/base.hh index 28cae56a33..02578ae563 100644 --- a/src/cpu/base.hh +++ b/src/cpu/base.hh @@ -82,7 +82,10 @@ enum CsrRegIndex mip, mie, mscratch, sscratch, mideleg, medeleg, - pc + pc ,v, + mtval2, mtinst, hstatus, hideleg, hedeleg, + hcounteren, htval, htinst, hgatp, vsstatus, + vstvec, vsepc, vscause, vstval, vsatp, vsscratch }; struct AddressMonitor @@ -685,14 +688,24 @@ class BaseCPU : public ClockedObject bool dumpCommitFlag; int dumpStartNum; bool enableRVV{false}; + bool enableRVHDIFF{false}; + bool enabledifftesInstTrace{false}; std::shared_ptr diffAllStates{}; + enum diffRegConfig + { + diffAllNum = 96,// 32 regs + 32fprs +32 vprs + diffCsrNum = 36, + }; + virtual void readGem5Regs() { panic("difftest:readGem5Regs() is not implemented\n"); } - std::pair diffWithNEMU(ThreadID tid, InstSeqNum seq); + void csrDiffMessage(uint64_t gem5_val, uint64_t ref_val, int error_num, uint64_t &error_reg, InstSeqNum seq, + std::string error_csr_name,int &diff_at); + std::pair diffWithNEMU(ThreadID tid, InstSeqNum seq); std::string diffMsg; void reportDiffMismatch(ThreadID tid, InstSeqNum seq) { @@ -739,8 +752,8 @@ class BaseCPU : public ClockedObject uint8_t *goldenValue; uint64_t amoOldGoldenValue; // Register address causing difftest error - bool errorRegsValue[96];// 32 regs + 32fprs +32 vprs - bool errorCsrsValue[32];// CsrRegIndex + bool errorRegsValue[diffAllNum]; + bool errorCsrsValue[diffCsrNum]; // CsrRegIndex bool errorPcValue; std::queue lastCommittedMsg; @@ -772,10 +785,9 @@ class BaseCPU : public ClockedObject void setSCSuccess(bool success, paddr_t addr); - void setExceptionGuideExecInfo(uint64_t exception_num, uint64_t mtval, - uint64_t stval, - // force set jump target - bool force_set_jump_target, uint64_t jump_target); + void setExceptionGuideExecInfo(uint64_t exception_num, uint64_t mtval, uint64_t stval, + // force set jump target + bool force_set_jump_target, uint64_t jump_target, ThreadID tid); void clearGuideExecInfo(); diff --git a/src/cpu/difftest.hh b/src/cpu/difftest.hh index 2e8ad31b43..8b41555a16 100644 --- a/src/cpu/difftest.hh +++ b/src/cpu/difftest.hh @@ -60,6 +60,12 @@ struct riscv64_CPU_regfile uint64_t mideleg, medeleg; uint64_t pc; + //hypervisor + uint64_t v; // virtualization mode + uint64_t mtval2, mtinst, hstatus, hideleg, hedeleg; + uint64_t hcounteren, htval, htinst, hgatp, vsstatus; + uint64_t vstvec, vsepc, vscause, vstval, vsatp, vsscratch; + //vector union { @@ -126,6 +132,10 @@ struct ExecutionGuide { uint64_t exception_num; uint64_t mtval; uint64_t stval; + // need enable h + uint64_t mtval2; + uint64_t htval; + uint64_t vstval; // force set jump target bool force_set_jump_target; uint64_t jump_target; diff --git a/src/cpu/o3/commit.cc b/src/cpu/o3/commit.cc index 74f12f4fec..e4c30a2d52 100644 --- a/src/cpu/o3/commit.cc +++ b/src/cpu/o3/commit.cc @@ -165,6 +165,14 @@ Commit::Commit(CPU *_cpu, branch_prediction::BPredUnit *_bp, const BaseO3CPUPara } simout.close(out_handle); }); + + faultNum.insert(RiscvISA::ExceptionCode::LOAD_PAGE); + faultNum.insert(RiscvISA::ExceptionCode::STORE_PAGE); + faultNum.insert(RiscvISA::ExceptionCode::INST_PAGE); + faultNum.insert(RiscvISA::ExceptionCode::INST_G_PAGE); + faultNum.insert(RiscvISA::ExceptionCode::LOAD_G_PAGE); + faultNum.insert(RiscvISA::ExceptionCode::STORE_G_PAGE); + } std::string Commit::name() const { return cpu->name() + ".commit"; } @@ -193,6 +201,8 @@ Commit::CommitStats::CommitStats(CPU *cpu, Commit *commit) "Number of insts commited each cycle"), ADD_STAT(instsCommitted, statistics::units::Count::get(), "Number of instructions committed"), + ADD_STAT(pagefaulttimes, statistics::units::Count::get(), + "Number of pagefaulttimes"), ADD_STAT(opsCommitted, statistics::units::Count::get(), "Number of ops (including micro ops) committed"), ADD_STAT(memRefs, statistics::units::Count::get(), @@ -257,6 +267,10 @@ Commit::CommitStats::CommitStats(CPU *cpu, Commit *commit) .init(cpu->numThreads) .flags(total); + pagefaulttimes + .init(cpu->numThreads) + .flags(total); + opsCommitted .init(cpu->numThreads) .flags(total); @@ -1041,6 +1055,12 @@ Commit::commit() } } +void +Commit::updateMstatusSd(ThreadID tid){ + RiscvISA::STATUS mstatus = cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_STATUS, tid); + mstatus.sd = (mstatus.fs == 3) || (mstatus.vs == 3); + cpu->setMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_STATUS, (RegVal)mstatus, tid); +} void Commit::commitInsts() @@ -1170,6 +1190,28 @@ Commit::commitInsts() } dbftb->notifyInstCommit(head_inst); } + if (head_inst->isUpdateVsstatusSd()) { + auto v = cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VIRMODE, tid); + RiscvISA::HSTATUS hstatus = cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_HSTATUS, tid); + RiscvISA::VSSTATUS vsstatus = + cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, tid); + RiscvISA::VSSTATUS32 vsstatus32 = + cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, tid); + + if (v) { + if (hstatus.vsxl ==1) { + vsstatus32.sd = (vsstatus32.fs == 3); + cpu->setMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, (RegVal)vsstatus32, tid); + } else { + vsstatus.sd = (vsstatus.fs == 3); + cpu->setMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, (RegVal)vsstatus, tid); + } + } + + } + if (head_inst->isUpdateMstatusSd()) { + updateMstatusSd(tid); + } ++num_committed; stats.committedInstType[tid][head_inst->opClass()]++; @@ -1224,6 +1266,31 @@ Commit::commitInsts() tc->getDecoderPtr()->as().setVtype(new_vtype); } } + if (head_inst->isFloating() && head_inst->isLoad()){ + RiscvISA::STATUS status = cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_STATUS, tid); + status.sd = 1; + status.fs = 3; + cpu->setMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_STATUS, (RegVal)status, tid); + } + if (head_inst->isUpdateVsstatusSd()) { + auto v = cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VIRMODE, tid); + RiscvISA::HSTATUS hstatus = cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_HSTATUS, tid); + RiscvISA::VSSTATUS vsstatus = + cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, tid); + RiscvISA::VSSTATUS32 vsstatus32 = + cpu->readMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, tid); + + if (v) { + if (hstatus.vsxl ==1) { + vsstatus32.sd = (vsstatus32.fs == 3); + cpu->setMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, (RegVal)vsstatus32, tid); + } else { + vsstatus.sd = (vsstatus.fs == 3); + cpu->setMiscRegNoEffect(RiscvISA::MiscRegIndex::MISCREG_VSSTATUS, (RegVal)vsstatus, tid); + } + } + + } if (cpu->difftestEnabled()) { diffInst(tid, head_inst); @@ -1431,6 +1498,9 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num) tid, head_inst->seqNum); return false; } + if (faultNum.find(inst_fault->exception()) != faultNum.end()) { + stats.pagefaulttimes[tid]++; + } head_inst->setCompleted(); @@ -1469,11 +1539,6 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num) head_inst->staticInst->disassemble( head_inst->pcState().instAddr()).c_str(), inst_fault->name()); - DPRINTF( - Faults, "[tid:%i] [sn:%llu] Fault instruction machInst: %lx\n", - tid, head_inst->seqNum, - dynamic_cast( - *head_inst->staticInst).machInst); if (head_inst->traceData) { // We ignore ReExecution "faults" here as they are not real @@ -1508,19 +1573,12 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num) cause = cpu->readMiscReg( RiscvISA::MiscRegIndex::MISCREG_UCAUSE, tid); } - if (cause == RiscvISA::ExceptionCode::LOAD_PAGE || - cause == RiscvISA::ExceptionCode::STORE_PAGE || - cause == RiscvISA::ExceptionCode::INST_PAGE) { - DPRINTF(Commit, - "Force to raise No.%lu exception at page fault\n", - cause); + auto exception_no =inst_fault->exception(); + if (faultNum.find(exception_no) != faultNum.end()) { + DPRINTF(Commit, "Force to raise No.%lu exception at page fault\n", inst_fault); cpu->setExceptionGuideExecInfo( - cause, - cpu->readMiscReg( - RiscvISA::MiscRegIndex::MISCREG_MTVAL, tid), - cpu->readMiscReg( - RiscvISA::MiscRegIndex::MISCREG_STVAL, tid), - false, 0); + exception_no, cpu->readMiscReg(RiscvISA::MiscRegIndex::MISCREG_MTVAL, tid), + cpu->readMiscReg(RiscvISA::MiscRegIndex::MISCREG_STVAL, tid), false, 0, tid); } if (cause == RiscvISA::ExceptionCode::ECALL_USER || cause == RiscvISA::ExceptionCode::ECALL_SUPER || diff --git a/src/cpu/o3/commit.hh b/src/cpu/o3/commit.hh index 2203a2af63..8228cf010d 100644 --- a/src/cpu/o3/commit.hh +++ b/src/cpu/o3/commit.hh @@ -126,6 +126,7 @@ class Commit CommitStatus _status; /** Next commit status, to be set at the end of the cycle. */ CommitStatus _nextStatus; + std::set faultNum; /** Per-thread status. */ ThreadStatus commitStatus[MaxThreads]; @@ -303,6 +304,7 @@ class Commit /** Commits as many instructions as possible. */ void commitInsts(); + void updateMstatusSd(ThreadID tid); /** Tries to commit the head ROB instruction passed in. * @param head_inst The instruction to be committed. @@ -519,6 +521,8 @@ class Commit /** Total number of instructions committed. */ statistics::Vector instsCommitted; + /** Total number of pagefault.*/ + statistics::Vector pagefaulttimes; /** Total number of ops (including micro ops) committed. */ statistics::Vector opsCommitted; /** Stat for the total number of committed memory references. */ diff --git a/src/cpu/o3/dyn_inst.hh b/src/cpu/o3/dyn_inst.hh index 3dae156eb1..3b575df5b4 100644 --- a/src/cpu/o3/dyn_inst.hh +++ b/src/cpu/o3/dyn_inst.hh @@ -584,6 +584,7 @@ class DynInst : public ExecContext, public RefCounted bool isNop() const { return staticInst->isNop() || isEmptyMove; } bool isMemRef() const { return staticInst->isMemRef(); } bool isLoad() const { return staticInst->isLoad(); } + bool isHInst() const { return staticInst->isHInst(); } bool isStore() const { return staticInst->isStore(); } bool isAtomic() const { return staticInst->isAtomic(); } bool isStoreConditional() const @@ -616,6 +617,8 @@ class DynInst : public ExecContext, public RefCounted bool isReadBarrier() const { return staticInst->isReadBarrier(); } bool isWriteBarrier() const { return staticInst->isWriteBarrier(); } bool isNonSpeculative() const { return staticInst->isNonSpeculative(); } + bool isUpdateVsstatusSd() const {return staticInst->isUpdateVsstatusSd(); } + bool isUpdateMstatusSd() const {return staticInst->isUpdateMstatusSd(); } bool isQuiesce() const { return staticInst->isQuiesce(); } bool isUnverifiable() const { return staticInst->isUnverifiable(); } bool isSyscall() const { return staticInst->isSyscall(); } diff --git a/src/cpu/o3/lsq.cc b/src/cpu/o3/lsq.cc index d26f5903b2..7b716ca780 100644 --- a/src/cpu/o3/lsq.cc +++ b/src/cpu/o3/lsq.cc @@ -1188,6 +1188,7 @@ LSQ::LSQRequest::LSQRequest( _inst->isStoreConditional() || _inst->isAtomic() || _inst->isLoad()); flags.set(Flag::IsAtomic, _inst->isAtomic()); + flags.set(Flag::IsHInst, _inst->isHInst()); install(); } @@ -1274,6 +1275,9 @@ void LSQ::LSQRequest::sendFragmentToTranslation(int i) { numInTranslationFragments++; + if (_inst->isHInst()){ + req(i)->setHInst(_inst->isHInst()); + } _port.getMMUPtr()->translateTiming(req(i), _inst->thread->getTC(), this, isLoad() ? BaseMMU::Read : BaseMMU::Write); } diff --git a/src/cpu/o3/lsq.hh b/src/cpu/o3/lsq.hh index fc9a66c04f..1e9afcad02 100644 --- a/src/cpu/o3/lsq.hh +++ b/src/cpu/o3/lsq.hh @@ -223,7 +223,8 @@ class LSQ WritebackScheduled = 0x00001000, WritebackDone = 0x00002000, /** True if this is an atomic request */ - IsAtomic = 0x00004000 + IsAtomic = 0x00004000, + IsHInst = 0x00008000 }; FlagsType flags; @@ -285,6 +286,12 @@ class LSQ return flags.isSet(Flag::IsLoad); } + bool + isHInst() const + { + return flags.isSet(Flag::IsHInst); + } + bool isAtomic() const { diff --git a/src/cpu/o3/lsq_unit.cc b/src/cpu/o3/lsq_unit.cc index 290c5384ad..f9d212b4df 100644 --- a/src/cpu/o3/lsq_unit.cc +++ b/src/cpu/o3/lsq_unit.cc @@ -1819,10 +1819,12 @@ LSQUnit::read(LSQRequest *request, ssize_t load_idx) if (!load_inst->isVector() && request->mainReq()->getSize() > 1 && request->mainReq()->getVaddr() % request->mainReq()->getSize() != 0) { - DPRINTF(LSQUnit, "request: size: %u, Addr: %#lx, code: %d\n", request->mainReq()->getSize(), - request->mainReq()->getVaddr(), RiscvISA::ExceptionCode::LOAD_ADDR_MISALIGNED); + DPRINTF(LSQUnit, "request: size: %u, Addr: %#lx, code: %d\n", + request->mainReq()->getSize(), request->mainReq()->getVaddr(), + RiscvISA::ExceptionCode::LOAD_ADDR_MISALIGNED); return std::make_shared(request->mainReq()->getVaddr(), - RiscvISA::ExceptionCode::LOAD_ADDR_MISALIGNED); + request->mainReq()->getgPaddr(), + RiscvISA::ExceptionCode::LOAD_ADDR_MISALIGNED); } load_entry.setRequest(request); diff --git a/src/cpu/simple/timing.cc b/src/cpu/simple/timing.cc index ab0560f42a..5f1ed83069 100644 --- a/src/cpu/simple/timing.cc +++ b/src/cpu/simple/timing.cc @@ -1066,6 +1066,8 @@ TimingSimpleCPU::completeDataAccess(PacketPtr pkt) pkt->req->getPaddr()); fault = std::make_shared( pkt->req->getVaddr(), + 0 + , RiscvISA::ExceptionCode::LOAD_ADDR_MISALIGNED); } else { fault = curStaticInst->completeAcc(pkt, t_info, diff --git a/src/cpu/static_inst.hh b/src/cpu/static_inst.hh index 649569e0dd..50d74d212b 100644 --- a/src/cpu/static_inst.hh +++ b/src/cpu/static_inst.hh @@ -152,6 +152,7 @@ class StaticInst : public RefCounted, public StaticInstFlags { return flags[IsLoad] || flags[IsStore] || flags[IsAtomic]; } + bool isHInst() const { return flags[IsHInst];} bool isLoad() const { return flags[IsLoad]; } bool isStore() const { return flags[IsStore]; } bool isAtomic() const { return flags[IsAtomic]; } @@ -193,6 +194,8 @@ class StaticInst : public RefCounted, public StaticInstFlags bool isReadBarrier() const { return flags[IsReadBarrier]; } bool isWriteBarrier() const { return flags[IsWriteBarrier]; } bool isNonSpeculative() const { return flags[IsNonSpeculative]; } + bool isUpdateVsstatusSd() const{ return flags[IsUpdateVsstatusSd];} + bool isUpdateMstatusSd() const{return flags[IsUpdateMstatusSd];} bool isQuiesce() const { return flags[IsQuiesce]; } bool isUnverifiable() const { return flags[IsUnverifiable]; } bool isSyscall() const { return flags[IsSyscall]; } diff --git a/src/dev/riscv/NemuPlic.py b/src/dev/riscv/NemuPlic.py new file mode 100644 index 0000000000..5cb09ef6be --- /dev/null +++ b/src/dev/riscv/NemuPlic.py @@ -0,0 +1,11 @@ +from m5.params import * +from m5.proxy import * + +from m5.objects.Device import BasicPioDevice, PioDevice, IsaFake, BadAddr + +class NemuPlic(BasicPioDevice): + type = 'NemuPlic' + cxx_header = "dev/riscv/nemu_plic.hh" + cxx_class = 'gem5::NemuPlic' + pio_addr = 0x3c000000 + pio_size = Param.Addr(0x4000000, "NemuPlic space size") \ No newline at end of file diff --git a/src/dev/riscv/SConscript b/src/dev/riscv/SConscript index 2d58fc9669..15bf707400 100755 --- a/src/dev/riscv/SConscript +++ b/src/dev/riscv/SConscript @@ -40,12 +40,14 @@ SimObject('RTC.py', sim_objects=['RiscvRTC'], tags='riscv isa') SimObject('RiscvVirtIOMMIO.py', sim_objects=['RiscvMmioVirtIO'], tags='riscv isa') SimObject('NemuMMC.py', sim_objects=['NemuMMC'], tags='riscv isa') +SimObject('NemuPlic.py', sim_objects=['NemuPlic'], tags='riscv isa') DebugFlag('Clint', tags='riscv isa') DebugFlag('Lint', tags='riscv isa') DebugFlag('Plic', tags='riscv isa') DebugFlag('VirtIOMMIO', tags='riscv isa') DebugFlag('NemuMMC', tags='riscv isa') +DebugFlag('NemuPlic', tags='riscv isa') Source('pci_host.cc', tags='riscv isa') @@ -58,3 +60,4 @@ Source('plic.cc', tags='riscv isa') Source('rtc.cc', tags='riscv isa') Source('vio_mmio.cc', tags='riscv isa') Source('nemu_mmc.cc', tags='riscv isa') +Source('nemu_plic.cc', tags='riscv isa') diff --git a/src/dev/riscv/nemu_plic.cc b/src/dev/riscv/nemu_plic.cc new file mode 100644 index 0000000000..04b833d951 --- /dev/null +++ b/src/dev/riscv/nemu_plic.cc @@ -0,0 +1,31 @@ +// +//created by zhenh on 2024/04/19 +// + +#include "base/trace.hh" +#include "cpu/base.hh" +#include "mem/packet_access.hh" +#include "nemu_plic.hh" +#include "sim/system.hh" + +namespace gem5 +{ +NemuPlic::NemuPlic(const NemuPlicParams &p) : BasicPioDevice(p, p.pio_size) {} + +Tick +NemuPlic::read(PacketPtr pkt) +{ + pkt->setLE(0); + warn("now plic read always is 0\n"); + pkt->makeAtomicResponse(); + return pioDelay; +} + +Tick +NemuPlic::write(PacketPtr pkt) +{ + pkt->makeAtomicResponse(); + return pioDelay; +} + +} diff --git a/src/dev/riscv/nemu_plic.hh b/src/dev/riscv/nemu_plic.hh new file mode 100644 index 0000000000..c46f77fc19 --- /dev/null +++ b/src/dev/riscv/nemu_plic.hh @@ -0,0 +1,25 @@ +// +//created by zhenh on 2024/04/19 +// + +#ifndef GEM5_NEMUPLIC_HH +#define GEM5_NEMUPLIC_HH + +#include "dev/io_device.hh" +#include "params/NemuPlic.hh" + +namespace gem5 +{ + +class NemuPlic : public BasicPioDevice +{ + public: + typedef NemuPlicParams Params; + NemuPlic(const Params &p); + + Tick read(PacketPtr pkt) override; + Tick write(PacketPtr pkt) override; +}; + +} +#endif \ No newline at end of file diff --git a/src/mem/physical.cc b/src/mem/physical.cc index 148f235dbe..c5de0f36db 100644 --- a/src/mem/physical.cc +++ b/src/mem/physical.cc @@ -700,6 +700,10 @@ PhysicalMemory::overrideGCptRestorer(unsigned store_id) { uint8_t* pmem = backingStore[store_id].pmem; if (restoreFromXiangshanCpt && !gCptRestorerPath.empty()) { + if (gCptRestorerPath == "None") { + warn("gcpt restore is None\n"); + return; + } warn("Overriding Gcpt restorer\n"); warn("gCptRestorerPath: %s\n", gCptRestorerPath.c_str()); diff --git a/src/mem/request.hh b/src/mem/request.hh index 37d278e5e7..d5f4148e46 100644 --- a/src/mem/request.hh +++ b/src/mem/request.hh @@ -507,6 +507,7 @@ class Request /** The virtual address of the request. */ Addr _vaddr = MaxAddr; + Addr _gPaddr = MaxAddr; Addr _forward_pre_vaddr = MaxAddr; Addr _back_pre_vaddr = MaxAddr; @@ -544,6 +545,19 @@ class Request bool forward_pre_tlb = false; bool back_pre_tlb = false; + bool _twoStageTranslate = false; + int _virt = 0; + int _twoStageTranslateMode = 0; + bool _vsatp_0_mode = false; + bool _inGstage = true; + int _level = 2; + int _twoStageLevel = 2; + Addr _ppn = 0; + uint64_t _pte = 0; + bool _finishGVA = false; + + bool _isHInst = false; + public: /** @@ -936,6 +950,8 @@ class Request return _vaddr; } + Addr getgPaddr() const { return _gPaddr; } + Addr getForwardPreVaddr() const { return _forward_pre_vaddr; } bool get_forward_pre_tlb() const { return forward_pre_tlb; } @@ -945,15 +961,54 @@ class Request forward_pre_tlb = true; } + void setgPaddr(Addr gPaddr) { _gPaddr = gPaddr; } + Addr getBackPreVaddr() const { return _back_pre_vaddr; } bool get_back_pre_tlb() const { return back_pre_tlb; } + bool get_two_stage_state() const {return _twoStageTranslate;} + + int get_virt() const {return _virt;} + + int get_twoStageTranslateMode () const {return _twoStageTranslateMode;} + + bool get_vsatp_0_mode() const { return _vsatp_0_mode; } + + bool get_h_inst() const { return _isHInst; } + bool get_h_gstage() const { return _inGstage; } + int get_level() const { return _level; } + int get_two_stage_level() const { return _twoStageLevel; } + Addr get_ppn() const { return _ppn; } + uint64_t get_pte() const { return _pte; } + bool get_finish_gva() const { return _finishGVA; } + void setBackPreVaddr(Addr back_pre_vaddr) { _back_pre_vaddr = back_pre_vaddr; back_pre_tlb = true; } + void setPte(uint64_t pte) { _pte = pte; } + + void setTwoStageState(bool two_stage_translate,int virt,int two_stage_translate_mode) + { + _twoStageTranslate = two_stage_translate; + _virt = virt; + _twoStageTranslateMode = two_stage_translate_mode; + } + + void setTwoPtwWalk(bool inGstage, int level, int twoStageLevel, Addr ppn, bool finishGVA) + { + _inGstage = inGstage; + _level = level; + _twoStageLevel = twoStageLevel; + _ppn = ppn; + _finishGVA = finishGVA; + } + + void setVsatp0Mode(bool vstap_0_mode) { _vsatp_0_mode = vstap_0_mode; } + + void setHInst(bool is_h_inst) { _isHInst = is_h_inst; } /** Accesssor for the requestor id. */ RequestorID diff --git a/src/sim/faults.hh b/src/sim/faults.hh index ff9e0b9f88..20b7cc97f7 100644 --- a/src/sim/faults.hh +++ b/src/sim/faults.hh @@ -59,6 +59,7 @@ class FaultBase { public: virtual bool isFromISA() const { return true; }; + virtual uint64_t exception() const { return 0; } virtual FaultName name() const = 0; virtual void invoke(ThreadContext * tc, const StaticInstPtr &inst= nullStaticInstPtr);