Support for multiple dbus filters and artificial cache delay

src/main/scala/riscv/Core.scala

@@ -60,6 +60,9 @@ object createStaticPipeline {
         prefetcher,
         new Cache(sets = 2, ways = 2, backbone.filterIBus, Some(prefetcher), maxPrefetches = 2),
         new Cache(sets = 8, ways = 2, backbone.filterDBus, cacheable = (_ >= 0x80000000L)),
+        // Different cache levels can be specified by adding more cache plugins.
+        // The order in which you add the caches is the order in which they will be connected:
+        // new Cache(sets = 16, ways = 4, backbone.filterDBus, delay = 5),
         new CsrFile(pipeline.writeback, pipeline.writeback), // TODO: ugly
         new Timers,
         new MachineMode(pipeline.execute),

src/main/scala/riscv/plugins/Cache.scala

@@ -10,7 +10,8 @@ class Cache(
     busFilter: ((Stage, MemBus, MemBus) => Unit) => Unit,
     prefetcher: Option[PrefetchService] = None,
     maxPrefetches: Int = 1,
-    cacheable: (UInt => Bool) = (_ => True)
+    cacheable: (UInt => Bool) = (_ => True),
+    delay: Int = 1
 )(implicit config: Config)
     extends Plugin[Pipeline] {
   private val byteIndexBits = log2Up(config.xlen / 8)
@@ -104,9 +105,15 @@ class Cache(
       private val rspBuffer = Reg(MemBusRsp(internal.config))
       private val returningCache = Reg(Bool()).init(False)
 
+      // Delay cache response with a fixed delay
+      // Note that a minimal delay of 1 clock cycle is required to prevent
+      // combinatorial loops in case of multiple dbus filters.
+      private val internalRspBuffer = Stream(MemBusRsp(internal.config))
+      internal.rsp << internalRspBuffer.delay(delay)
+
       // initial state: not sending or acknowledging anything
-      internal.rsp.valid := False
-      internal.rsp.payload.assignDontCare()
+      internalRspBuffer.valid := False
+      internalRspBuffer.payload.assignDontCare()
       internal.cmd.ready := False
       external.cmd.valid := False
       external.cmd.payload.assignDontCare()
@@ -126,13 +133,13 @@ class Cache(
 
       private def forwardRspToInternal(): Unit = {
         sendingRsp := True
-        internal.rsp.valid := True
+        internalRspBuffer.valid := True
 
-        internal.rsp.rdata := external.rsp.rdata
+        internalRspBuffer.rdata := external.rsp.rdata
         // the index of 1's in internalIds indicate to which internal ids the response should be forwarded
         val internalId = OHToUInt(OHMasking.first(outstandingLoads(external.rsp.id).internalIds))
-        internal.rsp.id := internalId
-        when(internal.rsp.ready) {
+        internalRspBuffer.id := internalId
+        when(internalRspBuffer.ready) {
           // set the bit to 0 once it has been forwarded
           outstandingLoads(external.rsp.id).internalIds(internalId) := False
         }
@@ -188,7 +195,7 @@ class Cache(
           forwardRspToInternal()
           when(
             // when there is only one id left to forward, put result in cache and inform external bus we are done
-            internal.rsp.ready && CountOne(outstandingLoads(external.rsp.id).internalIds) === 1
+            internalRspBuffer.ready && CountOne(outstandingLoads(external.rsp.id).internalIds) === 1
           ) {
             insertRspInCache(address)
             alreadySendingRsp := False
@@ -206,10 +213,10 @@ class Cache(
           rspBuffer.id := internal.cmd.id
           rspBuffer.rdata := cacheLine.value
           when(!sendingRsp) {
-            internal.rsp.valid := True
-            internal.rsp.id := internal.cmd.id
-            internal.rsp.rdata := cacheLine.value
-            when(!internal.rsp.ready) {
+            internalRspBuffer.valid := True
+            internalRspBuffer.id := internal.cmd.id
+            internalRspBuffer.rdata := cacheLine.value
+            when(!internalRspBuffer.ready) {
               returningCache := True
             }
           } otherwise {
@@ -221,9 +228,9 @@ class Cache(
 
       when(returningCache && !sendingRsp) {
         // when not forwarding rsp but have a stored cache hit, return that
-        internal.rsp.valid := True
-        internal.rsp.payload := rspBuffer
-        when(internal.rsp.ready) {
+        internalRspBuffer.valid := True
+        internalRspBuffer.payload := rspBuffer
+        when(internalRspBuffer.ready) {
           returningCache := False
         }
       }

src/main/scala/riscv/plugins/memory/DynamicMemoryBackbone.scala

@@ -9,6 +9,7 @@ import scala.collection.mutable
 class DynamicMemoryBackbone(implicit config: Config) extends MemoryBackbone with Resettable {
 
   private var activeFlush: Bool = null
+  private var unifiedInternalDBus: Stream[MemBus] = null
 
   override def build(): Unit = {
     pipeline plug new Area {
@@ -22,9 +23,7 @@ class DynamicMemoryBackbone(implicit config: Config) extends MemoryBackbone with
     super.finish()
 
     pipeline plug new Area {
-      externalDBus = master(new MemBus(config.dbusConfig)).setName("dbus")
-
-      private val unifiedInternalDBus = Stream(MemBus(config.dbusConfig))
+      unifiedInternalDBus = Stream(MemBus(config.dbusConfig))
 
       unifiedInternalDBus.cmd.valid := False
       unifiedInternalDBus.cmd.address.assignDontCare()
@@ -193,10 +192,9 @@ class DynamicMemoryBackbone(implicit config: Config) extends MemoryBackbone with
             }
           }
       }
-
-      dbusFilter.foreach(_(internalWriteDBusStage, unifiedInternalDBus, externalDBus))
-      dbusObservers.foreach(_(internalWriteDBusStage, unifiedInternalDBus))
     }
+
+    setupExternalDBus(unifiedInternalDBus)
   }
 
   override def createInternalDBus(

src/main/scala/riscv/plugins/memory/MemoryBackbone.scala

@@ -15,7 +15,7 @@ abstract class MemoryBackbone(implicit config: Config) extends Plugin with Memor
   var internalWriteDBus: MemBus = null
   var internalReadDBusStages: Seq[Stage] = null
   var internalWriteDBusStage: Stage = null
-  var dbusFilter: Option[MemBusFilter] = None
+  var dbusFilters = mutable.ArrayBuffer[MemBusFilter]()
   var ibusFilter: Option[MemBusFilter] = None
   val dbusObservers = mutable.ArrayBuffer[MemBusObserver]()
 
@@ -43,17 +43,36 @@ abstract class MemoryBackbone(implicit config: Config) extends Plugin with Memor
     }
   }
 
-  override def finish(): Unit = {
-    setupIBus()
+  def setupExternalDBus(internalDBus: MemBus): Unit = {
+    pipeline plug new Area {
+      externalDBus = master(new MemBus(config.dbusConfig)).setName("dbus")
+
+      if (dbusFilters.nonEmpty) {
+        var previous_level = internalDBus
 
-    // DBUS
-    if (dbusFilter.isEmpty) {
-      dbusFilter = Some((_, idbus, edbus) => {
-        idbus <> edbus
-      })
+        dbusFilters.zipWithIndex.foreach { case (f, i) =>
+          if (i < dbusFilters.size - 1) {
+            val intermediateDBus =
+              Stream(MemBus(config.dbusConfig)).setName("intermediate_dbus" + i)
+            f(internalWriteDBusStage, previous_level, intermediateDBus)
+
+            previous_level = intermediateDBus
+          } else {
+            f(internalWriteDBusStage, previous_level, externalDBus)
+          }
+        }
+      } else {
+        internalDBus <> externalDBus
+      }
+
+      dbusObservers.foreach(_(internalWriteDBusStage, internalDBus))
     }
   }
 
+  override def finish(): Unit = {
+    setupIBus()
+  }
+
   override def getExternalIBus: MemBus = {
     assert(externalIBus != null)
     externalIBus
@@ -80,8 +99,7 @@ abstract class MemoryBackbone(implicit config: Config) extends Plugin with Memor
   }
 
   override def filterDBus(filter: MemBusFilter): Unit = {
-    assert(dbusFilter.isEmpty)
-    dbusFilter = Some(filter)
+    dbusFilters += filter
   }
 
   override def filterIBus(filter: MemBusFilter): Unit = {

src/main/scala/riscv/plugins/memory/StaticMemoryBackbone.scala

@@ -11,11 +11,7 @@ class StaticMemoryBackbone(implicit config: Config) extends MemoryBackbone {
   override def finish(): Unit = {
     super.finish()
 
-    pipeline plug new Area {
-      externalDBus = master(new MemBus(config.dbusConfig)).setName("dbus")
-      dbusFilter.foreach(_(internalWriteDBusStage, internalWriteDBus, externalDBus))
-      dbusObservers.foreach(_(internalWriteDBusStage, internalWriteDBus))
-    }
+    setupExternalDBus(internalWriteDBus)
   }
 
   override def createInternalDBus(