phy/lpddr5/sim: add initial data commands handling

litedram/phy/lpddr5/sim.py

@@ -4,26 +4,35 @@
 # Copyright (c) 2021 Antmicro <www.antmicro.com>
 # SPDX-License-Identifier: BSD-2-Clause
 
-# import math
+from operator import or_
-# from operator import or_
+from functools import reduce
-# from functools import reduce
 from collections import OrderedDict
 
 from migen import *
 
-# from litex.soc.interconnect.stream import ClockDomainCrossing
+from litex.soc.interconnect import stream
 from litex.soc.interconnect.csr import AutoCSR
 #
 from litedram.common import TappedDelayLine
-# from litedram.phy.utils import delayed, edge
+from litedram.phy.utils import edge
 from litedram.phy.sim_utils import SimLogger, PulseTiming, log_level_getter
-# from litedram.phy.lpddr4.commands import MPC
 
 
+CMD_INFO_LAYOUT = [
+    ("we", 1),
+    ("masked", 1),
+    ("burst32", 1),
+    ("bank", 4),
+    ("row", 18),
+    ("col", 6),
+]
+
+gtkw_dbg = {}
+
 class LPDDR5Sim(Module, AutoCSR):
     """LPDDR5 DRAM simulation
     """
-    def __init__(self, pads, *, ck_freq, log_level):
+    def __init__(self, pads, *, ck_freq, wck_freq, log_level):
         log_level = log_level_getter(log_level)
 
         self.clock_domains.cd_ck = ClockDomain(reset_less=True)
@@ -33,9 +42,23 @@ class LPDDR5Sim(Module, AutoCSR):
             self.cd_ck_n.clk.eq(~pads.ck),
         ]
 
-        cmd = CommandsSim(pads, ck_freq=ck_freq, log_level=log_level("cmd"))
+        self.clock_domains.cd_wck = ClockDomain(reset_less=True)
+        self.clock_domains.cd_wck_n = ClockDomain(reset_less=True)
+        self.comb += [
+            self.cd_wck.clk.eq(pads.wck),
+            self.cd_wck_n.clk.eq(~pads.wck),
+        ]
+
+        # CommandsSim and DataSim communicate via this endpoint
+        cmd_info = stream.Endpoint(CMD_INFO_LAYOUT)
+        gtkw_dbg["cmd_info"] = cmd_info
+
+        cmd = CommandsSim(pads, cmd_info, ck_freq=ck_freq, log_level=log_level("cmd"))
         self.submodules.cmd = ClockDomainsRenamer("ck")(cmd)
 
+        data = DataSim(pads, cmd_info, cmd.data_timer.ready_p, wck_freq=wck_freq, log_level=log_level("data"))
+        self.submodules.data = ClockDomainsRenamer("wck")(data)
+
 
 def nested_case(mapping, *, on_leaf, variables, default=None, **kwargs):
     """Generate a nested Case from a mapping
@@ -114,7 +137,8 @@ class ModeRegisters(Module, AutoCSR):
         wl = (1, (7, 4)),
         rl = (2, (3, 0)),
         set_ab = (3, (5, 5)),
-        ckr = (18, (7, 7))
+        bank_org = (3, (4, 3)),
+        ckr = (18, (7, 7)),
     )
 
     def __init__(self, *, ck_freq, log_level):
@@ -126,7 +150,7 @@ class ModeRegisters(Module, AutoCSR):
             for addr in range(64)
         ])
 
-        fields = {}
+        self.fields = fields = {}
         for name, (addr, (bit_hi, bit_lo)) in self.FIELD_DEFS.items():
             fields[name] = Signal(bit_hi - bit_lo + 1)
             self.comb += fields[name].eq(self.mr[addr][bit_lo:bit_hi+1])
@@ -185,11 +209,21 @@ class ModeRegisters(Module, AutoCSR):
             ],
         )
 
+class Sync(list):
+    # Helper for combining comb and sync
+    def __init__(self, arg):
+        if not isinstance(arg, list):
+            arg = [arg]
+        super().__init__(arg)
+
+
 class CommandsSim(Module, AutoCSR):
-    def __init__(self, pads, *, ck_freq, log_level):
+    def __init__(self, pads, cmd_info, *, ck_freq, log_level):
         self.submodules.log = log = SimLogger(log_level=log_level, clk_freq=ck_freq)
         self.log.add_csrs()
+        self.comb += self.log.info("Simulation start")
 
+        self.cmd_info = cmd_info
         self.submodules.mode_regs = ModeRegisters(log_level=log_level, ck_freq=ck_freq)
 
         self.nbanks = 16
@@ -221,12 +255,26 @@ class CommandsSim(Module, AutoCSR):
 
         self.handle_cmd = Signal()
 
+        self.data_latency = Signal(max(len(self.mode_regs.wl), len(self.mode_regs.rl)))
+        data_latency = Signal.like(self.data_latency)
+        data_latency_reg = Signal.like(self.data_latency)
+        self.submodules.data_timer = PulseTiming(data_latency)
+        self.sync += If(self.data_timer.trigger,
+            data_latency_reg.eq(self.data_latency)
+        )
+        self.comb += If(self.data_timer.trigger,
+            data_latency.eq(self.data_latency),
+        ).Else(
+            data_latency.eq(data_latency_reg),
+        ),
+
         cmds_enabled = Signal()
         cmd_handlers = OrderedDict(
             ACT = self.activate_handler(),
             PRE = self.precharge_handler(),
             REF = self.refresh_handler(),
             MRW = self.mrw_handler(),
+            DATA = self.data_handler(),
             # WRITE/MASKED-WRITE
             # READ
             # CAS
@@ -251,7 +299,6 @@ class CommandsSim(Module, AutoCSR):
         row3 = Signal(4)
         row4 = Signal(7)
         row = Signal(18)
-        t_aad = PulseTiming(8 - 1)
         return self.cmd_two_step("ACTIVATE",
             cond1 = self.ca_p[:3] == 0b111,
             body1 = [
@@ -279,7 +326,7 @@ class CommandsSim(Module, AutoCSR):
         all_banks = Signal()
         return self.cmd_one_step("PRECHARGE",
             cond = self.ca_p[:7] == 0b1111000,
-            comb = [
+            body = [
                 all_banks.eq(self.ca_n[6]),
                 If(all_banks,
                     self.log.info("PRE: all banks"),
@@ -288,8 +335,7 @@ class CommandsSim(Module, AutoCSR):
                     self.log.info("PRE: bank = %d", bank),
                     bank.eq(self.ca_n[:4]),
                 ),
-            ],
+                Sync(
-            sync = [
                     If(all_banks,
                         *[self.active_banks[b].eq(0) for b in range(2**len(bank))]
                     ).Else(
@@ -298,7 +344,8 @@ class CommandsSim(Module, AutoCSR):
                             self.log.warn("PRE on inactive bank: bank=%d", bank)
                         ),
                     ),
-            ]
+                )
+            ],
         )
 
     def refresh_handler(self):
@@ -307,7 +354,7 @@ class CommandsSim(Module, AutoCSR):
         all_banks = Signal()
         return self.cmd_one_step("REFRESH",
             cond = self.ca_p[:7] == 0b0111000,
-            comb = [
+            body = [
                 all_banks.eq(self.ca_n[6]),
                 If(reduce(or_, self.active_banks),
                     self.log.error("Not all banks precharged during REFRESH")
@@ -331,14 +378,81 @@ class CommandsSim(Module, AutoCSR):
             ]
         )
 
-    def cmd_one_step(self, name, cond, comb, sync=None):
+    def data_handler(self):
+        data_cmds = {
+            "MASKED-WRITE": self.ca_p[:3] == 0b010,
+            "WRITE":        self.ca_p[:3] == 0b110,
+            "WRITE32":      self.ca_p[:4] == 0b0100,
+            "READ":         self.ca_p[:3] == 0b001,
+            "READ32":       self.ca_p[:3] == 0b101,
+        }
+
+        bank           = Signal(max=self.nbanks)
+        row            = Signal(18)
+        col            = Signal(6)
+        auto_precharge = Signal()
+
+        return self.cmd_one_step("DATA",
+            cond = reduce(or_, data_cmds.values()),
+            body = [
+                bank.eq(self.ca_n[:4]),
+                row.eq(self.active_rows[bank]),
+                col.eq(Cat(self.ca_p[3], self.ca_n[4:6], self.ca_p[4:7])),
+                auto_precharge.eq(self.ca_n[6]),
+                # push to DataSim
+                self.cmd_info.we.eq(data_cmds["MASKED-WRITE"] | data_cmds["WRITE"] | data_cmds["WRITE32"]),
+                self.cmd_info.masked.eq(data_cmds["MASKED-WRITE"]),
+                self.cmd_info.burst32.eq(data_cmds["WRITE32"] | data_cmds["READ32"]),
+                self.cmd_info.bank.eq(bank),
+                self.cmd_info.row.eq(row),
+                self.cmd_info.col.eq(col),
+                self.cmd_info.valid.eq(1),
+                If(~self.cmd_info.ready,
+                    self.log.error("Simulator CMD-to-DATA overflow")
+                ),
+                # data latency
+                If(self.cmd_info.we,
+                    self.data_latency.eq(self.mode_regs.wl - 2),
+                    If(self.mode_regs.wl < 2,
+                        self.log.error("WL < 2 is currently not supported")
+                    ),
+                ).Else(
+                    self.data_latency.eq(self.mode_regs.rl - 2),
+                    If(self.mode_regs.rl < 2,
+                        self.log.error("RL < 2 is currently not supported")
+                    ),
+                ),
+                self.data_timer.trigger.eq(1),
+                # command info
+                *[If(cond, self.log.info(f"{name}: bank=%d row=%d col=%d", bank, row, col))
+                    for name, cond in data_cmds.items()],
+                # auto precharge
+                If(auto_precharge,
+                    self.log.info("AUTO-PRECHARGE: bank=%d row=%d", bank, row),
+                ),
+                Sync(If(auto_precharge,
+                    self.active_banks[bank].eq(0),
+                )),
+                # sanity checks
+                If(~self.active_banks[bank],
+                    self.log.error("CAS command on inactive bank: bank=%d row=%d col=%d", bank, row, col)
+                ),
+                If(self.cmd_info.masked & ~((self.mode_regs.fields["bank_org"] == 0b00) | (self.mode_regs.fields["bank_org"] == 0b10)),
+                    self.log.error("READ32/WRITE32 are valid in BG/16B mode only")
+                ),
+            ],
+        )
+
+    def cmd_one_step(self, name, cond, body):
         matched = Signal()
+        comb = list(filter(lambda i: not isinstance(i, Sync), body))
+        sync = list(filter(lambda i: isinstance(i, Sync), body))
         self.comb += If(self.handle_cmd & cond,
             self.log.debug(name),
             matched.eq(1),
             *comb
         )
-        if sync is not None:
+        if len(sync) > 0:
             self.sync += If(self.handle_cmd & cond,
                 *sync
             )
@@ -380,3 +494,136 @@ class CommandsSim(Module, AutoCSR):
         self.submodules += fsm
 
         return matched
+
+
+class DataSim(Module, AutoCSR):
+    def __init__(self, pads, cmd_info, latency_ready, *, wck_freq, log_level, nrows=32768, ncols=1024, nbanks=16):
+        self.submodules.log = log = SimLogger(log_level=log_level, clk_freq=wck_freq)
+        self.log.add_csrs()
+
+        # CommandsSim produces the data required for handling a data command via cmd_info endpoint.
+        # Using stream.ClockDomainCrossing introduces too much latency, so we do a simplistic CDC
+        # and store the information in a FIFO, so that it is possible to pipeline data commands.
+        self.submodules.cmds = stream.SyncFIFO(CMD_INFO_LAYOUT, depth=4)
+        gtkw_dbg["cmds"] = self.cmds
+        self.comb += [
+            cmd_info.connect(self.cmds.sink, omit={"ready", "valid"}),
+            # ~ready will signalize that somehow our FIFO is full, which is an internal error
+            cmd_info.ready.eq(cmd_info.valid & self.cmds.sink.ready),
+            # to latch a command only once we use an edge here, which we can do as there is no way
+            # for 2 valid commands cycle-by-cycle
+            self.cmds.sink.valid.eq(edge(self, cmd_info.valid)),
+        ]
+
+        wr_start = Signal()
+        rd_start = Signal()
+        self.comb += [
+            wr_start.eq(self.cmds.source.valid & self.cmds.source.we & latency_ready),
+            rd_start.eq(self.cmds.source.valid & ~self.cmds.source.we & latency_ready),
+        ]
+
+        # After the WL signal arives we require the data to arrive some time later and then we start
+        # reading it. This would be adjustable on hardware, but in simulation we rather must set this
+        # so that it matches the delay that PHY introduces.
+        t_wckdqi = 2 - 1 -1
+
+        wr_start_d = wr_start
+        for _ in range(t_wckdqi):
+            _wr_start_d = Signal()
+            self.sync += _wr_start_d.eq(wr_start_d)
+            wr_start_d = _wr_start_d
+
+        current_cmd = stream.Endpoint(CMD_INFO_LAYOUT)
+        gtkw_dbg["current_cmd"] = current_cmd
+        cmd_buf = stream.PipeValid(CMD_INFO_LAYOUT)
+        gtkw_dbg["cmd_buf"] = cmd_buf
+        self.submodules += cmd_buf
+        self.comb += [
+            self.cmds.source.connect(cmd_buf.sink),
+            cmd_buf.source.connect(current_cmd),
+        ]
+
+        burst_counter = Signal(max=32)
+        burst_length = Signal.like(burst_counter)
+        self.comb += [
+            If(current_cmd.burst32,
+                burst_length.eq(32 - 1)
+            ).Else(
+                burst_length.eq(16 - 1)
+            )
+        ]
+
+        class BurstWriter(Module):
+            def __init__(self, ports, burst_start):
+                self.enable = Signal()
+
+                self.submodules.log = log = SimLogger(log_level=log_level, clk_freq=wck_freq)
+                self.log.add_csrs()
+
+                mem_addr = Signal(max=nrows * ncols)
+                current_col = Signal(max=ncols)
+                burst_beat = Signal.like(current_col, reset=burst_start)
+
+                self.sync += If(self.enable,
+                    burst_beat.eq(burst_beat + 2)
+                ).Else(
+                    burst_beat.eq(burst_start)
+                )
+                self.comb += [
+                    If(self.enable,
+                        current_col.eq(current_cmd.col + burst_beat),
+                        mem_addr.eq(current_cmd.row * ncols + current_col),
+                        ports[current_cmd.bank].we.eq(2**len(ports[current_cmd.bank].we) - 1),
+                        ports[current_cmd.bank].adr.eq(mem_addr),
+                        ports[current_cmd.bank].dat_w.eq(pads.dq),
+                        self.log.debug("WRITE[%d]: bank=%d, row=%d, col=%d, dq=0x%04x dm=0x%02b",
+                            burst_beat, current_cmd.bank, current_cmd.row, current_col, pads.dq, pads.dmi,
+                            once=False
+                        ),
+                    ),
+                ]
+
+        # DRAM Memory storage
+        mems = [Memory(len(pads.dq), depth=nrows * ncols) for _ in range(nbanks)]
+        ports_p = [mem.get_port(write_capable=True, we_granularity=8, async_read=True, clock_domain="wck") for mem in mems]
+        ports_n = [mem.get_port(write_capable=True, we_granularity=8, async_read=True, clock_domain="wck_n") for mem in mems]
+        self.specials += mems + ports_p + ports_n
+        ports_p = Array(ports_p)
+        ports_n = Array(ports_n)
+
+        self.submodules.write_p = ClockDomainsRenamer("wck")(BurstWriter(ports_p, 0))
+        self.submodules.write_n = ClockDomainsRenamer("wck_n")(BurstWriter(ports_n, 1))
+        write_enable = Signal()
+        self.sync.wck_n += If(write_enable,
+            self.write_p.enable.eq(1),
+            self.write_n.enable.eq(1),
+        ).Else(
+            self.write_p.enable.eq(0),
+            self.write_n.enable.eq(0),
+        )
+
+        self.submodules.fsm = fsm = FSM()
+        fsm.act("IDLE",
+            If(wr_start_d,
+                current_cmd.ready.eq(1),
+                NextValue(burst_counter, 0),
+                NextState("WRITE-BURST"),
+            )
+        )
+        fsm.act("WRITE-BURST",
+            write_enable.eq(1),
+            If(burst_counter == burst_length[1:],
+                # TODO: continuous bursts
+                # If(wr_start, NextValue(burst_counter, current_cmd.burst32)),
+                NextValue(burst_counter, 0),
+                NextState("IDLE")
+            ).Else(
+                NextValue(burst_counter, burst_counter + 1),
+            ),
+        )
+        fsm.act("READ-BURST",
+        )
+
+
+
+        pass

litedram/phy/lpddr5/simsoc.py

@@ -140,6 +140,7 @@ class SimSoC(SoCCore):
         self.submodules.lpddr5sim = LPDDR5Sim(
             pads          = self.ddrphy.pads,
             ck_freq       = sys_clk_freq,
+            wck_freq      = wck_ck_ratio*sys_clk_freq,
             log_level     = log_level,
         )
 
@@ -264,6 +265,7 @@ def generate_gtkw_savefile(builder, vns, trace_fst):
         # dram pads
         save.group([s for s in vars(soc.ddrphy.pads).values() if isinstance(s, Signal)],
             group_name = "pads",
+            closed = False,
             mappers = [
                 gtkw.regex_filter(["_[io]$"], negate=True),
                 gtkw.regex_sorter(gtkw.suffixes2re(["reset_n", "ck", "cs", "ca", "dq", "wck", "dmi", "rdqs"])),
@@ -275,6 +277,13 @@ def generate_gtkw_savefile(builder, vns, trace_fst):
             ],
         )
 
+        from litedram.phy.lpddr5.sim import gtkw_dbg
+        for name in "cmd_info cmds cmd_buf current_cmd".split():
+            save.add(gtkw_dbg[name], group_name=name, closed=False,
+                # mappers=[gtkw.endpoint_filter(payload=False)],
+                mappers=[gtkw.endpoint_filter()],
+            )
+
 def main():
     parser = argparse.ArgumentParser(description="Generic LiteX SoC Simulation")
     builder_args(parser.add_argument_group(title="Builder"))