Merge pull request #95 from lschuermann/dev/compliant-xgmii

phy/xgmii: handle IFG insertion in PHY, support deficit idle count

liteeth/mac/core.py

@@ -125,7 +125,8 @@ class LiteEthMACCore(Module, AutoCSR):
             if core_dw != 8:
                 tx_datapath.add_last_be()
         # Gap insertion has to occurr in phy tx domain to ensure gap is correctly maintained
-        tx_datapath.add_gap()
+        if not getattr(phy, "integrated_ifg_inserter", False):
+            tx_datapath.add_gap()
         tx_datapath.pipeline.append(phy)
         self.submodules.tx_datapath = tx_datapath
 

liteeth/phy/xgmii.py

@@ -17,66 +17,382 @@ XGMII_START = Constant(0xFB, bits_sign=8)
 XGMII_END = Constant(0xFD, bits_sign=8)
 
 class LiteEthPHYXGMIITX(Module):
-    def __init__(self, pads, dw):
+    def __init__(self, pads, dw, dic=True):
         # Enforce 64-bit data path
         assert dw == 64
 
         # Sink for data to transmit
         self.sink = sink = stream.Endpoint(eth_phy_description(dw))
 
+        # ---------- Generic signals ----------
+
+        # Masked last_be signal of current clock cycle. last_be should only be
+        # respected when last is also asserted.
+        masked_last_be = Signal.like(sink.last_be)
+        self.comb += [
+            If(sink.last,
+                masked_last_be.eq(sink.last_be),
+            ),
+        ]
+
+        # ---------- Inter-frame gap state ----------
+
+        # State to keep track of the current inter-frame gap we are required to
+        # maintain. We must take care to always have an inter-frame gap of at
+        # least 96 bits (12 bytes), with an exception for the deficit idle gap
+        # mechanism. Because XGMII transactions can only start on the first or
+        # fifth byte in a 64-bit bus word, it's sufficient to represent this as
+        # - 0: less than 4 bytes of IFG transmitted
+        # - 1: less than 8 bytes of IFG transmitted
+        # - 2: less than 12 bytes of IFG transmitted
+        # - 3: 12 or more bytes of IFG transmitted
+        current_ifg = Signal(max=4, reset=3)
+        next_ifg = Signal(max=4)
+
+        # Shortcut "functions" to add a 32-bit or 64-bit idle bus word to the
+        # current inter-frame gap without worring about wrapping, or to reset it
+        # (typically on the start of a new transmission). This can be useful to
+        # see the effect on the next_ifg value and thus make decisions about
+        # subsequent signal states (e.g. sink.ready).
+        ifg_reset = Signal()
+        ifg_add_double = Signal()
+        ifg_add_single = Signal()
+        self.comb += [
+            If(ifg_reset,
+                next_ifg.eq(0),
+            ).Elif(ifg_add_single,
+                If(current_ifg < 3,
+                    next_ifg.eq(current_ifg + 1)
+                ),
+            ).Elif(ifg_add_double,
+                If(current_ifg < 2,
+                    next_ifg.eq(current_ifg + 2),
+                ).Else(
+                    next_ifg.eq(3),
+                ),
+            ).Else(
+                next_ifg.eq(current_ifg),
+            ),
+        ]
+
+        self.sync += current_ifg.eq(next_ifg)
+
+        # ---------- Deficit idle count mechanism state ----------
+
+        # Because XGMII only allows start of frame characters to be placed on
+        # lane 0 (first and fifth octet in a 64-bit bus word), when a packet's
+        # length % 4 != 0, we can't transmit exactly 12 XGMII idle characters
+        # inter-frame gap (the XGMII end of frame character counts towards the
+        # inter-frame gap, while start of frame does not). Given we are required
+        # to transmit a minimum of 12 bytes IFG, it's allowed to send packet
+        # length % 4 bytes additional IFG bytes. However this would waste
+        # precious bandwidth transmitting these characters.
+        #
+        # Thus, 10Gbit/s Ethernet and above allow using the deficit idle count
+        # mechanism. It allows to delete some idle characters, as long as an
+        # average count of >= 12 bytes IFG is maintained. This is to be
+        # implemented as a two bit counter as specified in IEEE802.3-2018,
+        # section four, 46.3.1.4 Start control character alignment.
+        #
+        # This module implements the deficit idle count algorithm as described
+        # by Eric Lynskey of the UNH InterOperability Lab[1]:
+        #
+        # | current |             |             |             |             |
+        # | count   |           0 |           1 |           2 |           3 |
+        # |---------+-----+-------+-----+-------+-----+-------+-----+-------|
+        # |         |     | new   |     | new   |     | new   |     | new   |
+        # | pkt % 4 | IFG | count | IFG | count | IFG | count | IFG | count |
+        # |---------+-----+-------+-----+-------+-----+-------+-----+-------|
+        # |       0 |  12 |     0 |  12 |     1 |  12 |     2 |  12 |     3 |
+        # |       1 |  11 |     1 |  11 |     2 |  11 |     3 |  15 |     0 |
+        # |       2 |  10 |     2 |  10 |     3 |  14 |     0 |  14 |     1 |
+        # |       3 |   9 |     3 |  13 |     0 |  13 |     1 |  13 |     2 |
+        #
+        # [1]: https://www.iol.unh.edu/sites/default/files/knowledgebase/10gec/10GbE_DIC.pdf
+
+
+        # Additional state to keep track of exactly how many bytes % 4 we've
+        # transmitted in the last packet. We need this information to judge
+        # whether we've had a sufficiently large IFG given the current DIC
+        # count. Value the range of [0; 3].
+        #
+        # If we disable the deficit idle count, we replace this with a constant
+        # of 0, meaning that we pretend to not have transmitted any additional
+        # IDLE characters. This should allow significant logic optimizations
+        # while having the same effect as not implementing DIC at all.
+        if dic:
+            last_packet_rem = Signal(max=4)
+        else:
+            last_packet_rem = Constant(0, bits_sign=2)
+
+        # Bounded counter of deleted XGMII idle characters. Must be within [0;
+        # 3]. If we disable the deficit idle count mechanism, this signal should
+        # not change. However, it's still present to avoid the logic below
+        # getting too complex.
+        current_dic = Signal(max=4, reset=3)
+
+
+        # ---------- Shifted transmit state ----------
+
+        # Whether the current transmission is shifted, meaning that the packet's
+        # transmission started on the fifth octect within the 64-bit bus
+        # word. As a consequence of the shifted transmission, given that we
+        # receive 64 valid bits from the sink, we need to store and delay the
+        # upper half of the current clock cycle's data to the next.
+        #
+        # This register is to be set when transitioning out of the IDLE
+        # state.
+        transmit_shifted = Signal()
+
+        # Upper half of the data of the previous clock cycle.
+        prev_valid_data = Signal(dw)
+        prev_valid_last_be = Signal(dw // 8)
+        self.sync += [
+            If(sink.valid & sink.ready,
+               prev_valid_data.eq(sink.data),
+               If(sink.last,
+                  prev_valid_last_be.eq(masked_last_be)
+               ).Else(
+                   prev_valid_last_be.eq(0),
+               ),
+            ),
+        ]
+
+        # Previous clock cycle sink valid signal
+        prev_valid = Signal()
+        self.sync += prev_valid.eq(sink.valid)
+
+        # Adjusted sink data & last_be. If our transmission is shifted, this
+        # will contain the upper-half of the previous and lower-half of the
+        # current clock cycle. Otherwise, simply equal to data and the masked
+        # last_be.
+        adjusted_sink_valid = Signal()
+        adjusted_sink_valid_data = Signal.like(sink.data)
+        adjusted_sink_valid_last_be = Signal.like(sink.last_be)
+        self.comb += [
+            If(transmit_shifted,
+                # Because we are injecting data from the previous cycle, we need
+                # to respect it's valid. It's fine that adjusted_sink_valid
+                # therefore is deasserted for the very first bus word, given
+                # this is handled in the IDLE fsm state still. This assumes a
+                # non-hostile sink where valid is constantly asserted during a
+                # single transmission.
+                adjusted_sink_valid.eq(prev_valid),
+                adjusted_sink_valid_data.eq(Cat(
+                    prev_valid_data[(dw // 2):],
+                    sink.data[:(dw // 2)],
+                )),
+                adjusted_sink_valid_last_be.eq(Cat(
+                    prev_valid_last_be[(dw // 8 // 2):],
+                    masked_last_be[:(dw // 8 // 2)],
+                )),
+            ).Else(
+                adjusted_sink_valid.eq(sink.valid),
+                adjusted_sink_valid_data.eq(sink.data),
+                adjusted_sink_valid_last_be.eq(masked_last_be),
+            ),
+        ]
+
+        # ---------- XGMII transmission logic ----------
+
         # Transmit FSM
         self.submodules.fsm = fsm = FSM(reset_state="IDLE")
 
+        # This block will be executed by the FSM below in the IDLE state, when
+        # it's time to start a transmission aligned on the FIRST byte in a
+        # 64-bit bus word. This can happen both because we've waited the 12 byte
+        # IFG and coincidentally the first byte is the next valid start point,
+        # or because we reduced the IFG to 8 bytes because of the deficit idle
+        # count mechanism. Thus have it as a reusable component here.
+        unshifted_idle_transmit = [
+            # Currently idling, but a new frame is ready for transmission
+            # and we had at least the full IFG idle before. Thus transmit
+            # the preamble, but replace the first byte with the XGMII start
+            # of frame control character. Accept more data.
+            ifg_reset.eq(1),
+            pads.tx_ctl.eq(0x01),
+            pads.tx_data.eq(Cat(XGMII_START, sink.data[8:dw])),
+            NextValue(transmit_shifted, 0),
+            NextValue(sink.ready, 1),
+            NextState("TRANSMIT"),
+        ]
+
+        # This block will be executed by the FSM below in the IDLE state, when
+        # it's time to start a transmission aligned on the FIFTH byte in a
+        # 64-bit bus word. This can happen either because we've waited the 8
+        # byte IFG and need to insert only four bytes more in this cycle, or
+        # because the deficit idle count mechanism allows transmit with a
+        # smaller IFG (e.g. 1 bits packet remainder + 4 bytes TRANSMIT IFG in
+        # previous cycle + 4 bytes IDLE ID in current cycle = 9 bytes total
+        # IFG).
+        shifted_idle_transmit = [
+            # Currently idling, but a new frame is ready for transmission and
+            # there is only 4 bytes missing in the IFG (or we have created an
+            # acceptable IFG deficit). Thus transmit the preamble on the second
+            # 32-bit bus word, but replace the first byte with the XGMII start
+            # of frame control character. Accept more data.
+            pads.tx_ctl.eq(0x1F),
+            pads.tx_data.eq(Cat(
+                Replicate(XGMII_IDLE, 4),
+                XGMII_START,
+                sink.data[8:(dw // 2)],
+            )),
+            ifg_reset.eq(1),
+            NextValue(transmit_shifted, 1),
+            NextValue(sink.ready, 1),
+            NextState("TRANSMIT"),
+        ]
+
         fsm.act("IDLE",
-            If(sink.valid,
+            If(sink.valid & (current_ifg == 3),
-                # Currently idling, but a new frame is ready for
+               # Branch A: we've transmitted at least the full 12 bytes
-                # transmission. Thus transmit the preamble, but replace the
+               # IFG. This means that we can unconditionally start transmission
-                # first byte with the XGMII start of frame control
+               # on the first octet. In addition to that, we may have inserted
-                # character. Accept more data.
+               # some extra XGMII, thus we can reduce the deficit.
-                pads.tx_ctl.eq(0x01),
+               *unshifted_idle_transmit,
-                pads.tx_data.eq(Cat(XGMII_START, sink.data[8:dw])),
+               If(current_dic - last_packet_rem < 0,
-                NextValue(sink.ready, 1),
+                   NextValue(current_dic, 0),
-                NextState("TRANSMIT"),
+               ).Else(
+                   NextValue(current_dic, current_dic - last_packet_rem),
+               )
+            ).Elif(sink.valid & (current_ifg == 2),
+                # Branch B: we've transmitted at least 8 bytes of IFG. This
+                # means that we can either, depending on the DIC start
+                # transmission on the first or fith octect. Manipulate the DIC
+                # count accordingly.
+                If((last_packet_rem != 0)
+                   & (current_dic + last_packet_rem <= 3),
+                    # We've taken some extra IFG bytes (added to the deficit)
+                    *unshifted_idle_transmit,
+                    NextValue(current_dic, current_dic + last_packet_rem),
+                ).Else(
+                    # We might have inserted some extra IFG bytes (subtracted
+                    # from the deficit)
+                    *shifted_idle_transmit,
+                    If(current_dic - last_packet_rem < 0,
+                        NextValue(current_dic, 0),
+                    ).Else(
+                        NextValue(current_dic, current_dic - last_packet_rem),
+                    )
+                ),
+            ).Elif(sink.valid & (current_ifg == 1) & (last_packet_rem != 0)
+                   & (current_dic + last_packet_rem <= 3),
+                # Branch C: we've transmitted at least 4 bytes of IFG. Whether
+                # we can start a new transmission here depends on the DIC. In
+                # any case, we're deleting at least one XGMII idle character,
+                # which we need to keep track of. Furthermore, transmission can
+                # only ever start on the fifth octect here.
+                *shifted_idle_transmit,
+                NextValue(current_dic, current_dic + last_packet_rem),
             ).Else(
-                # Idling, transmit XGMII IDLE control characters
+                # Idling, transmit XGMII IDLE control characters only and add
-                # only. Accept more data.
+                # them to the IFG.
                 pads.tx_ctl.eq(0xFF),
                 pads.tx_data.eq(Cat(*([XGMII_IDLE] * 8))),
-                NextValue(sink.ready, 1),
+                ifg_add_double.eq(1),
+
+                # Accept more data if we've had a sufficiently large inter-frame
+                # gap (accounting for deficit idle count). For this we need to
+                # determine whether the next sink.valid clock cycle will take a
+                # given branch of A, B or C.
+                If((next_ifg >= 2)
+                   | ((next_ifg == 1) & (last_packet_rem != 0)
+                      & (current_dic + last_packet_rem <= 3)),
+                    # Branch A, B or C will be taken as soon as sink.valid
+                    # again, thus accept more data.
+                    NextValue(sink.ready, 1),
+                ).Else(
+                    # We haven't transmitted a sufficient IFG. The next
+                    # sink.valid clock cycle will not start a transmission.
+                    NextValue(sink.ready, 0),
+                ),
+
+                # If we've remained in IDLE because the sink is not yet valid,
+                # even though the full IFG has been sent already, remove any
+                # deficit idle count. We've made up for that by now.
+                If(current_ifg >= 2,
+                    NextValue(current_dic, 0),
+                ),
+
                 NextState("IDLE"),
             )
         )
 
+        # How many bytes % 4 we've transmitted in the current packet. This
+        # signal is to be asserted when the packet ends in the current clock
+        # cycle.
+        #
+        # If we disable the deficit idle count, we replace this with a constant
+        # of 0, meaning that we pretend to not have transmitted any additional
+        # IDLE characters. This should allow significant logic optimizations.
+        if dic:
+            current_packet_rem = Signal(max=4)
+        else:
+            current_packet_rem = Constant(0, bits_sign=2)
+
+        # Wether the current transmission must be ended in the next clock
+        # cycle. This might be required if we haven't transmitted the XGMII end
+        # of frame control character, but send all other data of the packet.
+        end_transmission = Signal()
+
         fsm.act("TRANSMIT",
             # Check whether the data is still valid first or we are are not
             # ready to accept a new transmission.
-            If(~sink.valid | ~sink.ready,
+            If(end_transmission | ~adjusted_sink_valid,
-                # Data isn't valid, or we aren't ready to accept a new
+                # Data isn't valid, but we're still in the transmit state. This
-                # transmission yet as another one has ended but the XGMII end of
+                # can happen because we've finished transmitting all packet
-                # frame control character has not been transmitted. We must
+                # data, but must still transmit the XGMII end of frame control
-                # transmit the end of frame marker and return to
+                # character. Thus put this control character and IDLE on the
-                # afterwards. Immediately accept more data, given we have
+                # line, return to IDLE afterwards.
-                # transmitted the end of frame control character.
                 pads.tx_ctl.eq(0xFF),
                 pads.tx_data.eq(Cat(XGMII_END, Replicate(XGMII_IDLE, 7))),
+                # Also, we're transmitting 64 bits worth of idle characters.
+                ifg_add_double.eq(1),
+                # We're transmitting 8 bytes of IFG in this cycle. Thus we know
+                # that in the next cycle we can for sure start a new
+                # transmission, irrespective of whether we use DIC (either on
+                # the first or fifth byte in the 64-bit word). Thus set
+                # sink.ready accordingly.
                 NextValue(sink.ready, 1),
+                # Packet transmission is complete, return to IDLE and reset the
+                # end_transmission register.
+                NextValue(end_transmission, 0),
                 NextState("IDLE"),
             ).Else(
                 # The data is valid. For each byte, determine whether it is
                 # valid or must be an XGMII idle or end of frame control
                 # character based on the value of last_be.
                 *[
-                    If(~sink.last | (sink.last_be >= (1 << i)),
+                    If((adjusted_sink_valid_last_be == 0)
+                       | (adjusted_sink_valid_last_be >= (1 << i)),
                         # Either not the last data word or last_be indicates
                         # this byte is still valid
                         pads.tx_ctl[i].eq(0),
-                        pads.tx_data[8*i:8*(i+1)].eq(sink.data[8*i:8*(i+1)]),
+                        pads.tx_data[8*i:8*(i+1)].eq(
-                    ).Elif((sink.last_be == (1 << (i - 1))) if i > 0 else 0,
+                            adjusted_sink_valid_data[8*i:8*(i+1)]
+                        ),
+                    ).Elif((adjusted_sink_valid_last_be == (1 << (i - 1)))
+                           if i > 0 else 0,
                         # last_be indicates that this byte is the first one
                         # which is no longer valid, hence transmit the XGMII end
                         # of frame character
                         pads.tx_ctl[i].eq(1),
                         pads.tx_data[8*i:8*(i+1)].eq(XGMII_END),
+                        # Also, starting from this character, the inter-frame
+                        # gap starts. Depending on where we are in the bus word
+                        # (index 0 to 4) we can already count cycle as one
+                        # 32-bit IFG step (the XGMII end of frame character
+                        # counts towards the IFG).
+                        If(i < 5,
+                            ifg_add_single.eq(1),
+                        ),
+                        # If the DIC mechanism is enabled, furthermore keep
+                        # track of the remainder (mod 4) of IDLE bytes being
+                        # sent.
+                        *([
+                            current_packet_rem.eq(i % 4),
+                            NextValue(last_packet_rem, i % 4),
+                        ] if dic else []),
                     ).Else(
                         # We must've transmitted the XGMII end of frame control
                         # character, all other bytes must be XGMII idle control
@@ -93,15 +409,42 @@ class LiteEthPHYXGMIITX(Module):
                 # XGMII bus word containing the XGMII end of frame and idle
                 # control characters. This happens if we remain in the TRANSMIT
                 # state.
-                If(~sink.last,
+                If(adjusted_sink_valid_last_be == 0,
-                    NextValue(sink.ready, 1),
+                    # This hasn't been the last bus word. However, before we can
+                    # tell the data sink to send us additional data, in case
+                    # we're performing a shifted transmission, we must see
+                    # whether the current sink data word already indicates the
+                    # end of data in it's upper half. If so, we must not request
+                    # additional data. Otherwise we could loose valid data, as
+                    # we're transmitting the IFG first.
+                    If(transmit_shifted & sink.last
+                       & ((sink.last_be & 0xF0) != 0),
+                        # We're in a shifted transmit and already have received
+                        # the last data bytes from the sink.
+                        NextValue(sink.ready, 0),
+                    ).Else(
+                        # Everything's good, the sink hasn't yet asserted last.
+                        NextValue(sink.ready, 1),
+                    ),
                     NextState("TRANSMIT"),
-                ).Elif(sink.last_be == (1 << 7),
+                ).Elif(adjusted_sink_valid_last_be == (1 << 7),
-                    # Last data word, but all bytes were valid.
+                    # Last data word, but all bytes were valid. Thus we still
+                    # need to transmit the XGMII end control character.
+                    NextValue(end_transmission, 1),
                     NextValue(sink.ready, 0),
                     NextState("TRANSMIT"),
                 ).Else(
-                    NextValue(sink.ready, 1),
+                    # We did already transmit the XGMII end control
+                    # character. Depending on the interframegap sent as part of
+                    # this cycle and the current deficit idle count, we might
+                    # already be able to accept data in the next clock cycle.
+                    If((next_ifg >= 2)
+                       | ((next_ifg == 1) & (last_packet_rem != 0)
+                          & (current_dic + last_packet_rem <= 3)),
+                        NextValue(sink.ready, 1),
+                    ).Else(
+                        NextValue(sink.ready, 0),
+                    ),
                     NextState("IDLE"),
                 )
             )
@@ -310,12 +653,19 @@ class LiteEthPHYXGMII(Module, AutoCSR):
                  pads,
                  model=False,
                  dw=64,
-                 with_hw_init_reset=True):
+                 with_hw_init_reset=True,
+                 dic=True,
+                 ):
         self.dw = dw
         self.cd_eth_tx, self.cd_eth_rx = "eth_tx", "eth_rx"
+        self.integrated_ifg_inserter = True
         self.submodules.crg = LiteEthPHYXGMIICRG(clock_pads, model)
         self.submodules.tx = ClockDomainsRenamer(self.cd_eth_tx)(
-            LiteEthPHYXGMIITX(pads, self.dw))
+            LiteEthPHYXGMIITX(
+                pads,
+                self.dw,
+                dic=dic,
+            ))
         self.submodules.rx = ClockDomainsRenamer(self.cd_eth_rx)(
             LiteEthPHYXGMIIRX(pads, self.dw))
         self.sink, self.source = self.tx.sink, self.rx.source

test/test_stream.py

@@ -187,6 +187,7 @@ def stream_collector(
         source,
         dest=[],
         expect_npackets=None,
+        stop_cond=None,
         seed=42,
         ready_rand=50,
         debug_print=False):
@@ -194,6 +195,10 @@ def stream_collector(
     `source`. If `source` has a `last_be` signal, that is respected
     properly.
 
+    `stop_cond` can be passed a function which is invoked whenever the
+    collector has a chance to cease operations (when a packet has been
+    fully read). If when invoked it returns true, this function will
+    exit.
     """
 
     prng = random.Random(seed)
@@ -219,6 +224,10 @@ def stream_collector(
 
     # Loop for collecting individual packets, separated by source.last
     while expect_npackets == None or len(dest) < expect_npackets:
+        if stop_cond is not None:
+            if stop_cond():
+                break
+
         # Buffer for the current packet
         collected_bytes = []
         param_signal_states = {}

test/test_xgmii_phy.py

@@ -0,0 +1,572 @@
+#
+# This file is part of LiteEth.
+#
+# Copyright (c) 2021 Leon Schuermann <leon@is.currently.online>
+# SPDX-License-Identifier: BSD-2-Clause
+
+import unittest
+import random
+import csv
+from pathlib import Path
+
+from migen import *
+
+from litex.soc.interconnect.stream import *
+from liteeth.phy.xgmii import LiteEthPHYXGMII, LiteEthPHYXGMIIRX
+
+from .test_stream import StreamPacket, stream_inserter, stream_collector, \
+    compare_packets
+
+def mask_last_be(dw, data, last_be):
+    """Mark some data by a last_be data qualifier. The rest of the data
+    passed in will be zeroed.
+    """
+    masked_data = 0
+
+    for byte in range(dw // 8):
+        if 2**byte > last_be:
+            break
+        masked_data |= data & (0xFF << (byte * 8))
+
+    return masked_data
+
+class XGMIICollector:
+    def __init__(self, min_interframegap=12, tolerate_dic=True,
+                 debug_print=False):
+        # Minimum IFG legal to be accepted on the XGMII interface (excluding
+        # DIC, if tolerated). On the receiving send, when accounting for
+        # potential IFG shrinkage and allowing the minimum receive IFG as
+        # mandated by IEEE 802.3 (e.g. `min_interframegap` = 5 bytes IFG for
+        # 10Gbit/s Ethernet), tolerate_dic should thus be disabled.
+        self.min_interframegap = min_interframegap
+
+        # Whether the collector should spit out debug information about received
+        # signal states. Will always print error conditions.
+        self.debug_print = debug_print
+
+        # Whether the deficit idle count mechanism should be tolerated. This
+        # will allow the receiver to temporarily accept IFGs < 12 bytes, as long
+        # as an average inter-frame gap of >= 12 is maintained. This must be
+        # implemented as a 2-bit counter, as per IEEE 802.3-2018, section four,
+        # 46.3.1.4 Start control character alignment.
+        self.tolerate_dic = tolerate_dic
+
+        # Proper deficit idle count, implemented as a two bit counter using the
+        # algorithm described by Eric Lynskey of the UNH InterOperability
+        # Lab[1]:
+        #
+        # | current |             |             |             |             |
+        # | count   |           0 |           1 |           2 |           3 |
+        # |---------+-----+-------+-----+-------+-----+-------+-----+-------|
+        # |         |     | new   |     | new   |     | new   |     | new   |
+        # | pkt % 4 | IFG | count | IFG | count | IFG | count | IFG | count |
+        # |---------+-----+-------+-----+-------+-----+-------+-----+-------|
+        # |       0 |  12 |     0 |  12 |     1 |  12 |     2 |  12 |     3 |
+        # |       1 |  11 |     1 |  11 |     2 |  11 |     3 |  15 |     0 |
+        # |       2 |  10 |     2 |  10 |     3 |  14 |     0 |  14 |     1 |
+        # |       3 |   9 |     3 |  13 |     0 |  13 |     1 |  13 |     2 |
+        #
+        # [1]: https://www.iol.unh.edu/sites/default/files/knowledgebase/10gec/10GbE_DIC.pdf
+        self.dic = 0
+
+        # How many additional IDLE characters we've seen. We are faithful that
+        # the device is complying and initialize this to the mandated IFG byte
+        # count + 3 bytes extra IDLE characters inserted through DIC, in case we
+        # listen in to a captured stream and a new packet starts immediately.
+        self.interframegap = 15
+
+        # Received packets, array of arrays of bytes.
+        self.packets = []
+
+        # Packet currently being received. Array of bytes.
+        self.current_packet = None
+
+        # History of observed inter-frame gaps for debugging purposes.
+        self.interframegaps = []
+
+        # Whether the collector is currently collecting data or done. This can
+        # be very useful information for designing composite test systems and
+        # running until all data has been gathered.
+        self.collecting = False
+
+    def inject_32b_bus_word(self, ctl_word, data_word):
+        for i in range(4):
+            ctl = (ctl_word >> i) & 1
+            data = (data_word >> (i * 8)) & 0xFF
+
+            if ctl == 0 and self.current_packet is not None:
+                # Data byte _and_ currently reading a packet, all fine!
+                self.current_packet += [data]
+
+            elif ctl == 1 and data == 0xFB:
+                # XGMII start of frame control character
+                if self.current_packet is not None:
+                    raise ValueError("Got start of frame control character "
+                                     + "while reading packet")
+
+                if i != 0:
+                    raise ValueError("Got start of frame control character on "
+                                     + "lane {}".format(i))
+
+                # Check and validate the observed IFG
+                if self.tolerate_dic:
+                    if self.interframegap < self.min_interframegap:
+                        # Produced some deficit, check if it's legal
+                        self.dic += self.min_interframegap - self.interframegap
+                        if self.dic > 3:
+                            raise ValueError("DIC bounds exceeded. Observed {} "
+                                             + "bytes IFG, but DIC would have "
+                                             + "allowed {}.".format(
+                                                 self.interfamegap,
+                                                 3 - self.dic))
+
+                    elif self.interframegap > self.min_interframegap:
+                        # Inserted some extra IDLE, subtract from the deficit
+                        self.dic = min(0, self.dic - (
+                            self.interframegap - self.min_interframegap
+                        ))
+
+                elif self.interframegap < self.min_interframegap:
+                    # DIC is disabled
+                    raise ValueError("IFG violated. Oberserved {} bytes, which "
+                                     + "is less than the minimum of {}".format(
+                                         self.interframegap,
+                                         self.min_interframegap
+                                     ))
+
+                # Store the observed IFG for debugging purposes and reset it
+                self.interframegaps += [self.interframegap]
+                self.interframegap = 0
+
+                # Start a new packet. The XGMII start of frame character
+                # replaces the first preamble octet, so store that as the first
+                # byte.
+                self.current_packet = [0x55]
+
+            elif ctl == 1 and data == 0xFD:
+                # XGMII end of frame control character
+
+                if self.current_packet is None:
+                    if len(self.packets) == 0 and self.debug_print:
+                        print("INFO: got end of frame control character while "
+                              + "not reading a packet. This can be valid for "
+                              + "the first partial packet in the capture, but "
+                              + "not afterwards.")
+                    elif len(self.packets) != 0:
+                        raise ValueError("Got end of frame control character "
+                                         + "while not reading a packet.")
+                else:
+                    if self.debug_print:
+                        print("Received XGMII packet {}.".format(
+                            len(self.packets)
+                        ))
+
+                    # Transmission ended, store the packet and reset the current
+                    # packet.
+                    self.packets += [self.current_packet]
+                    self.current_packet = None
+
+                    # The XGMII end of frame control character does count
+                    # towards the IFG
+                    self.interframegap = 1
+
+                    # All following bytes MUST be XGMII IDLE control
+                    # characters. We will want to verify that and
+                    # count the number of bytes until i % 4 == 0
+                    # towards the DIC counter.
+                    end_of_frame = True
+
+            elif ctl == 1 and data == 0x07:
+                # XGMII idle control character
+
+                if self.current_packet is not None:
+                    raise ValueError("Got idle control character in the middle "
+                                     + "of a packet")
+
+                self.interframegap += 1
+
+            elif ctl == 1:
+                # Unrecognized XGMII control character
+
+                raise ValueError("Invalid XGMII control character {:02x}"
+                                 .format(data))
+
+    def inject_bus_word(self, dw, ctl_word, data_word):
+        if dw == 32:
+            self.inject_32b_bus_word(ctl_word, data_word)
+        elif dw == 64:
+            self.inject_32b_bus_word(
+                ctl_word & 0xF,
+                data_word & 0xFFFFFFFF
+            )
+            self.inject_32b_bus_word(
+                (ctl_word >> 4) & 0xF,
+                (data_word >> 32) & 0xFFFFFFFF,
+            )
+        else:
+            raise ValueError("Unknown data width: {} bits".format(dw))
+
+
+    def collect(self, xgmii_interface, tap_signals="tx", stop_cond=None):
+        self.collecting = True
+
+        # Which signals to attach to in the passed XGMII interface
+        assert tap_signals in ["tx", "rx"]
+
+        if stop_cond is None:
+            stop_cond = lambda: False
+
+        while not stop_cond():
+            if tap_signals == "tx":
+                ctl_word = yield xgmii_interface.tx_ctl
+                data_word = yield xgmii_interface.tx_data
+                dw = len(xgmii_interface.tx_data)
+            elif tap_signals == "rx":
+                ctl_word = yield xgmii_interface.rx_ctl
+                data_word = yield xgmii_interface.rx_data
+                dw = len(xgmii_interface.rx_data)
+            self.inject_bus_word(dw, ctl_word, data_word)
+            yield
+
+        self.collecting = False
+
+class XGMII64bCSVReader:
+    def __init__(self, filename, extract_signals_pattern="rx",
+                 complete_trailing_transaction=True):
+        # Whether we should attempt to complete a trailing XGMII transaction by
+        # inserting an XGMII end control character.
+        self.complete_trailing_transaction = complete_trailing_transaction
+
+        # Store filename for reference
+        self.filename = filename
+
+        # Open the CSV file and create a CSV reader over it
+        self.filehandle = open(filename, 'r')
+        self.reader = csv.reader(
+            # Support comments in the CSV file
+            filter(lambda line: not line.startswith("#"), self.filehandle),
+            delimiter=','
+        )
+
+        # Extract the headers and correlate them with the required signal
+        # pattern to find out the matching columns
+        self.column_headers = next(self.reader, None)
+        assert self.column_headers is not None, \
+            "Failed to load column header row from CSV"
+
+        self.rx_ctl_col = None
+        self.rx_data_col = None
+        for i, header in enumerate(self.column_headers):
+            if "{}_ctl".format(extract_signals_pattern) in header:
+                self.rx_ctl_col = i
+            elif "{}_data".format(extract_signals_pattern) in header:
+                self.rx_data_col = i
+
+        assert self.rx_ctl_col is not None, \
+            "Failed to find RX CTL signal column in CSV"
+        assert self.rx_data_col is not None, \
+            "Failed to find RX DATA signal column in CSV"
+
+        self.datatype_headers = next(self.reader, None)
+        assert self.datatype_headers is not None, \
+            "Failed to load data type header row from CSV"
+        assert self.datatype_headers[self.rx_ctl_col] == "HEX", \
+            "XGMII CTL signal is not hex-encoded"
+        assert self.datatype_headers[self.rx_data_col] == "HEX", \
+            "XGMII DATA signal is not hex-encoded"
+
+        # Whether the inserter is currently in the middle of a packet
+        self.currently_in_packet = False
+
+        # Hack around Python not having a peekable iterator by always taking the
+        # next element at the end of a function.
+        self.next_row = next(self.reader, None)
+
+        # Upper XGMII bus word, for when 32 bit words are accessed
+        self.upper_ctl = None
+        self.upper_data = None
+
+    def __del__(self):
+        self.filehandle.close()
+
+    def get_64b_bus_word(self):
+        if self.next_row is None:
+            return None
+
+        ctl_word = int(self.next_row[self.rx_ctl_col], 16)
+        data_word = int(self.next_row[self.rx_data_col], 16)
+
+        new_packet = False
+
+        # Detect whether this is just starting a new packet
+        for i in range(8):
+            ctl = (ctl_word >> i) & 1
+            data = (data_word >> (i * 8)) & 0xFF
+
+            if ctl == 1 and data == 0xFB:
+                new_packet = True
+                self.currently_in_packet = True
+
+            if ctl == 1 and data == 0xFD:
+                self.currently_in_packet = False
+
+        # Store for below
+        prev_row = self.next_row
+        self.next_row = next(self.reader, None)
+
+        # Let's make sure we have now half / dangling packet at the end. If this
+        # was the last column and we're currently receiving one, make sure we
+        # end it properly. This might go terribly wrong if this is condition
+        # aries at the start of a packet though, in this case just throw an
+        # error.
+        if self.next_row is None and self.currently_in_packet \
+            and self.complete_trailing_transaction:
+
+            if new_packet:
+                raise ValueError("CSV ends just at the start of a new packet, "
+                                 + "we can't handle that!")
+
+            self.next_row = prev_row
+            self.next_row[self.rx_ctl_col] = "ff"
+            self.next_row[self.rx_data_col] = "07070707070707fd"
+
+        return (ctl_word, data_word)
+
+    def get_bus_word(self, dw=64):
+        if dw == 64:
+            assert self.upper_data is None and self.upper_ctl is None, \
+                "Cannot query 32bit and 64bit bus words interchangeably"
+
+            return self.get_64b_bus_word()
+        elif dw == 64:
+            if self.upper_data is not None:
+                assert self.upper_ctl is not None
+
+                return (self.upper_ctl, self.upper_data)
+            else:
+                xgmii_64b = self.get_64b_bus_word()
+
+                self.upper_ctl = (xgmii_64b[0] >> 4) & 0xF
+                self.upper_data = (xgmii_64b[1] >> 32) & 0xFFFFFFFF
+
+                return (
+                    xgmii_64b[0] & 0xF,
+                    xgmii_64b[1] & 0xFFFFFFFF,
+                )
+        else:
+            raise ValueError("Unknown dw {}!".format(dw))
+
+    def done(self):
+        return self.next_row is None
+
+    def inject(self, xgmii_interface, stop_cond=None):
+        proper_stop_cond = True
+
+        if stop_cond is None:
+            proper_stop_cond = False
+            stop_cond = lambda: False
+
+        while not self.done():
+            if stop_cond():
+                return
+
+            (ctl, data) = self.get_bus_word(len(xgmii_interface.rx_data))
+            yield xgmii_interface.rx_ctl.eq(ctl)
+            yield xgmii_interface.rx_data.eq(data)
+            yield
+
+        while self.done() and proper_stop_cond and not stop_cond():
+            yield xgmii_interface.rx_ctl.eq(0xFF)
+            yield xgmii_interface.rx_data.eq(0x0707070707070707)
+            yield
+
+
+class TestXGMIIPHY(unittest.TestCase):
+    def test_xgmii_rx(self):
+        # Read XGMII data from the CSV file
+        csv_file = Path(__file__).parent / "assets" / "xgmii_bus_capture.csv"
+        xgmii_injector = XGMII64bCSVReader(
+            csv_file.resolve(),
+            complete_trailing_transaction=True
+        )
+
+        # Collect the XGMII transactions from the reader with a minimum
+        # inter-frame gap of 5 (accounted for potential IFG shrinkage).
+        xgmii_collector = XGMIICollector(
+            min_interframegap=5,
+            tolerate_dic=False,
+            debug_print=True,
+        )
+
+        # XGMII interface
+        xgmii_interface = Record([
+            ("rx_ctl", 8),
+            ("rx_data", 64),
+            ("tx_ctl", 8),
+            ("tx_data", 64),
+        ])
+
+        # We simply test the receiver component (XGMII -> stream) here.
+        dut = LiteEthPHYXGMIIRX(
+            xgmii_interface,
+            64,
+        )
+
+        recvd_packets = []
+        run_simulation(
+            dut,
+            [
+                xgmii_injector.inject(
+                    xgmii_interface,
+                ),
+                xgmii_collector.collect(
+                    xgmii_interface,
+                    tap_signals="rx",
+                    stop_cond=lambda: xgmii_injector.done() \
+                        and xgmii_collector.current_packet is None,
+                ),
+                stream_collector(
+                    dut.source,
+                    dest=recvd_packets,
+                    stop_cond=xgmii_injector.done,
+                    seed=42,
+                    debug_print=True,
+                    # The XGMII PHY RX part deliberately does not support a
+                    # deasserted ready signal. The sink is assumed to be always
+                    # ready.
+                    ready_rand=0,
+                ),
+            ],
+        )
+
+        self.assertTrue(
+            len(recvd_packets) == len(xgmii_collector.packets),
+            "Different number of received and sent packets: {} vs. {}!"
+            .format(len(recvd_packets), len(xgmii_collector.packets))
+        )
+        for p, (recvd, sent) in enumerate(
+                zip(recvd_packets, xgmii_collector.packets)):
+            self.assertTrue(
+                len(recvd.data) == len(sent),
+                ("Packet sent and received with different length: {} vs. {} "
+                 + "at packet {}!"
+                ).format(len(recvd.data), len(sent), p)
+            )
+            for i, (a, b) in enumerate(zip(recvd.data, sent)):
+                self.assertTrue(
+                    a == b,
+                    ("Byte sent and received differ: {} vs. {} at {} byte of "
+                     + "packet {}"
+                    ).format(a, b, i, p)
+                )
+
+    def test_xgmii_stream_loopback(self):
+        # Read XGMII data from the CSV file
+        csv_file = Path(__file__).parent / "assets" / "xgmii_bus_capture.csv"
+        xgmii_injector = XGMII64bCSVReader(
+            csv_file.resolve(),
+            complete_trailing_transaction=True
+        )
+
+        # Collect the XGMII transactions from the CSV reader with a minimum
+        # inter-frame gap of 5 (accounted for potential IFG shrinkage).
+        xgmii_rx_collector = XGMIICollector(
+            min_interframegap=5,
+            tolerate_dic=False,
+
+            debug_print=True
+        )
+
+        # Collect the XGMII transactions from the TX PHY with a minimum
+        # inter-frame gap of 5. As a dumb loopback (akin to the behavior of a
+        # repeater) this is the smallest IPG value which may be put on the wire
+        # again.
+        xgmii_tx_collector = XGMIICollector(
+            min_interframegap=12,
+            tolerate_dic=True,
+            debug_print=True
+        )
+
+        class DUT(Module):
+            def __init__(self):
+                # XGMII signals
+                self.xgmii_interface = Record([
+                    ("rx_ctl", 8),
+                    ("rx_data", 64),
+                    ("tx_ctl", 8),
+                    ("tx_data", 64),
+                ])
+
+                # PHY with TX and RX side
+                self.submodules.ethphy = ClockDomainsRenamer({
+                    "eth_tx": "sys",
+                    "eth_rx": "sys",
+                })(LiteEthPHYXGMII(
+                    Record([("rx", 1), ("tx", 1)]),
+                    self.xgmii_interface,
+                    model=True,
+                ))
+
+                # Insert a synchronous FIFO to allow some variability of the
+                # inter-frame gap. If it overflows we know that we're not
+                # processing data at line rate, thus we must make sure that we
+                # detect such a case.
+                self.submodules.loopback_fifo = SyncFIFO(
+                    self.ethphy.source.payload.layout,
+                    4,
+                    True,
+                )
+
+                self.comb += [
+                    self.ethphy.source.connect(self.loopback_fifo.sink),
+                    self.loopback_fifo.source.connect(self.ethphy.sink),
+                ]
+
+        dut = DUT()
+        run_simulation(
+            dut,
+            [
+                xgmii_rx_collector.collect(
+                    dut.xgmii_interface,
+                    tap_signals="rx",
+                    stop_cond=lambda: xgmii_injector.done() \
+                        and xgmii_rx_collector.current_packet is None,
+                ),
+                xgmii_tx_collector.collect(
+                    dut.xgmii_interface,
+                    tap_signals="tx",
+                    stop_cond=lambda: xgmii_injector.done() \
+                        and xgmii_tx_collector.current_packet is None \
+                        and len(xgmii_tx_collector.packets) \
+                            >= len(xgmii_rx_collector.packets),
+                ),
+                xgmii_injector.inject(
+                    dut.xgmii_interface,
+                    stop_cond=lambda: not xgmii_rx_collector.collecting \
+                        and not xgmii_tx_collector.collecting,
+                ),
+            ],
+            vcd_name="xgmii_fuuuuuck.vcd",
+        )
+
+        self.assertTrue(
+            len(xgmii_rx_collector.packets) == len(xgmii_tx_collector.packets),
+            "Different number of sent and received packets: {} vs. {}!"
+            .format(len(xgmii_rx_collector.packets),
+                    len(xgmii_tx_collector.packets))
+        )
+        for p, (recvd, sent) in enumerate(
+                zip(xgmii_tx_collector.packets, xgmii_rx_collector.packets)):
+            self.assertTrue(
+                len(recvd) == len(sent),
+                ("Packet sent and received with different length: {} vs. {} at "
+                 + "packet {}!"
+                ).format(len(recvd), len(sent), p)
+            )
+            for i, (a, b) in enumerate(zip(recvd, sent)):
+                self.assertTrue(
+                    a == b,
+                    ("Byte sent and received differ: {} vs. {} at {} byte of "
+                     + "packet {}"
+                    ).format(a, b, i, p)
+                )