diff --git a/applications/rdma_demo/libraries/rdma_packetiser/cocotb/dp_bus.py b/applications/rdma_demo/libraries/rdma_packetiser/cocotb/dp_bus.py
new file mode 100644
index 0000000000000000000000000000000000000000..b54687aeb717139b5900ae4c2639bb76f80861a8
--- /dev/null
+++ b/applications/rdma_demo/libraries/rdma_packetiser/cocotb/dp_bus.py
@@ -0,0 +1,386 @@
+# ##########################################################################
+# Copyright 2024
+# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/>
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ##########################################################################
+# Author:
+# . Reinier vd Walle
+#
+# Purpose:
+# . Provide abstraction layer for using DP bus (sosi/siso) in cocotb.
+#
+# Description:
+# . This file contains classes to create an abstraction layer for
+# interacting with DP sosi/siso signals in cocotb testbenches.
+# . The classes are copied from AvalonSTPkts in [1] and slightly adapted to
+# use sosi/siso signals
+#
+# References:
+# [1] - https://docs.cocotb.org/en/v1.5.1/_modules/cocotb_bus/drivers/avalon.html
+# ##########################################################################
+from typing import Iterable, Union, Optional
+
+import cocotb
+from cocotb.utils import hexdump
+from cocotb.triggers import RisingEdge, ReadOnly
+from cocotb.binary import BinaryValue
+from cocotb.result import TestError
+from cocotb_bus.drivers import ValidatedBusDriver
+from cocotb_bus.monitors import BusMonitor
+
+class SosiDriver(ValidatedBusDriver):
+ _optional_signals = ['valid', 'sop', 'eop', 'sync', 'bsn', 'data', 're', 'im', 'empty', 'channel', 'err']
+ _signals = []
+ _default_config = {
+ "dataBitsPerSymbol" : 8,
+ "firstSymbolInHighOrderBits" : True,
+ "maxChannel" : 1,
+ "readyLatency" : 1
+ }
+ def __init__(self, dut, name, clk, data_w = None, bsn_init = 0):
+ ValidatedBusDriver.__init__(self, dut, name, clk, bus_separator='.')
+ self.config = self._default_config.copy()
+ self.use_empty = True
+ self.config["useEmpty"] = self.use_empty
+ if data_w == None:
+ self.data_w = len(self.bus.data)
+ else:
+ self.data_w = data_w
+ self.bsn = bsn_init
+
+ async def _wait_ready(self):
+ """Wait for a ready cycle on the bus before continuing.
+
+ Can no longer drive values this cycle...
+
+ FIXME assumes readyLatency of 0
+ """
+ await ReadOnly()
+ while False: #not self.bus.ready.value:
+ await RisingEdge(self.clock)
+ await ReadOnly()
+
+ async def _send_string(self, string: bytes, sync: bool = True, channel: Optional[int] = None) -> None:
+ """Args:
+ string: A string of bytes to send over the bus.
+ channel: Channel to send the data on.
+ """
+ # Avoid spurious object creation by recycling
+ clkedge = RisingEdge(self.clock)
+ firstword = True
+
+ # FIXME: buses that aren't an integer numbers of bytes
+ bus_width = int(self.data_w / 8)
+
+ word = BinaryValue(n_bits=len(self.bus.data),
+ bigEndian=self.config["firstSymbolInHighOrderBits"])
+
+ single = BinaryValue(n_bits=1, bigEndian=False)
+ if self.use_empty:
+ empty = BinaryValue(n_bits=len(self.bus.empty), bigEndian=False)
+
+ # Drive some defaults since we don't know what state we're in
+ if self.use_empty:
+ self.bus.empty.value = 0
+ self.bus.sop.value = 0
+ self.bus.eop.value = 0
+ self.bus.valid.value = 0
+ if hasattr(self.bus, 'err'):
+ self.bus.err.value = 0
+
+ if hasattr(self.bus, 'channel'):
+ self.bus.channel.value = 0
+ elif channel is not None:
+ raise TestError("%s does not have a channel signal" % self.name)
+
+ while string:
+ if not firstword or (firstword and sync):
+ await clkedge
+
+ # Insert a gap where valid is low
+ if not self.on:
+ self.bus.valid.value = 0
+ for _ in range(self.off):
+ await clkedge
+
+ # Grab the next set of on/off values
+ self._next_valids()
+
+ # Consume a valid cycle
+ if self.on is not True and self.on:
+ self.on -= 1
+
+ self.bus.valid.value = 1
+ if hasattr(self.bus, 'channel'):
+ if channel is None:
+ self.bus.channel.value = 0
+ elif channel > self.config['maxChannel'] or channel < 0:
+ raise TestError("%s: Channel value %d is outside range 0-%d" %
+ (self.name, channel, self.config['maxChannel']))
+ else:
+ self.bus.channel.value = channel
+
+ if firstword:
+ self.bus.sop.value = 1
+ self.bus.bsn.value = self.bsn
+ self.bsn += 1
+ firstword = False
+ else:
+ self.bus.sop.value = 0
+
+ nbytes = min(len(string), bus_width)
+ data = string[:nbytes]
+
+ # set unused bits of dp_sosi.data field to 0
+ data = bytes(word.n_bits // 8 - bus_width) + data
+ word.buff = data
+
+ if len(string) <= bus_width:
+ self.bus.eop.value = 1
+ if self.use_empty:
+ self.bus.empty.value = bus_width - len(string)
+ string = b""
+ else:
+ string = string[bus_width:]
+
+ self.bus.data.value = word
+
+ # If this is a bus with a ready signal, wait for this word to
+ # be acknowledged
+ if hasattr(self.bus, "ready"):
+ await self._wait_ready()
+
+ await clkedge
+ self.bus.valid.value = 0
+ self.bus.eop.value = 0
+ word.binstr = "x" * self.data_w
+ single.binstr = "x"
+ self.bus.data.value = word
+ self.bus.sop.value = single
+ self.bus.eop.value = single
+
+ if self.use_empty:
+ empty.binstr = "x" * len(self.bus.empty)
+ self.bus.empty.value = empty
+ if hasattr(self.bus, 'channel'):
+ channel_value = BinaryValue(n_bits=len(self.bus.channel), bigEndian=False,
+ value="x" * len(self.bus.channel))
+ self.bus.channel.value = channel_value
+
+ async def _send_iterable(self, pkt: Iterable, sync: bool = True) -> None:
+ """Args:
+ pkt: Will yield objects with attributes matching the
+ signal names for each individual bus cycle.
+ """
+ clkedge = RisingEdge(self.clock)
+ firstword = True
+
+ for word in pkt:
+ if not firstword or (firstword and sync):
+ await clkedge
+
+ firstword = False
+
+ # Insert a gap where valid is low
+ if not self.on:
+ self.bus.valid.value = 0
+ for _ in range(self.off):
+ await clkedge
+
+ # Grab the next set of on/off values
+ self._next_valids()
+
+ # Consume a valid cycle
+ if self.on is not True and self.on:
+ self.on -= 1
+
+ if not hasattr(word, "valid"):
+ self.bus.valid.value = 1
+ else:
+ self.bus.value = word
+
+ # Wait for valid words to be acknowledged
+ if not hasattr(word, "valid") or word.valid:
+ if hasattr(self.bus, "ready"):
+ await self._wait_ready()
+
+ await clkedge
+ self.bus.valid.value = 0
+
+ async def _driver_send(self, pkt: Union[bytes, Iterable], sync: bool = True, channel: Optional[int] = None):
+ """Send a packet over the bus.
+
+ Args:
+ pkt: Packet to drive onto the bus.
+ channel: Channel attributed to the packet.
+
+ If ``pkt`` is a string, we simply send it word by word
+
+ If ``pkt`` is an iterable, it's assumed to yield objects with
+ attributes matching the signal names.
+ """
+
+ # Avoid spurious object creation by recycling
+ if isinstance(pkt, bytes):
+ self.log.debug("Sending packet of length %d bytes", len(pkt))
+ self.log.debug(hexdump(pkt))
+ await self._send_string(pkt, sync=sync, channel=channel)
+ self.log.debug("Successfully sent packet of length %d bytes", len(pkt))
+ elif isinstance(pkt, str):
+ raise TypeError("pkt must be a bytestring, not a unicode string")
+ else:
+ if channel is not None:
+ self.log.warning("%s is ignoring channel=%d because pkt is an iterable", self.name, channel)
+ await self._send_iterable(pkt, sync=sync)
+
+class SisoDriver(ValidatedBusDriver):
+ _optional_signals = ['ready', 'xon']
+ _signals = []
+ def __init__(self, dut, name, clk):
+ ValidatedBusDriver.__init__(self, dut, name, clk, bus_separator='.')
+
+class SosiMonitor(BusMonitor):
+ _optional_signals = ['valid', 'sop', 'eop', 'sync', 'bsn', 'data', 're', 'im', 'empty', 'channel', 'err']
+ _signals = []
+
+ _default_config = {
+ "dataBitsPerSymbol" : 8,
+ "firstSymbolInHighOrderBits" : True,
+ "maxChannel" : 1,
+ "readyLatency" : 1,
+ "invalidTimeout" : 0,
+ }
+ def __init__(self, dut, name, clk, data_w=None, reset=None, reset_n=None, report_channel=False):
+ BusMonitor.__init__(self, dut, name, clk, reset=reset, reset_n=reset_n, bus_separator='.')
+ self.config = self._default_config.copy()
+ self.use_empty = True
+ self.config["useEmpty"] = self.use_empty
+ self.report_channel = report_channel
+ self.data_w = data_w
+ async def _monitor_recv(self):
+ """Watch the pins and reconstruct transactions."""
+
+ # Avoid spurious object creation by recycling
+ clkedge = RisingEdge(self.clock)
+ pkt = b""
+ in_pkt = False
+ invalid_cyclecount = 0
+ channel = None
+
+ def valid():
+ if hasattr(self.bus, 'ready'):
+ return self.bus.valid.value and self.bus.ready.value
+ return self.bus.valid.value
+
+ while True:
+ await clkedge
+
+ if self.in_reset:
+ continue
+
+ if valid():
+ invalid_cyclecount = 0
+
+ if self.bus.sop.value:
+ if pkt:
+ raise Exception("Duplicate start-of-packet received on %s" %
+ str(self.bus.sop))
+ pkt = b""
+ in_pkt = True
+
+ if not in_pkt:
+ raise Exception("Data transfer outside of "
+ "packet")
+
+ # Handle empty and X's in empty / data
+ vec = BinaryValue()
+ if not self.bus.eop.value:
+ value = self.bus.data.value.get_binstr()[-1*self.data_w:]
+ vec.assign(value)
+ else:
+ value = self.bus.data.value.get_binstr()[-1*self.data_w:]
+ if self.config["useEmpty"] and self.bus.empty.value.integer:
+ empty = self.bus.empty.value.integer * self.config["dataBitsPerSymbol"]
+ if self.config["firstSymbolInHighOrderBits"]:
+ value = value[:-empty]
+ else:
+ value = value[empty:]
+
+ vec.assign(value)
+ if not vec.is_resolvable:
+ raise Exception("After empty masking value is still bad? "
+ "Had empty {:d}, got value {:s}".format(empty,
+ self.bus.data.value.get_binstr()))
+ vec.big_endian = self.config['firstSymbolInHighOrderBits']
+ pkt += vec.buff
+
+ if hasattr(self.bus, 'channel'):
+ if channel is None:
+ channel = self.bus.channel.value.integer
+ if channel > self.config["maxChannel"]:
+ raise Exception("Channel value (%d) is greater than maxChannel (%d)" %
+ (channel, self.config["maxChannel"]))
+ elif self.bus.channel.value.integer != channel:
+ raise Exception("Channel value changed during packet")
+
+ if self.bus.eop.value:
+ self.log.info("Received a packet of %d bytes", len(pkt))
+ self.log.debug(hexdump(pkt))
+ self.channel = channel
+ if self.report_channel:
+ self._recv({"data": pkt, "channel": channel})
+ else:
+ self._recv(pkt)
+ pkt = b""
+ in_pkt = False
+ channel = None
+ else:
+ if in_pkt:
+ invalid_cyclecount += 1
+ if self.config["invalidTimeout"]:
+ if invalid_cyclecount >= self.config["invalidTimeout"]:
+ raise Exception(
+ "In-Packet Timeout. Didn't receive any valid data for %d cycles!" %
+ invalid_cyclecount)
+
+class SisoMonitor(BusMonitor):
+ _optional_signals = ['ready', 'xon']
+ _signals = []
+ def __init__(self, dut, name, clk, reset=None, reset_n=None):
+ BusMonitor.__init__(self, dut, name, clk, reset=reset, reset_n=reset_n, bus_separator='.')
+ self.clk = clk
+
+ async def _monitor_recv(self):
+ """Watch the pins and reconstruct transactions."""
+
+ # Avoid spurious object creation by recycling
+ clkedge = RisingEdge(self.clock)
+
+ # NB could await on valid here more efficiently?
+ while True:
+ await clkedge
+ self._recv({"ready": self.bus.ready.value, "xon": self.bus.xon.value})
+
+class DpStream:
+ # at the time, cocotb_bus does not officially support VHDL record type bus.
+ # It does work in the following way:
+ # 1. Only use _optional_signals, no _signals.
+ # 2. Use bus_separator='.'
+ def __init__(self, dut, sosi_name, siso_name, clk, rst, data_w = None, bsn_init = 0):
+ self.sosi_drv = SosiDriver(dut, sosi_name, clk, data_w, bsn_init)
+ self.siso_drv = SisoDriver(dut, siso_name, clk)
+ self.sosi_mon = SosiMonitor(dut, sosi_name, clk, data_w, reset=rst)
+ self.siso_mon = SisoMonitor(dut, siso_name, clk, reset=rst)
+ cocotb.start_soon(self.siso_mon._monitor_recv())
diff --git a/applications/rdma_demo/libraries/rdma_packetiser/cocotb/mm_bus.py b/applications/rdma_demo/libraries/rdma_packetiser/cocotb/mm_bus.py
new file mode 100644
index 0000000000000000000000000000000000000000..992505c4f8f88aec2a48cfcbf440c3e8f0e48787
--- /dev/null
+++ b/applications/rdma_demo/libraries/rdma_packetiser/cocotb/mm_bus.py
@@ -0,0 +1,211 @@
+# ##########################################################################
+# Copyright 2024
+# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/>
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ##########################################################################
+# Author:
+# . Reinier vd Walle
+#
+# Purpose:
+# . Provide abstraction layer for using MM bus (copi/cipo) in cocotb.
+#
+# Description:
+# . This file contains classes to create an abstraction layer for
+# interacting with MM copi/cipo signals in cocotb testbenches.
+# . The classes are copied from AvalonMM in [1] and slightly adapted to
+# use copi/cipo signals
+#
+# References:
+# [1] - https://docs.cocotb.org/en/v1.5.1/_modules/cocotb_bus/drivers/avalon.html
+# ##########################################################################
+from cocotb.log import SimLog
+from cocotb.triggers import RisingEdge, ReadOnly
+from cocotb.binary import BinaryValue
+from cocotb.result import TestError
+from cocotb_bus.drivers import BusDriver
+from cocotb.decorators import coroutine
+
+class CopiDriver(BusDriver):
+ _signals = []
+ _optional_signals = ["rd", "wr", "wrdata", "address"]
+
+ def __init__(self, entity, name, clock, **kwargs):
+ BusDriver.__init__(self, entity, name, clock, **kwargs)
+ self._can_read = False
+ self._can_write = False
+
+ # Drive some sensible defaults (setimmediatevalue to avoid x asserts)
+ if hasattr(self.bus, "rd"):
+ self.bus.rd.setimmediatevalue(0)
+ self._can_read = True
+
+ if hasattr(self.bus, "wr"):
+ self.bus.wr.setimmediatevalue(0)
+ v = self.bus.wrdata.value
+ v.binstr = "x" * len(self.bus.wrdata)
+ self.bus.wrdata.value = v
+ self._can_write = True
+
+ if hasattr(self.bus, "address"):
+ v = self.bus.address.value
+ v.binstr = "x" * len(self.bus.address)
+ self.bus.address.setimmediatevalue(v)
+
+ def read(self, address):
+ pass
+
+ def write(self, address, value):
+ pass
+
+
+class CipoDriver(BusDriver):
+ _signals = []
+ _optional_signals = ["rddata", "waitrequest","rdval"]
+
+ def __init__(self, entity, name, clock, **kwargs):
+ BusDriver.__init__(self, entity, name, clock, **kwargs)
+ self._can_read = True
+ self._can_write = False
+
+ # Drive some sensible defaults (setimmediatevalue to avoid x asserts)
+ if hasattr(self.bus, "rdval"):
+ self.bus.rdval.setimmediatevalue(0)
+
+ if hasattr(self.bus, "rddata"):
+ v = self.bus.rddata.value
+ v.binstr = "x" * len(self.bus.rddata)
+ self.bus.rddata.value = v
+
+ if hasattr(self.bus, "waitrequest"):
+ self.bus.waitrequest.setimmediatevalue(0)
+
+ def read(self, address):
+ pass
+
+ def write(self, address, value):
+ pass
+
+class MMController():
+ """Memory Mapped Interface (MM) Controller."""
+
+ def __init__(self, entity, copi_name, cipo_name, clock):
+ self._log = SimLog("cocotb.%s.%s" % (entity._name, copi_name))
+ self._log.debug("Memory Mapped Controller created")
+ self.copi = CopiDriver(entity, copi_name, clock, bus_separator='.')
+ self.cipo = CipoDriver(entity, cipo_name, clock, bus_separator='.')
+ self.clock = clock
+
+ def __len__(self):
+ return 2**len(self.copi.bus.address)
+
+ @coroutine
+ async def read(self, address: int, sync: bool = True) -> BinaryValue:
+ """Issue a request to the bus and block until this comes back.
+
+ Simulation time still progresses
+ but syntactically it blocks.
+
+ Args:
+ address: The address to read from.
+ sync: Wait for rising edge on clock initially.
+ Defaults to True.
+
+ Returns:
+ The read data value.
+
+ Raises:
+ :any:`TestError`: If controller is write-only.
+ """
+ if not self.copi._can_read:
+ self._log.error("Cannot read - have no read signal")
+ raise TestError("Attempt to read on a write-only MM Controller")
+
+ await self.copi._acquire_lock()
+
+ # Apply values for next clock edge
+ if sync:
+ await RisingEdge(self.clock)
+ self.copi.bus.address.value = address
+ self.copi.bus.rd.value = 1
+
+
+ # Wait for waitrequest to be low
+ if hasattr(self.cipo.bus, "waitrequest"):
+ await self.cipo._wait_for_nsignal(self.cipo.bus.waitrequest)
+ await RisingEdge(self.clock)
+
+ # Deassert read
+ self.copi.bus.rd.value = 0
+ #v = self.copi.bus.address.value
+ #v.binstr = "x" * len(self.copi.bus.address)
+ #self.copi.bus.address.value = v
+
+ if hasattr(self.cipo.bus, "rdval"):
+ while True:
+ await ReadOnly()
+ if int(self.cipo.bus.rdval):
+ break
+ await RisingEdge(self.clock)
+ else:
+ # Assume readLatency = 1 if no readdatavalid
+ # FIXME need to configure this,
+ # should take a dictionary of Avalon properties.
+ await ReadOnly()
+
+ # Get the data
+ data = self.cipo.bus.rddata.value
+
+ self.copi._release_lock()
+ return data
+
+ @coroutine
+ async def write(self, address: int, value: int) -> None:
+ """Issue a write to the given address with the specified
+ value.
+
+ Args:
+ address: The address to write to.
+ value: The data value to write.
+
+ Raises:
+ :any:`TestError`: If controller is read-only.
+ """
+ if not self.copi._can_write:
+ self._log.error("Cannot write - have no write signal")
+ raise TestError("Attempt to write on a read-only MM Controller")
+
+ await self.copi._acquire_lock()
+
+ # Apply values to bus
+ await RisingEdge(self.clock)
+ self.copi.bus.address.value = address
+ self.copi.bus.wrdata.value = value
+ self.copi.bus.wr.value = 1
+
+ # Wait for waitrequest to be low
+ if hasattr(self.cipo.bus, "waitrequest"):
+ await self.cipo._wait_for_nsignal(self.cipo.bus.waitrequest)
+
+ # Deassert write
+ await RisingEdge(self.clock)
+ self.copi.bus.wr.value = 0
+ #v = self.copi.bus.address.value
+ #v.binstr = "x" * len(self.copi.bus.address)
+ #self.copi.bus.address.value = v
+
+ #v = self.copi.bus.wrdata.value
+ #v.binstr = "x" * len(self.copi.bus.wrdata)
+ #self.copi.bus.wrdata.value = v
+ self.copi._release_lock()
diff --git a/applications/rdma_demo/libraries/rdma_packetiser/cocotb/tb_rdma_packetiser.py b/applications/rdma_demo/libraries/rdma_packetiser/cocotb/tb_rdma_packetiser.py
new file mode 100644
index 0000000000000000000000000000000000000000..b58cb4822bc1522874b155906b6690aabe6f7f38
--- /dev/null
+++ b/applications/rdma_demo/libraries/rdma_packetiser/cocotb/tb_rdma_packetiser.py
@@ -0,0 +1,333 @@
+# ##########################################################################
+# Copyright 2024
+# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/>
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ##########################################################################
+# Author:
+# . Reinier vd Walle
+#
+# Purpose:
+# . Test the rdma_packetiser HDL dut using cocotb.
+#
+# Description:
+# The testbench performs the following actions
+# . Write and read/verify the rdma_packetiser configuration and header values over MM.
+# . Send a set of DP packets containing just data (no headers).
+# . Verify the output DP packets, the following parts of the packet is verified.
+# . Packet header (ETH + IP + UDP + RDMA)
+# . Packet payload (should be identical to the input data)
+# . RDMA icrc value
+#
+# Usage:
+# . do tb_rdma_packetiser.do
+# . as 5
+# . run -all
+# ##########################################################################
+from zlib import crc32
+
+import cocotb
+from cocotb.utils import hexdump
+from cocotb.triggers import FallingEdge, Timer
+from cocotb.clock import Clock
+from cocotb.binary import BinaryValue
+
+from dp_bus import DpStream
+from mm_bus import MMController
+
+# Global constants
+c_bth_first = 0x26
+c_bth_middle = 0x27
+c_bth_last = 0x28
+c_bth_last_imm = 0x29
+c_bth_wo = 0x2A
+c_bth_wo_imm = 0x2B
+
+async def perform_rst(rst, clk, cycles):
+ rst.value = 1
+ await Timer(cycles * clk.period)
+ rst.value = 0
+
+async def read_mm_dict(mm: MMController, read_dict: dict) -> dict:
+ # remove all write-only entries
+ d = {k: v for k, v in read_dict.items() if 'r' in v['access'].lower()}
+ return_dict = {}
+ for k, v in d.items():
+ os = v['offset']
+ rddata = [(await mm.read(addr)).binstr for addr in range(os, os + v['size'])]
+ # concat to 32 bits
+ rddata = [x[-32:] for x in rddata]
+ # reverse data order to match the write order
+ rddata = [x for x in rddata[::-1]]
+ rddata = BinaryValue(''.join(rddata))
+ return_dict[k] = rddata
+ return return_dict
+
+async def write_mm_dict(mm: MMController, write_dict: dict) -> None:
+ # remove all read-only entries
+ d = {k: v for k, v in write_dict.items() if 'w' in v['access'].lower()}
+ for k, v in d.items():
+ # get data to write from dictionary
+ os = v['offset']
+ data_w = 32
+ n_bits = v['size'] * data_w
+ val = BinaryValue(v['value'], n_bits, False).binstr
+ wrdata = [BinaryValue(val[i:i+data_w]).integer for i in range(0, n_bits, data_w)]
+ # reverse data order to match the write order
+ wrdata = [x for x in wrdata[::-1]]
+ # write data
+ for i in range(v['size']):
+ await mm.write(v['offset'] + i, wrdata[i])
+
+def verify_mm_regs(actual_dict, exp_dict):
+ # assume exp_dict is a dictionary of dictionaries. actual_dict is a 1d dictionary.
+ # Only check register that are 'rw'
+ d = {k: v for k, v in exp_dict.items() if 'rw' == v['access'].lower()}
+ for k, v in d.items():
+ assert actual_dict[k].integer == v["value"], (
+ f'ERROR: Wrong value when reading back register, expected {k} = {v["value"]} but got {actual_dict[k]}')
+
+async def send_multi_dp_packet(dp_stream: DpStream, data, n):
+ for i in range(n):
+ await cocotb.start_soon(dp_stream.sosi_drv._driver_send(data))
+
+def carry_around_add(a, b):
+ c = a + b
+ return (c & 0xffff) + (c >> 16)
+
+def compute_ip_checksum(msg):
+ # assume msg is packet complete network packet (ETH + IP + etc...)
+ ip_header = msg[14:24] + msg[26:34] # IP header without checksum
+ s = 0
+ for i in range(0, len(ip_header), 2):
+ w = ip_header[i+1] + (ip_header[i] << 8)
+ s = carry_around_add(s, w)
+ return ~s & 0xffff
+
+def extract_header(data):
+ opcode = data[42] # assume BTH opcode is at byte 42
+ hdr_length = 54 # bth = write_middle or write_last
+ hdr = {
+ "eth_dst_mac": int.from_bytes(data[0:6], "big"),
+ "eth_src_mac": int.from_bytes(data[6:12], "big"),
+ "eth_type": int.from_bytes(data[12:14], "big"),
+ "ip_version": data[14] >> 4,
+ "ip_header_length": data[14] & 0x0F,
+ "ip_services": data[15],
+ "ip_total_length": int.from_bytes(data[16:18], "big"),
+ "ip_identification": int.from_bytes(data[18:20], "big"),
+ "ip_flags": data[20] >> 5,
+ "ip_fragment_offset": int.from_bytes(data[20:22], "big") & 0x1FFF,
+ "ip_time_to_live": data[22],
+ "ip_protocol": data[23],
+ "ip_header_checksum": int.from_bytes(data[24:26], "big"),
+ "ip_src_addr": int.from_bytes(data[26:30], "big"),
+ "ip_dst_addr": int.from_bytes(data[30:34], "big"),
+ "udp_src_port": int.from_bytes(data[34:36], "big"),
+ "udp_dst_port": int.from_bytes(data[36:38], "big"),
+ "udp_total_length": int.from_bytes(data[38:40], "big"),
+ "udp_checksum": int.from_bytes(data[40:42], "big"),
+ "bth_opcode": data[42],
+ "bth_se": data[43] >> 7,
+ "bth_m": (data[43] >> 6) & 0x01,
+ "bth_pad": (data[43] >> 4) & 0x03,
+ "bth_tver": data[43] & 0x0F,
+ "bth_partition_key": int.from_bytes(data[44:46], "big"),
+ "bth_fres": data[46] >> 7,
+ "bth_bres": (data[46] >> 6) & 0x01,
+ "bth_dest_qp": int.from_bytes(data[47:49], "big"),
+ "bth_ack_req": data[49] >> 7,
+ "bth_psn": int.from_bytes(data[50:54], "big")
+ }
+ # Set optional header fields and header length based on opcode
+ if opcode in [c_bth_first, c_bth_wo, c_bth_wo_imm]:
+ hdr_length += 16 # + RETH
+ hdr["reth_virtual_address"] = int.from_bytes(data[54:62], "big")
+ hdr["reth_r_key"] = int.from_bytes(data[62:66], "big")
+ hdr["reth_dma_length"] = int.from_bytes(data[66:70], "big")
+ if opcode == c_bth_wo_imm:
+ hdr_length += 4 # + immediate
+ hdr["immediate_data"] = int.from_bytes(data[70:74], "big")
+ if opcode == c_bth_last_imm:
+ hdr_length += 4 # + immediate
+ hdr["immediate_data"] = int.from_bytes(data[54:58], "big")
+ return hdr, hdr_length
+
+def verify_header(hdr, exp_hdr):
+ for k, v in hdr.items():
+ assert exp_hdr[k]["value"] == v, (
+ f'ERROR: Wrong header got {k} = {v} but expected {exp_hdr[k]["value"]}')
+
+@cocotb.test()
+async def tb_rdma_packetiser(dut):
+ """Try accessing the design. run with < run -a >"""
+ # Constants
+ c_bsn_init = 17 # some bsn as starting bsn
+ n_bytes = dut.c_nof_byte.value
+ c_data_w = dut.c_data_w.value
+ n_words = 120 # = 7680 bytes
+ c_block_len = n_words * n_bytes
+ c_nof_packets_in_msg = 5
+ c_dma_len = c_block_len * c_nof_packets_in_msg
+
+ # Packet header definition and config
+ hdr_dict = {
+ "eth_dst_mac": {"access": "RW", "size": 2, "offset": 44, "value": 0xCAFEBABE1996},
+ "eth_src_mac": {"access": "RW", "size": 2, "offset": 42, "value": 0x1DECAFC0FFEE},
+ "eth_type": {"access": "RW", "size": 1, "offset": 41, "value": 0x0800},
+
+ "ip_version": {"access": "RO", "size": 1, "offset": 40, "value": 4},
+ "ip_header_length": {"access": "RO", "size": 1, "offset": 39, "value": 5},
+ "ip_services": {"access": "RO", "size": 1, "offset": 38, "value": 0},
+ "ip_total_length": {"access": "RO", "size": 1, "offset": 37, "value": -1},
+ "ip_identification": {"access": "RO", "size": 1, "offset": 36, "value": 0},
+ "ip_flags": {"access": "RO", "size": 1, "offset": 35, "value": 2},
+ "ip_fragment_offset": {"access": "RO", "size": 1, "offset": 34, "value": 0},
+ "ip_time_to_live": {"access": "RO", "size": 1, "offset": 33, "value": 127},
+ "ip_protocol": {"access": "RO", "size": 1, "offset": 32, "value": 17},
+ "ip_header_checksum": {"access": "RO", "size": 1, "offset": 31, "value": -1},
+ "ip_src_addr": {"access": "RW", "size": 1, "offset": 30, "value": 0xFACE0FF},
+ "ip_dst_addr": {"access": "RW", "size": 1, "offset": 29, "value": 0x7DECADE},
+
+ "udp_src_port": {"access": "RW", "size": 1, "offset": 28, "value": 1234},
+ "udp_dst_port": {"access": "RW", "size": 1, "offset": 27, "value": 4791}, # 4791 is RoCEv2 port
+ "udp_total_length": {"access": "RO", "size": 1, "offset": 26, "value": -1},
+ "udp_checksum": {"access": "RO", "size": 1, "offset": 25, "value": 0},
+
+ "bth_opcode": {"access": "RO", "size": 1, "offset": 24, "value": -1},
+ "bth_se": {"access": "RW", "size": 1, "offset": 23, "value": 0},
+ "bth_m": {"access": "RW", "size": 1, "offset": 22, "value": 0},
+ "bth_pad": {"access": "RW", "size": 1, "offset": 21, "value": 0},
+ "bth_tver": {"access": "RW", "size": 1, "offset": 20, "value": 0},
+ "bth_partition_key": {"access": "RW", "size": 1, "offset": 19, "value": 65535},
+ "bth_fres": {"access": "RW", "size": 1, "offset": 18, "value": 0},
+ "bth_bres": {"access": "RW", "size": 1, "offset": 17, "value": 0},
+ "bth_reserved_a": {"access": "RO", "size": 1, "offset": 16, "value": 0},
+ "bth_dest_qp": {"access": "RW", "size": 1, "offset": 15, "value": 0},
+ "bth_ack_req": {"access": "RW", "size": 1, "offset": 14, "value": 0},
+ "bth_reserved_b": {"access": "RO", "size": 1, "offset": 13, "value": 0},
+ "bth_psn": {"access": "RO", "size": 1, "offset": 12, "value": -1},
+
+ "reth_virtual_address": {"access": "RO", "size": 2, "offset": 10, "value": -1},
+ "reth_r_key": {"access": "RW", "size": 1, "offset": 9, "value": 0},
+ "reth_dma_length": {"access": "RW", "size": 1, "offset": 8, "value": c_dma_len},
+
+ "immediate_data": {"access": "RW", "size": 1, "offset": 7, "value": 0xABADCAFE},
+
+ "config_reserved": {"access": "RO", "size": 1, "offset": 6, "value": 0},
+ "config_use_immediate": {"access": "RW", "size": 1, "offset": 5, "value": 1},
+ "config_use_msg_cnt_as_immediate": {"access": "RW", "size": 1, "offset": 4, "value": 0},
+ "config_nof_packets_in_msg": {"access": "RW", "size": 1, "offset": 3, "value": c_nof_packets_in_msg},
+ "config_nof_msg": {"access": "RW", "size": 1, "offset": 2, "value": 3},
+ "config_start_address": {"access": "RW", "size": 2, "offset": 0, "value": 1000000},
+ }
+
+ # Determine expected BTH opcodes for each packet in a message based on configuration.
+ exp_bth_opcodes = [c_bth_middle] * (c_nof_packets_in_msg - 2)
+ if c_nof_packets_in_msg > 1:
+ bth_last = c_bth_last_imm if hdr_dict["config_use_immediate"]["value"] == 1 else c_bth_last
+ exp_bth_opcodes = [c_bth_first] + exp_bth_opcodes + [bth_last]
+ else: # write only
+ exp_bth_opcodes = [c_bth_wo_imm] if hdr_dict["config_use_immediate"]["value"] == 1 else [c_bth_wo]
+
+ # simple counter value (0..255) to serve as input data for snk_in.
+ snk_in_data = b''.join([(i % 2**8).to_bytes(1, 'little') for i in range(n_words * n_bytes)])
+
+ # Create clocks
+ dpClock = Clock(dut.dp_clk, 5, units="ns")
+ mmClock = Clock(dut.mm_clk, 1, units="ns")
+
+ # DP streams
+ in_stream = DpStream(dut, 'snk_in', 'snk_out', dut.dp_clk, dut.dp_rst, c_data_w, c_bsn_init)
+ out_stream = DpStream(dut, 'src_out', 'src_in', dut.dp_clk, dut.dp_rst, c_data_w)
+
+ # MM busses
+ reg_hdr_dat = MMController(dut, 'reg_hdr_dat_copi', 'reg_hdr_dat_cipo', dut.mm_clk)
+
+ cocotb.start_soon(dpClock.start()) # run the dp clock "in the background"
+ cocotb.start_soon(mmClock.start()) # run the mm clock "in the background"
+ cocotb.start_soon(perform_rst(dut.dp_rst, dpClock, 7))
+ cocotb.start_soon(perform_rst(dut.mm_rst, mmClock, 7))
+
+ # set block_len
+ dut.block_len.value = c_block_len
+
+ # Stimuli
+ await Timer(dpClock.period * 10) # wait a bit for resets to occur
+ await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge"
+ await write_mm_dict(reg_hdr_dat, hdr_dict) # write MM stimuli
+ await Timer(dpClock.period * 10) # wait a bit for resets to occur
+ await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge"
+ mm_rd = await read_mm_dict(reg_hdr_dat, hdr_dict) # read back MM registers
+ verify_mm_regs(mm_rd, hdr_dict) # verify that the read (RW)registers are the same as what is written.
+ await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge"
+ cocotb.start_soon(send_multi_dp_packet(in_stream, snk_in_data, 10)) # Send DP packets
+
+ # Verify output packets
+ for i in range(10):
+ # Receive packet
+ packet = await out_stream.sosi_mon.wait_for_recv()
+
+ # Verify bth_opcode
+ exp_bth_opcode = exp_bth_opcodes[i % len(exp_bth_opcodes)]
+ bth_opcode = packet[42]
+ assert exp_bth_opcode == bth_opcode, (
+ f'ERROR: Unexpected BTH opcode value {bth_opcode}, expected {exp_bth_opcode}')
+
+ # Verify header
+ hdr, hdr_length = extract_header(packet)
+ dut._log.debug(f"header of length {hdr_length} = \n")
+ for k, v in hdr.items():
+ dut._log.debug(f'{k} = {hex(v)} \t(hex)\t\t{v} \t(dec)')
+
+ start_addr = hdr_dict["config_start_address"]["value"]
+ nof_msg = hdr_dict["config_nof_msg"]["value"]
+ msg_cnt = (i // c_nof_packets_in_msg)
+
+ exp_hdr = hdr_dict.copy()
+ exp_hdr["ip_total_length"]["value"] = len(packet) - 14 # subtract ETH header length
+ exp_hdr["ip_header_checksum"]["value"] = compute_ip_checksum(packet)
+ exp_hdr["udp_total_length"]["value"] = len(packet) - 20 - 14 # subtract ETH + IP header lengths
+ exp_hdr["bth_opcode"]["value"] = exp_bth_opcode
+ exp_hdr["bth_psn"]["value"] = c_bsn_init + i
+ exp_hdr["reth_virtual_address"]["value"] = start_addr + c_dma_len * (msg_cnt % nof_msg)
+ if hdr_dict["config_use_msg_cnt_as_immediate"]["value"] == 1:
+ exp_hdr["immediate_data"]["value"] = msg_cnt
+ verify_header(hdr, exp_hdr)
+
+ # Verify data
+ data = packet[hdr_length:-4]
+ assert snk_in_data == data, (
+ 'ERROR: Data in output packet is not identical to input packet.')
+ dut._log.debug("src_out data = \n%s", hexdump(data))
+
+ # Verify icrc
+ # calculation of icrc is done as in:
+ # https://github.com/secdev/scapy/blob/master/scapy/contrib/roce.py
+ # ICRC could be calculated using the scapy library (if installed) with the code below.
+ # from scapy.layers.l2 import Ether
+ # from scapy.contrib.roce import BTH # for compute_icrc function
+ # icrc = int.from_bytes(Ether(packet)['BTH'].compute_icrc(None), 'little')
+ icrc = int.from_bytes(packet[-4:], "big")
+ ones = (0xFF).to_bytes(1, 'little')
+ pseudo_packet = ([ones] * 8 + [packet[14:15]] +
+ [ones] + [packet[16:22]] + [ones] +
+ [packet[23:24]] + [ones] * 2 +
+ [packet[26:40]] + [ones] * 2 +
+ [packet[42:46]] + [ones] + [packet[47:-4]]
+ )
+ pseudo_packet = b''.join(pseudo_packet)
+ exp_icrc = crc32(pseudo_packet) & 0xffffffff
+ assert exp_icrc == icrc, (
+ f'ERROR: Wrong ICRC, expected = {hex(exp_icrc)}, actual = {hex(icrc)}')
diff --git a/applications/rdma_demo/libraries/rdma_packetiser/hdllib.cfg b/applications/rdma_demo/libraries/rdma_packetiser/hdllib.cfg
index 70a9f1437ef434bb1c44fe5f959ffb0cf9e57f7f..8de4a7e73f9eaf591a7f6b7bce173f33189bff6d 100644
--- a/applications/rdma_demo/libraries/rdma_packetiser/hdllib.cfg
+++ b/applications/rdma_demo/libraries/rdma_packetiser/hdllib.cfg
@@ -16,6 +16,8 @@ test_bench_files =
regression_test_vhdl =
tb/vhdl/tb_tb_rdma_packetiser_assemble_header.vhd
+cocotb_files =
+ cocotb/tb_rdma_packetiser.py # DUT is whatever comes after tb_ (so work.rdma_packetiser)
[modelsim_project_file]