Skip to content
Snippets Groups Projects
Commit eab1704f authored by Reinier van der Walle's avatar Reinier van der Walle
Browse files

Cleaned up and added comments

parent 10da10ce
No related branches found
No related tags found
1 merge request!382Resolve HPR-150
Pipeline #73218 passed
Stage: linting
Hide whitespace changes
Inline Side-by-side
applications/rdma_demo/libraries/rdma_packetiser/cocotb/dp_bus.py
+ 9
− 7
View file @ eab1704f
from typing import Iterable, Union, Optional from typing import Iterable, Union, Optional
import cocotb import cocotb
...@@ -9,7 +8,8 @@ from cocotb.result import TestError ...@@ -9,7 +8,8 @@ from cocotb.result import TestError
from cocotb_bus.drivers import ValidatedBusDriver from cocotb_bus.drivers import ValidatedBusDriver
from cocotb_bus.monitors import BusMonitor from cocotb_bus.monitors import BusMonitor
# Note: Copied from AvalonSTPkts in [1] and slightly adapted to use sosi/siso signals
# [1] - https://docs.cocotb.org/en/v1.5.1/_modules/cocotb_bus/drivers/avalon.html
class SosiDriver(ValidatedBusDriver): class SosiDriver(ValidatedBusDriver):
_optional_signals = ['valid', 'sop', 'eop', 'sync', 'bsn', 'data', 're', 'im', 'empty', 'channel', 'err'] _optional_signals = ['valid', 'sop', 'eop', 'sync', 'bsn', 'data', 're', 'im', 'empty', 'channel', 'err']
_signals = [] _signals = []
...@@ -19,7 +19,7 @@ class SosiDriver(ValidatedBusDriver): ...@@ -19,7 +19,7 @@ class SosiDriver(ValidatedBusDriver):
"maxChannel" : 1, "maxChannel" : 1,
"readyLatency" : 1 "readyLatency" : 1
} }
def __init__(self, dut, name, clk, data_w = None): def __init__(self, dut, name, clk, data_w = None, bsn_init = 0):
ValidatedBusDriver.__init__(self, dut, name, clk, bus_separator='.') ValidatedBusDriver.__init__(self, dut, name, clk, bus_separator='.')
self.config = self._default_config.copy() self.config = self._default_config.copy()
self.use_empty = True self.use_empty = True
...@@ -28,6 +28,7 @@ class SosiDriver(ValidatedBusDriver): ...@@ -28,6 +28,7 @@ class SosiDriver(ValidatedBusDriver):
self.data_w = len(self.bus.data) self.data_w = len(self.bus.data)
else: else:
self.data_w = data_w self.data_w = data_w
self.bsn = bsn_init
async def _wait_ready(self): async def _wait_ready(self):
"""Wait for a ready cycle on the bus before continuing. """Wait for a ready cycle on the bus before continuing.
...@@ -103,13 +104,15 @@ class SosiDriver(ValidatedBusDriver): ...@@ -103,13 +104,15 @@ class SosiDriver(ValidatedBusDriver):
if firstword: if firstword:
self.bus.sop.value = 1 self.bus.sop.value = 1
self.bus.bsn.value = self.bsn
self.bsn += 1
firstword = False firstword = False
else: else:
self.bus.sop.value = 0 self.bus.sop.value = 0
nbytes = min(len(string), bus_width) nbytes = min(len(string), bus_width)
data = string[:nbytes] data = string[:nbytes]
# set unused bits of dp_sosi.data field to 0 # set unused bits of dp_sosi.data field to 0
data = bytes(word.n_bits // 8 - bus_width) + data data = bytes(word.n_bits // 8 - bus_width) + data
word.buff = data word.buff = data
...@@ -345,10 +348,9 @@ class DpStream: ...@@ -345,10 +348,9 @@ class DpStream:
# It does work in the following way: # It does work in the following way:
# 1. Only use _optional_signals, no _signals. # 1. Only use _optional_signals, no _signals.
# 2. Use bus_separator='.' # 2. Use bus_separator='.'
def __init__(self, dut, sosi_name, siso_name, clk, rst, data_w = None): def __init__(self, dut, sosi_name, siso_name, clk, rst, data_w = None, bsn_init = 0):
self.sosi_drv = SosiDriver(dut, sosi_name, clk, data_w) self.sosi_drv = SosiDriver(dut, sosi_name, clk, data_w, bsn_init)
self.siso_drv = SisoDriver(dut, siso_name, clk) self.siso_drv = SisoDriver(dut, siso_name, clk)
self.sosi_mon = SosiMonitor(dut, sosi_name, clk, data_w, reset=rst) self.sosi_mon = SosiMonitor(dut, sosi_name, clk, data_w, reset=rst)
self.siso_mon = SisoMonitor(dut, siso_name, clk, reset=rst) self.siso_mon = SisoMonitor(dut, siso_name, clk, reset=rst)
cocotb.start_soon(self.siso_mon._monitor_recv()) cocotb.start_soon(self.siso_mon._monitor_recv())
applications/rdma_demo/libraries/rdma_packetiser/cocotb/mm_bus.py
+ 2
− 4
View file @ eab1704f
from typing import Iterable, Union, Optional
import cocotb.binary
from cocotb.log import SimLog from cocotb.log import SimLog
from cocotb.utils import hexdump
from cocotb.triggers import RisingEdge, ReadOnly from cocotb.triggers import RisingEdge, ReadOnly
from cocotb.binary import BinaryValue from cocotb.binary import BinaryValue
from cocotb.result import TestError from cocotb.result import TestError
from cocotb_bus.drivers import BusDriver from cocotb_bus.drivers import BusDriver
from cocotb.decorators import coroutine from cocotb.decorators import coroutine
# Note: Copied from AvalonMM in [1] and slightly adapted to use copi/cipo signals
# [1] - https://docs.cocotb.org/en/v1.5.1/_modules/cocotb_bus/drivers/avalon.html
class CopiDriver(BusDriver): class CopiDriver(BusDriver):
_signals = [] _signals = []
_optional_signals = ["rd", "wr", "wrdata", "address"] _optional_signals = ["rd", "wr", "wrdata", "address"]
......
applications/rdma_demo/libraries/rdma_packetiser/cocotb/tb_rdma_packetiser.py
+ 163
− 23
View file @ eab1704f
from zlib import crc32
import cocotb import cocotb
from cocotb.utils import hexdump from cocotb.utils import hexdump
from cocotb.triggers import FallingEdge, Timer, ReadOnly from cocotb.triggers import FallingEdge, Timer
from cocotb.clock import Clock from cocotb.clock import Clock
from cocotb.binary import BinaryValue from cocotb.binary import BinaryValue
from dp_bus import DpStream from dp_bus import DpStream
from mm_bus import MMController from mm_bus import MMController
# Global constants
c_bth_first = 0x26
c_bth_middle = 0x27
c_bth_last = 0x28
c_bth_last_imm = 0x29
c_bth_wo = 0x2A
c_bth_wo_imm = 0x2B
async def perform_rst(rst, clk, cycles): async def perform_rst(rst, clk, cycles):
rst.value = 1 rst.value = 1
...@@ -44,28 +52,103 @@ async def write_mm_dict(mm: MMController, write_dict: dict) -> None: ...@@ -44,28 +52,103 @@ async def write_mm_dict(mm: MMController, write_dict: dict) -> None:
# write data # write data
for i in range(v['size']): for i in range(v['size']):
await mm.write(v['offset'] + i, wrdata[i]) await mm.write(v['offset'] + i, wrdata[i])
def verify_mm_regs(actual_dict, exp_dict):
# assume exp_dict is a dictionary of dictionaries. actual_dict is a 1d dictionary.
# Only check register that are 'rw'
d = {k: v for k, v in exp_dict.items() if 'rw' == v['access'].lower()}
for k, v in d.items():
assert actual_dict[k].integer == v["value"], (
f'ERROR: Wrong value when reading back register, expected {k} = {v["value"]} but got {actual_dict[k]}')
async def send_multi_dp_packet(dp_stream: DpStream, data, n): async def send_multi_dp_packet(dp_stream: DpStream, data, n):
for i in range(n): for i in range(n):
await cocotb.start_soon(dp_stream.sosi_drv._driver_send(data)) await cocotb.start_soon(dp_stream.sosi_drv._driver_send(data))
def carry_around_add(a, b):
c = a + b
return (c & 0xffff) + (c >> 16)
def compute_ip_checksum(msg):
# assume msg is packet complete network packet (ETH + IP + etc...)
ip_header = msg[14:24] + msg[26:34] # IP header without checksum
s = 0
for i in range(0, len(ip_header), 2):
w = ip_header[i+1] + (ip_header[i] << 8)
s = carry_around_add(s, w)
return ~s & 0xffff
def extract_header(data):
opcode = data[42] # assume BTH opcode is at byte 42
hdr_length = 54 # bth = write_middle or write_last
hdr = {
"eth_dst_mac": int.from_bytes(data[0:6], "big"),
"eth_src_mac": int.from_bytes(data[6:12], "big"),
"eth_type": int.from_bytes(data[12:14], "big"),
"ip_version": data[14] >> 4,
"ip_header_length": data[14] & 0x0F,
"ip_services": data[15],
"ip_total_length": int.from_bytes(data[16:18], "big"),
"ip_identification": int.from_bytes(data[18:20], "big"),
"ip_flags": data[20] >> 5,
"ip_fragment_offset": int.from_bytes(data[20:22], "big") & 0x1FFF,
"ip_time_to_live": data[22],
"ip_protocol": data[23],
"ip_header_checksum": int.from_bytes(data[24:26], "big"),
"ip_src_addr": int.from_bytes(data[26:30], "big"),
"ip_dst_addr": int.from_bytes(data[30:34], "big"),
"udp_src_port": int.from_bytes(data[34:36], "big"),
"udp_dst_port": int.from_bytes(data[36:38], "big"),
"udp_total_length": int.from_bytes(data[38:40], "big"),
"udp_checksum": int.from_bytes(data[40:42], "big"),
"bth_opcode": data[42],
"bth_se": data[43] >> 7,
"bth_m": (data[43] >> 6) & 0x01,
"bth_pad": (data[43] >> 4) & 0x03,
"bth_tver": data[43] & 0x0F,
"bth_partition_key": int.from_bytes(data[44:46], "big"),
"bth_fres": data[46] >> 7,
"bth_bres": (data[46] >> 6) & 0x01,
"bth_dest_qp": int.from_bytes(data[47:49], "big"),
"bth_ack_req": data[49] >> 7,
"bth_psn": int.from_bytes(data[50:54], "big")
}
# Set optional header fields and header length based on opcode
if opcode in [c_bth_first, c_bth_wo, c_bth_wo_imm]:
hdr_length += 16 # + RETH
hdr["reth_virtual_address"] = int.from_bytes(data[54:62], "big")
hdr["reth_r_key"] = int.from_bytes(data[62:66], "big")
hdr["reth_dma_length"] = int.from_bytes(data[66:70], "big")
if opcode == c_bth_wo_imm:
hdr_length += 4 # + immediate
hdr["immediate_data"] = int.from_bytes(data[70:74], "big")
if opcode == c_bth_last_imm:
hdr_length += 4 # + immediate
hdr["immediate_data"] = int.from_bytes(data[54:58], "big")
return hdr, hdr_length
def verify_header(hdr, exp_hdr):
for k, v in hdr.items():
assert exp_hdr[k]["value"] == v, (
f'ERROR: Wrong header got {k} = {v} but expected {exp_hdr[k]["value"]}')
@cocotb.test() @cocotb.test()
async def tb_rdma_packetiser(dut): async def tb_rdma_packetiser(dut):
"""Try accessing the design. run with < run -a >""" """Try accessing the design. run with < run -a >"""
# Constants
c_bsn_init = 17 # some bsn as starting bsn
n_bytes = dut.c_nof_byte.value n_bytes = dut.c_nof_byte.value
c_data_w = dut.c_data_w.value c_data_w = dut.c_data_w.value
n_words = 120 # = 7680 bytes n_words = 120 # = 7680 bytes
c_block_len = n_words * n_bytes c_block_len = n_words * n_bytes
c_nof_packets_in_msg = 5 c_nof_packets_in_msg = 5
c_dma_len = c_block_len * c_nof_packets_in_msg c_dma_len = c_block_len * c_nof_packets_in_msg
n_hdr_regs = 46
# Packet header definition and config
hdr_dict = { hdr_dict = {
"eth_dst_mac": {"access": "RW", "size": 2, "offset": 44, "value": 0xCAFEBABE1996}, "eth_dst_mac": {"access": "RW", "size": 2, "offset": 44, "value": 0xCAFEBABE1996},
"eth_src_mac": {"access": "RW", "size": 2, "offset": 42, "value": 0x1DECAFC0FFEE}, "eth_src_mac": {"access": "RW", "size": 2, "offset": 42, "value": 0x1DECAFC0FFEE},
"eth_type": {"access": "RO", "size": 1, "offset": 41, "value": 0x0800}, "eth_type": {"access": "RW", "size": 1, "offset": 41, "value": 0x0800},
"ip_version": {"access": "RO", "size": 1, "offset": 40, "value": 4}, "ip_version": {"access": "RO", "size": 1, "offset": 40, "value": 4},
"ip_header_length": {"access": "RO", "size": 1, "offset": 39, "value": 5}, "ip_header_length": {"access": "RO", "size": 1, "offset": 39, "value": 5},
...@@ -81,7 +164,7 @@ async def tb_rdma_packetiser(dut): ...@@ -81,7 +164,7 @@ async def tb_rdma_packetiser(dut):
"ip_dst_addr": {"access": "RW", "size": 1, "offset": 29, "value": 0x7DECADE}, "ip_dst_addr": {"access": "RW", "size": 1, "offset": 29, "value": 0x7DECADE},
"udp_src_port": {"access": "RW", "size": 1, "offset": 28, "value": 1234}, "udp_src_port": {"access": "RW", "size": 1, "offset": 28, "value": 1234},
"udp_dst_port": {"access": "RW", "size": 1, "offset": 27, "value": 4321}, "udp_dst_port": {"access": "RW", "size": 1, "offset": 27, "value": 4791}, # 4791 is RoCEv2 port
"udp_total_length": {"access": "RO", "size": 1, "offset": 26, "value": -1}, "udp_total_length": {"access": "RO", "size": 1, "offset": 26, "value": -1},
"udp_checksum": {"access": "RO", "size": 1, "offset": 25, "value": 0}, "udp_checksum": {"access": "RO", "size": 1, "offset": 25, "value": 0},
...@@ -112,9 +195,16 @@ async def tb_rdma_packetiser(dut): ...@@ -112,9 +195,16 @@ async def tb_rdma_packetiser(dut):
"config_nof_msg": {"access": "RW", "size": 1, "offset": 2, "value": 3}, "config_nof_msg": {"access": "RW", "size": 1, "offset": 2, "value": 3},
"config_start_address": {"access": "RW", "size": 2, "offset": 0, "value": 1000000}, "config_start_address": {"access": "RW", "size": 2, "offset": 0, "value": 1000000},
} }
# simple counter value per byte # Determine expected BTH opcodes for each packet in a message based on configuration.
exp_bth_opcodes = [c_bth_middle] * (c_nof_packets_in_msg - 2)
if c_nof_packets_in_msg > 1:
bth_last = c_bth_last_imm if hdr_dict["config_use_immediate"]["value"] == 1 else c_bth_last
exp_bth_opcodes = [c_bth_first] + exp_bth_opcodes + [bth_last]
else: # write only
exp_bth_opcodes = [c_bth_wo_imm] if hdr_dict["config_use_immediate"]["value"] == 1 else [c_bth_wo]
# simple counter value (0..255) to serve as input data for snk_in.
snk_in_data = b''.join([(i % 2**8).to_bytes(1, 'little') for i in range(n_words * n_bytes)]) snk_in_data = b''.join([(i % 2**8).to_bytes(1, 'little') for i in range(n_words * n_bytes)])
# Create clocks # Create clocks
...@@ -122,7 +212,7 @@ async def tb_rdma_packetiser(dut): ...@@ -122,7 +212,7 @@ async def tb_rdma_packetiser(dut):
mmClock = Clock(dut.mm_clk, 1, units="ns") mmClock = Clock(dut.mm_clk, 1, units="ns")
# DP streams # DP streams
in_stream = DpStream(dut, 'snk_in', 'snk_out', dut.dp_clk, dut.dp_rst, c_data_w) in_stream = DpStream(dut, 'snk_in', 'snk_out', dut.dp_clk, dut.dp_rst, c_data_w, c_bsn_init)
out_stream = DpStream(dut, 'src_out', 'src_in', dut.dp_clk, dut.dp_rst, c_data_w) out_stream = DpStream(dut, 'src_out', 'src_in', dut.dp_clk, dut.dp_rst, c_data_w)
# MM busses # MM busses
...@@ -133,24 +223,74 @@ async def tb_rdma_packetiser(dut): ...@@ -133,24 +223,74 @@ async def tb_rdma_packetiser(dut):
cocotb.start_soon(perform_rst(dut.dp_rst, dpClock, 7)) cocotb.start_soon(perform_rst(dut.dp_rst, dpClock, 7))
cocotb.start_soon(perform_rst(dut.mm_rst, mmClock, 7)) cocotb.start_soon(perform_rst(dut.mm_rst, mmClock, 7))
# set block_len
dut.block_len.value = c_block_len
# Stimuli
await Timer(dpClock.period * 10) # wait a bit for resets to occur await Timer(dpClock.period * 10) # wait a bit for resets to occur
await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge" await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge"
await write_mm_dict(reg_hdr_dat, hdr_dict) await write_mm_dict(reg_hdr_dat, hdr_dict) # write MM stimuli
#print(snk_in_data)
await Timer(dpClock.period * 10) # wait a bit for resets to occur await Timer(dpClock.period * 10) # wait a bit for resets to occur
await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge" await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge"
mm_rd = await read_mm_dict(reg_hdr_dat, hdr_dict) mm_rd = await read_mm_dict(reg_hdr_dat, hdr_dict) # read back MM registers
for k, v in mm_rd.items(): verify_mm_regs(mm_rd, hdr_dict) # verify that the read (RW)registers are the same as what is written.
print(f'{k} = {hex(v.integer)} in hex, {v.integer} in dec')
await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge" await FallingEdge(dut.dp_clk) # wait for falling edge/"negedge"
cocotb.start_soon(send_multi_dp_packet(in_stream, snk_in_data, 10)) cocotb.start_soon(send_multi_dp_packet(in_stream, snk_in_data, 10)) # Send DP packets
#cocotb.start_soon(in_stream.sosi_drv._driver_send(snk_in_data))
# wait for packet to arrive on src_out # Verify output packets
for i in range(10): for i in range(10):
data = await out_stream.sosi_mon.wait_for_recv() # Receive packet
dut._log.info("src_out data = \n%s", hexdump(data)) packet = await out_stream.sosi_mon.wait_for_recv()
# Verify bth_opcode
exp_bth_opcode = exp_bth_opcodes[i % len(exp_bth_opcodes)]
bth_opcode = packet[42]
assert exp_bth_opcode == bth_opcode, (
f'ERROR: Unexpected BTH opcode value {bth_opcode}, expected {exp_bth_opcode}')
# Verify header
hdr, hdr_length = extract_header(packet)
dut._log.debug(f"header of length {hdr_length} = \n")
for k, v in hdr.items():
dut._log.debug(f'{k} = {hex(v)} \t(hex)\t\t{v} \t(dec)')
start_addr = hdr_dict["config_start_address"]["value"]
nof_msg = hdr_dict["config_nof_msg"]["value"]
msg_cnt = (i // c_nof_packets_in_msg)
exp_hdr = hdr_dict.copy()
exp_hdr["ip_total_length"]["value"] = len(packet) - 14 # subtract ETH header length
exp_hdr["ip_header_checksum"]["value"] = compute_ip_checksum(packet)
exp_hdr["udp_total_length"]["value"] = len(packet) - 20 - 14 # subtract ETH + IP header lengths
exp_hdr["bth_opcode"]["value"] = exp_bth_opcode
exp_hdr["bth_psn"]["value"] = c_bsn_init + i
exp_hdr["reth_virtual_address"]["value"] = start_addr + c_dma_len * (msg_cnt % nof_msg)
if hdr_dict["config_use_msg_cnt_as_immediate"]["value"] == 1:
exp_hdr["immediate_data"]["value"] = msg_cnt
verify_header(hdr, exp_hdr)
# Verify data
data = packet[hdr_length:-4]
assert snk_in_data == data, (
'ERROR: Data in output packet is not identical to input packet.')
dut._log.debug("src_out data = \n%s", hexdump(data))
# Verify icrc
# calculation of icrc is done as in:
# https://github.com/secdev/scapy/blob/master/scapy/contrib/roce.py
# ICRC could be calculated using the scapy library (if installed) with the code below.
# from scapy.layers.l2 import Ether
# from scapy.contrib.roce import BTH # for compute_icrc function
# icrc = int.from_bytes(Ether(packet)['BTH'].compute_icrc(None), 'little')
icrc = int.from_bytes(packet[-4:], "big")
ones = (0xFF).to_bytes(1, 'little')
pseudo_packet = ([ones] * 8 + [packet[14:15]] +
[ones] + [packet[16:22]] + [ones] +
[packet[23:24]] + [ones] * 2 +
[packet[26:40]] + [ones] * 2 +
[packet[42:46]] + [ones] + [packet[47:-4]]
)
pseudo_packet = b''.join(pseudo_packet)
exp_icrc = crc32(pseudo_packet) & 0xffffffff
assert exp_icrc == icrc, (
f'ERROR: Wrong ICRC, expected = {hex(exp_icrc)}, actual = {hex(icrc)}')
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment