diff --git a/RTCP/Cobalt/OutputProc/scripts/CMakeLists.txt b/RTCP/Cobalt/OutputProc/scripts/CMakeLists.txt
index 2e8ef0b32738f6b090a511bed86ff7d345c80d55..1d17a3feadaae78d5a33184aedb7a2da6aff0b2c 100644
--- a/RTCP/Cobalt/OutputProc/scripts/CMakeLists.txt
+++ b/RTCP/Cobalt/OutputProc/scripts/CMakeLists.txt
@@ -3,3 +3,6 @@
if(USE_THREADS)
lofar_add_sbin_program(udp-copy udp-copy.c common.c)
endif(USE_THREADS)
+
+add_executable(tbb-crc-test tbb-crc-test.cc)
+add_executable(tbb-dumpframes tbb-dumpframes.cc)
diff --git a/RTCP/Cobalt/OutputProc/scripts/tbb-crc-test.cc b/RTCP/Cobalt/OutputProc/scripts/tbb-crc-test.cc
new file mode 100644
index 0000000000000000000000000000000000000000..f1d6b730619f901143523d03c3a7179b59411342
--- /dev/null
+++ b/RTCP/Cobalt/OutputProc/scripts/tbb-crc-test.cc
@@ -0,0 +1,296 @@
+/* tbb-crc-test.cpp
+ * Author: Alexander S. van Amesfoort, ASTRON
+ * with code based on Python crc routines received from Gijs Schoonderbeek, ASTRON
+ * Last-modified: July 2012
+ * build: g++ -Wall -o tbb-crc-test tbb-crc-test.cc
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <endian.h>
+#if __BYTE_ORDER != __BIG_ENDIAN && __BYTE_ORDER != __LITTLE_ENDIAN
+#error Byte order is neither big endian nor little endian: not supported
+#endif
+#include <byteswap.h>
+
+#include <iostream>
+#include <fstream>
+
+#include <boost/crc.hpp>
+
+using namespace std;
+
+struct TBB_Header {
+ uint8_t stationId;
+ uint8_t rspID;
+ uint8_t rcuID;
+ uint8_t sampleFreq;
+
+ uint32_t seqNr;
+ uint32_t time;
+
+ union {
+ uint32_t sampleNr;
+ uint32_t bandsliceNr;
+ };
+
+ uint16_t nOfSamplesPerFrame;
+ uint16_t nOfFreqBands;
+
+ uint8_t bandSel[64];
+
+ uint16_t spare;
+ uint16_t crc16;
+};
+
+#define MAX_TRANSIENT_NSAMPLES 1298 // based on frames stored by TBB and (un)packing
+#define DEFAULT_TRANSIENT_NSAMPLES 1024 // int16_t
+#define DEFAULT_SPECTRAL_NSAMPLES 487 // complex int16_t
+struct TBB_Payload {
+ // For transient data, we typically receive 1024 samples per frame.
+ // uint32_t crc comes right after, so cannot easily declare it after data[], hence + 2.
+ int16_t data[MAX_TRANSIENT_NSAMPLES + 2];
+};
+
+struct TBB_Frame {
+ TBB_Header header;
+ TBB_Payload payload;
+};
+
+
+// Same truncated polynomials as standard crc16 and crc32, but with initial_remainder=0, final_xor_value=0, reflected_input=false, reflected_remainder_output=false.
+// The boost::crc_optimal<> declarations precompute lookup tables, so do not declare inside the checking routine.
+static boost::crc_optimal<16, 0x8005 /*, 0, 0, false, false*/> crc16tbbgen; // instead of crc_16_type
+//boost::crc_basic<16> crc16tbbgen(0x8005/*, 0, 0, false, false*/); // non-opt variant
+
+static boost::crc_optimal<32, 0x04C11DB7/*, 0, 0, false, false*/> crc32tbbgen; // instead of crc_32_type
+//boost::crc_basic<32> crc32gen(0x04C11DB7/*, 0, 0, false, false*/); // non-opt variant
+
+/*
+ * Assumes that the seqNr field in the TBB_Frame at buf has been zeroed.
+ * Takes a ptr to a complete header. (Drop too small frames earlier.)
+ */
+static bool crc16tbb_boost(const TBB_Header* header) {
+ crc16tbbgen.reset();
+
+ /*
+ * The header checksum is done like the data, i.e. on 16 bit little endian blocks at a time.
+ * As with the data, both big and little endian CPUs need to byte swap.
+ */
+ const int16_t* ptr = reinterpret_cast<const int16_t*>(header);
+ size_t i;
+ for (i = 0; i < (sizeof(*header) - sizeof(header->crc16)) / sizeof(int16_t); i++) {
+ int16_t val = __bswap_16(ptr[i]);
+ crc16tbbgen.process_bytes(&val, sizeof(int16_t));
+ }
+
+ // Byte swap the little endian checksum on big endian only.
+ // It is also possible to process header->crc16 and see if checksum() equals 0.
+ uint16_t crc16val = header->crc16;
+#if __BYTE_ORDER == __BIG_ENDIAN
+ crc16val = __bswap_16(crc16val);
+#endif
+ return crc16tbbgen.checksum() == crc16val;
+}
+
+/*
+ * Note: The nsamples arg is without the space taken by the crc32 in payload. (Drop too small frames earlier.)
+ */
+static bool crc32tbb_boost(const TBB_Payload* payload, size_t nsamples) {
+ crc32tbbgen.reset();
+
+ /*
+ * Both little and big endian CPUs need to byte swap, because the data always arrives
+ * in little and the boost routines treat it as uint8_t[] (big).
+ */
+ const int16_t* ptr = reinterpret_cast<const int16_t*>(payload->data);
+ size_t i;
+ for (i = 0; i < nsamples; i++) {
+ int16_t val = __bswap_16(ptr[i]);
+ crc32tbbgen.process_bytes(&val, sizeof(int16_t));
+ }
+
+ // Byte swap the little endian checksum on big endian only.
+ // It is also possible to process crc32val and see if checksum() equals 0.
+ uint32_t crc32val = *reinterpret_cast<const uint32_t*>(&ptr[nsamples]);
+#if __BYTE_ORDER == __BIG_ENDIAN
+ crc32val = __bswap_32(crc32val);
+#endif
+ return crc32tbbgen.checksum() == crc32val;
+}
+
+#if __BYTE_ORDER != __LITTLE_ENDIAN
+#warning Original crc routines were only developed for little endian. Skipping those.
+#else
+/*
+ * This code is translated from the Python ref/test code received from Gijs Schoonderbeek.
+ * It assumes that the seqNr field (buf[1]) has been zeroed.
+ * Do not call this function with len < 1; reject too small headers earlier.
+ */
+static uint16_t crc16tbb(const uint16_t* buf, size_t len) {
+ uint16_t CRC = 0;
+ const uint32_t CRC_poly = 0x18005;
+ const uint16_t bits = 16;
+ uint32_t data = 0;
+ const uint32_t CRCDIV = (CRC_poly & 0x7fffffff) << 15;
+
+ data = (buf[0] & 0x7fffffff) << 16;
+ for (uint32_t i = 1; i < len; i++) {
+ data += buf[i];
+ for (uint16_t j = 0; j < bits; j++) {
+ if ((data & 0x80000000) != 0) {
+ data = data ^ CRCDIV;
+ }
+ data = data & 0x7fffffff;
+ data = data << 1;
+ }
+ }
+ CRC = data >> 16;
+ return CRC;
+}
+
+/*
+ * This code is translated from the Python ref/test code received from Gijs Schoonderbeek.
+ * It computes a 32 bit result, 16 bits at a time.
+ * Do not call this function with len < 2; reject too small payloads earlier.
+ */
+static uint32_t crc32tbb(const uint16_t* buf, size_t len) {
+ uint32_t CRC = 0;
+ const uint64_t CRC_poly = 0x104C11DB7ULL;
+ const uint16_t bits = 16;
+ uint64_t data = 0;
+ const uint64_t CRCDIV = (CRC_poly & 0x7fffffffffffULL) << 15;
+
+ data = buf[0];
+ data = data & 0x7fffffffffffULL;
+ data = data << 16;
+ data = data + buf[1];
+ data = data & 0x7fffffffffffULL;
+ data = data << 16;
+ uint32_t i = 2;
+ for ( ; i < len-2; i++) {
+ data = data + buf[i];
+ for (uint32_t j = 0; j < bits; j++) {
+ if (data & 0x800000000000ULL) {
+ data = data ^ CRCDIV;
+ }
+ data = data & 0x7fffffffffffULL;
+ data = data << 1;
+ }
+ }
+
+ // Do the 32 bit checksum separately.
+ // Process the two 16 bit halves in reverse order, but keep the i < len cond.
+ for (buf += 1; i < len; i++, buf -= 2) {
+ data = data + buf[i];
+ for (uint32_t j = 0; j < bits; j++) {
+ if (data & 0x800000000000ULL) {
+ data = data ^ CRCDIV;
+ }
+ data = data & 0x7fffffffffffULL;
+ data = data << 1;
+ }
+ }
+
+ CRC = (uint32_t)(data >> 16);
+ return CRC;
+}
+#endif
+
+static int verify_crc(TBB_Frame& frame, size_t frameSize) {
+ int err = 0;
+
+ // Zero sequence number field before verification.
+ // It is set by TBB after the checksum has been computed. We do not need it later.
+ frame.header.seqNr = 0;
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ uint16_t headercrc = crc16tbb( reinterpret_cast<uint16_t*>(&frame.header), sizeof(TBB_Header) / sizeof(uint16_t) );
+ if (headercrc != 0) {
+ cerr << "crc16tbb(): Incorrect header crc: " << hex << headercrc << endl;
+ err = 1;
+ }
+#endif
+ if (!crc16tbb_boost(&frame.header)) {
+ cerr << "crc16tbb_boost(): Incorrect header crc" << endl;
+ err = 1;
+ }
+
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ uint32_t payloadcrc = crc32tbb( reinterpret_cast<uint16_t*>(frame.payload.data), ( frameSize - sizeof(TBB_Header) ) / sizeof(uint16_t) );
+ if (payloadcrc != 0) {
+ cerr << "crc32tbb(): Incorrect payload crc: " << hex << payloadcrc << endl;
+ err = 1;
+ }
+#endif
+ if (!crc32tbb_boost( &frame.payload, ( frameSize - sizeof(TBB_Header) - sizeof(uint32_t) ) / sizeof(int16_t) )) {
+ cerr << "crc32tbb_boost(): Incorrect payload crc" << endl;
+ err = 1;
+
+#if 0 // this guessing doesn't work: the wrong crc32 is different every time, even on the same data
+ TBB_Payload p;
+ unsigned i;
+ for (i = 0; i < 487; i++) {
+ memcpy(&p, &frame.payload, i * 2 * sizeof(int16_t)); // data
+ memcpy((char*)&p + i * 2 * sizeof(int16_t), (char*)(&frame.payload.data[2*487 + 2]) - sizeof(uint32_t), sizeof(uint32_t)); // crc32
+ if (crc32tbb_boost(&p, 2 * i)) {
+ cerr << "found it: i=" << i << endl;
+ break;
+ } else {
+ cerr << "doesn't work either: " << i << endl;
+ }
+ }
+#endif
+
+ }
+
+ return err;
+}
+
+int main(int argc, char* argv[]) {
+ if (argc < 2) {
+ cout << "Usage: " << argv[0] << " rawtbbframes.dat" << endl;
+ return 1;
+ }
+
+ bool transient;
+ ifstream iftype(argv[1], ios_base::binary);
+ if (!iftype) {
+ cerr << "Failed to open file " << argv[1] << endl;
+ return 1;
+ }
+ TBB_Header header;
+ iftype.read(reinterpret_cast<char*>(&header), sizeof header);
+ if (!iftype) {
+ cerr << "Failed to read first frame to determine transient or spectral mode" << endl;
+ return 1;
+ }
+ iftype.close();
+ transient = header.nOfFreqBands == 0;
+
+
+ ifstream ifs(argv[1], ios_base::binary);
+ if (!ifs) {
+ cerr << "Failed to open file " << argv[1] << endl;
+ return 1;
+ }
+
+ int err = 0;
+
+ TBB_Frame frame;
+ size_t frameSize;
+ if (transient) {
+ frameSize = sizeof(TBB_Header) + DEFAULT_TRANSIENT_NSAMPLES * sizeof(int16_t) + sizeof(uint32_t);
+ } else { // spectral
+ frameSize = sizeof(TBB_Header) + DEFAULT_SPECTRAL_NSAMPLES * 2 * sizeof(int16_t) + sizeof(uint32_t);
+ }
+
+ while (ifs.read(reinterpret_cast<char*>(&frame), frameSize)) {
+ err |= verify_crc(frame, frameSize);
+ }
+
+ ifs.close();
+ return err;
+}
+
diff --git a/RTCP/Cobalt/OutputProc/scripts/tbb-dumpframes.cc b/RTCP/Cobalt/OutputProc/scripts/tbb-dumpframes.cc
new file mode 100644
index 0000000000000000000000000000000000000000..7a4404353396c15f9788915dc0276da06b33f96e
--- /dev/null
+++ b/RTCP/Cobalt/OutputProc/scripts/tbb-dumpframes.cc
@@ -0,0 +1,240 @@
+/* tbb-dumpframes.cc
+ * Author: Alexander S. van Amesfoort, ASTRON
+ * Last-modified: Jun 2013
+ * build: g++ -Wall -o tbb-dumpframes tbb-dumpframes.cc
+ */
+
+#include <stdint.h>
+#include <cstdlib>
+#include <cstring>
+
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+using namespace std;
+
+struct TBB_Header {
+ uint8_t stationID; // Data source station identifier
+ uint8_t rspID; // Data source RSP board identifier
+ uint8_t rcuID; // Data source RCU board identifier
+ uint8_t sampleFreq; // Sample frequency in MHz of the RCU boards
+
+ uint32_t seqNr; // Used internally by TBB. Set to 0 by RSP (but written again before we receive it)
+ uint32_t time; // Time instance in seconds of the first sample in payload
+ // The time field is relative, but if used as UNIX time, uint32_t will wrap at 06:28:15 UTC on 07 Feb 2106 (int32_t wraps at 03:14:08 UTC on 19 Jan 2038).
+
+ union {
+ // In transient mode indicates sample number of the first payload sample in current seconds interval.
+ uint32_t sampleNr;
+
+ // In spectral mode indicates frequency band and slice (transform block of 1024 samples) of first payload sample.
+ uint32_t bandSliceNr; // bandNr[9:0] and sliceNr[31:10].
+ // Avoid bit fields, (portable) compilation support is messy. Instead use mask and shift to decode.
+#define TBB_BAND_NR_MASK ((1 << 10) - 1)
+#define TBB_SLICE_NR_SHIFT 10
+ };
+
+ uint16_t nOfSamplesPerFrame; // Total number of samples in the frame payload
+ uint16_t nOfFreqBands; // Number of frequency bands for each spectrum in spectral mode. Is set to 0 for transient mode.
+
+ uint8_t bandSel[64]; // Each bit in the band selector field indicates whether the band with the bit index is present in the spectrum or not.
+
+ uint16_t spare; // For future use. Set to 0.
+ uint16_t crc16; // CRC16 over frame header, with seqNr set to 0.
+};
+
+void timeToStr(time_t t, char* out, size_t out_sz) {
+ struct tm *tm = gmtime(&t);
+ // Format: Mo, 15-06-2009 20:20:00
+ strftime(out, out_sz, "%a, %d-%m-%Y %H:%M:%S", tm);
+}
+
+void printHeader(const TBB_Header& h) {
+ cout << "Station ID: " << (uint32_t)h.stationID << endl;
+ cout << "RSP ID: " << (uint32_t)h.rspID << endl;
+ cout << "RCU ID: " << (uint32_t)h.rcuID << endl;
+ cout << "Sample Freq: " << (uint32_t)h.sampleFreq << endl;
+ cout << "Seq Nr: " << h.seqNr << endl;
+ char buf[32];
+ timeToStr(h.time, buf, 32);
+ cout << "Time: " << h.time << " (dd-mm-yyyy: " << buf << " UTC)" << endl;
+ bool transient = h.nOfFreqBands == 0;
+ if (transient) {
+ cout << "Transient" << endl;
+ cout << "Sample Nr: " << h.sampleNr << endl;
+ } else {
+ cout << "Spectral" << endl;
+ cout << "Band Nr: " << (h.bandSliceNr & TBB_BAND_NR_MASK) << endl;
+ cout << "Slice Nr: " << (h.bandSliceNr >> TBB_SLICE_NR_SHIFT) << endl;
+ }
+ cout << "NSamples/fr: " << h.nOfSamplesPerFrame << endl;
+ if (!transient) {
+ cout << "NFreq Bands: " << h.nOfFreqBands << endl;
+
+ bool anyBandsPresent = false;
+ cout << "Band(s) present(?): ";
+ for (unsigned i = 0; i < 64; ++i) {
+ for (unsigned j = 8; j > 0; ) {
+ --j;
+ if (h.bandSel[i] & (1 << j)) {
+ cout << 8 * i + (8-1-j) << " ";
+ anyBandsPresent = true;
+ }
+ }
+ }
+ if (!anyBandsPresent) {
+ cout << "Warning: Spectral data, but no band present!" << endl;
+ } else {
+ cout << endl;
+ }
+ }
+
+ cout << "Spare (0): " << h.spare << endl;
+ cout << "crc16: " << h.crc16 << endl;
+}
+
+void printPayload(const int16_t* payload, size_t payload_len) {
+ size_t data_len = payload_len - sizeof(uint32_t) / sizeof(int16_t); // - crc32
+ unsigned i;
+
+ if (data_len == 1024) { // transient has 1024 samples + crc32
+ for (i = 0; i < data_len; i++) {
+ cout << payload[i] << " ";
+ }
+ } else { // spectral has up to 487 complex samples + crc32
+ for (i = 0; i < data_len; i += 2) {
+ cout << '(' << payload[i] << ' ' << payload[i+1] << ") "; // assumes data_len is even
+ }
+ }
+ cout << endl;
+
+ cout << "crc32: " << reinterpret_cast<uint32_t*>(payload[i]) << endl;
+}
+
+void printFakeInput() {
+ TBB_Header hdr0;
+
+ hdr0.stationID = 1;
+ hdr0.rspID = 2;
+ hdr0.rcuID = 3;
+ hdr0.sampleFreq = 200;
+ hdr0.seqNr = 10000;
+ hdr0.time = 1380240059;
+ hdr0.bandSliceNr = (17 << 10) | 11; // sliceNr=17; bandNr is 11
+ hdr0.nOfSamplesPerFrame = 487;
+ hdr0.nOfFreqBands = 487/8 * 7 + 7; // 427, as set in the sb bitmap below
+
+ // subband bitmap
+ // I'm not 100% if the bits are populated from most to least significant...
+ int i;
+ for (i = 0; i < 487/8; i++)
+ hdr0.bandSel[i] = 0x7f;
+ hdr0.bandSel[i++] = 0xfe; // remaining 7 bits to cover all 487 meaningful bits
+ for ( ; i < 64; i++)
+ hdr0.bandSel[i] = 0;
+
+ hdr0.spare = 0;
+ hdr0.crc16 = 1;
+
+ printHeader(hdr0);
+}
+
+int main(int argc, char* argv[]) {
+ bool printData = false;
+ bool fakeInput = false;
+ const char* filename = "/dev/stdin";
+ int nprinted = 8;
+
+ cout << "Usage: " << argv[0] << " [-d] [-t] [data/tbbdata.raw] [nframes]" << endl;
+
+ int argi = 1;
+ if (argc > argi) {
+ if (strcmp(argv[argi], "-d") == 0) {
+ printData = true;
+ argi += 1;
+ }
+
+ if (strcmp(argv[argi], "-t") == 0) {
+ fakeInput = true;
+ argi += 1;
+ }
+
+ if (argc > argi) {
+ filename = argv[argi];
+ argi += 1;
+ }
+
+ if (argc > argi) {
+ nprinted = std::atoi(argv[argi]);
+ argi += 1;
+ if (nprinted < 0) {
+ cerr << "Bad nframes argument" << endl;
+ return 1;
+ }
+ }
+ }
+
+
+ if (fakeInput) {
+ printFakeInput();
+ exit(0);
+ }
+
+ ifstream ifs(filename);
+ if (!ifs) {
+ cerr << "Failed to open " << filename << endl;
+ return 1;
+ }
+
+ cout << "Default frame size:" << " header=" << sizeof(TBB_Header) <<
+ " transient=" << sizeof(TBB_Header) + 1024 * sizeof(int16_t) + sizeof(uint32_t) <<
+ " spectral=" << sizeof(TBB_Header) + 487 * 2 * sizeof(int16_t) + sizeof(uint32_t) << endl << endl;
+
+ int exit_status = 0;
+
+ // This doesn't work directly with data from message-oriented streams like udp,
+ // because header and payload need to be read using a single read() under linux.
+ // We don't need that for dumping data from a file; buffers are separate here.
+ TBB_Header h;
+ int16_t* payload = NULL;
+ for (int i = 0; i < nprinted; i++) {
+ ifs.read(reinterpret_cast<char*>(&h), sizeof h);
+ if (!ifs || static_cast<size_t>(ifs.gcount()) < sizeof h) {
+ cerr << "Failed to read " << sizeof h << " frame header bytes from " << filename << endl;
+ exit_status = 1;
+ goto out;
+ }
+
+ printHeader(h);
+
+
+ size_t payload_len = h.nOfSamplesPerFrame;
+ if (h.nOfFreqBands != 0) {
+ payload_len *= 2; // spectral has complex nrs, so 2 * int16_t
+ }
+ payload_len += sizeof(uint32_t) / sizeof(int16_t); // crc32
+ if (payload == NULL) {
+ // assume this is enough for all future frames; this program is for formatted frame dumps, not for the real thing anyway
+ payload = new int16_t[payload_len]; // data + crc32
+ }
+
+ ifs.read(reinterpret_cast<char*>(payload), payload_len * sizeof(int16_t));
+ if (!ifs) {
+ cerr << "Failed to read " << payload_len * sizeof(int16_t) << " frame payload from " << filename << endl;
+ exit_status = 1;
+ goto out;
+ }
+ if (printData) {
+ printPayload(payload, payload_len);
+ }
+
+ cout << "----------------------------" << endl;
+ }
+
+out: // too lazy to use proper objects in this test prog, but avoid mem leaks.....
+ delete[] payload;
+ return exit_status;
+}
+