From 6db4242dd38a91e21bfa44c2d2ec706a4513c2e1 Mon Sep 17 00:00:00 2001
From: Alexander van Amesfoort <amesfoort@astron.nl>
Date: Thu, 3 Aug 2017 01:12:02 +0000
Subject: [PATCH] Task #5441: COBALT RSP raw support: initial mega commit to
LCS/Stream and RTCP/Cobalt
---
LCS/Common/include/Common/Thread/Semaphore.h | 8 +
.../Stream/FileDescriptorBasedStream.h | 10 +-
LCS/Stream/include/Stream/FileStream.h | 4 +-
LCS/Stream/include/Stream/FixedBufferStream.h | 9 +-
LCS/Stream/include/Stream/NamedPipeStream.h | 8 +-
LCS/Stream/include/Stream/NullStream.h | 7 +-
.../include/Stream/SharedMemoryStream.h | 6 +-
LCS/Stream/include/Stream/SocketStream.h | 26 +-
LCS/Stream/include/Stream/Stream.h | 6 +-
LCS/Stream/include/Stream/StringStream.h | 5 +-
LCS/Stream/src/FileDescriptorBasedStream.cc | 47 +++-
LCS/Stream/src/FixedBufferStream.cc | 65 ++++-
LCS/Stream/src/NamedPipeStream.cc | 14 +-
LCS/Stream/src/NullStream.cc | 35 ++-
LCS/Stream/src/SharedMemoryStream.cc | 14 +-
LCS/Stream/src/SocketStream.cc | 67 ++++-
LCS/Stream/src/StringStream.cc | 78 +++++-
RTCP/Cobalt/CoInterface/src/CMakeLists.txt | 1 +
RTCP/Cobalt/CoInterface/src/CorrelatedData.h | 1 -
RTCP/Cobalt/CoInterface/src/OutputTypes.h | 6 +-
RTCP/Cobalt/CoInterface/src/Parset.cc | 235 ++++++++++++++----
RTCP/Cobalt/CoInterface/src/Parset.h | 107 +++++++-
.../src/Station => CoInterface/src}/RSP.h | 29 +--
RTCP/Cobalt/CoInterface/src/RSPRawTransfer.cc | 207 +++++++++++++++
RTCP/Cobalt/CoInterface/src/RSPRawTransfer.h | 84 +++++++
RTCP/Cobalt/CoInterface/src/Stream.cc | 1 +
RTCP/Cobalt/CoInterface/src/StreamableData.h | 67 ++++-
RTCP/Cobalt/CoInterface/test/CMakeLists.txt | 1 +
.../CoInterface/test/tRSPRawTransfer.cc | 76 ++++++
RTCP/Cobalt/GPUProc/etc/CMakeLists.txt | 4 +
.../default/HardwareList.parset | 183 ++++++++++++--
.../default/HardwareUsed.parset | 23 ++
.../rspraw-enable.parset.OBSID | 58 +++++
RTCP/Cobalt/GPUProc/src/MPIReceiver.h | 4 +-
.../GPUProc/src/Station/StationInput.cc | 151 +++++++----
.../Cobalt/GPUProc/src/Station/StationInput.h | 22 +-
.../GPUProc/src/Station/StationTranspose.h | 2 +-
.../GPUProc/src/cuda/Pipelines/Pipeline.cc | 5 +-
RTCP/Cobalt/GPUProc/src/rtcp.cc | 26 +-
.../GPUProc/src/scripts/cobalt_functions.sh | 2 +-
RTCP/Cobalt/GPUProc/test/tMPIReceive.cc | 2 +-
RTCP/Cobalt/InputProc/src/Station/Generator.h | 2 +-
.../InputProc/src/Station/PacketFactory.h | 3 +-
.../InputProc/src/Station/PacketReader.cc | 16 +-
.../InputProc/src/Station/PacketReader.h | 19 +-
.../InputProc/src/Station/PacketStream.h | 76 +++++-
.../Cobalt/InputProc/src/Station/filterRSP.cc | 2 +-
.../InputProc/src/Station/generateRSP.cc | 2 +-
RTCP/Cobalt/InputProc/src/Station/printRSP.cc | 3 +-
.../Cobalt/InputProc/src/Station/repairRSP.cc | 2 +-
RTCP/Cobalt/InputProc/test/tPacketReader.cc | 2 +-
RTCP/Cobalt/InputProc/test/tRSP.cc | 2 +-
RTCP/Cobalt/OutputProc/src/CMakeLists.txt | 1 +
.../OutputProc/src/CommonLofarAttributes.cc | 2 +-
RTCP/Cobalt/OutputProc/src/GPUProcIO.cc | 64 ++++-
RTCP/Cobalt/OutputProc/src/GPUProcIO.h | 5 +-
RTCP/Cobalt/OutputProc/src/InputThread.cc | 14 +-
RTCP/Cobalt/OutputProc/src/InputThread.h | 10 +-
RTCP/Cobalt/OutputProc/src/MSWriterFile.cc | 5 +-
RTCP/Cobalt/OutputProc/src/MSWriterNull.cc | 2 +-
RTCP/Cobalt/OutputProc/src/OutputThread.cc | 145 +++++++----
RTCP/Cobalt/OutputProc/src/OutputThread.h | 19 +-
RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc | 93 +++++++
RTCP/Cobalt/OutputProc/src/RSPRawWriter.h | 75 ++++++
RTCP/Cobalt/OutputProc/src/SubbandWriter.cc | 11 +-
RTCP/Cobalt/OutputProc/src/SubbandWriter.h | 6 +-
RTCP/Cobalt/OutputProc/src/plotMS.cc | 2 +-
RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc | 4 +-
68 files changed, 1966 insertions(+), 327 deletions(-)
rename RTCP/Cobalt/{InputProc/src/Station => CoInterface/src}/RSP.h (91%)
create mode 100644 RTCP/Cobalt/CoInterface/src/RSPRawTransfer.cc
create mode 100644 RTCP/Cobalt/CoInterface/src/RSPRawTransfer.h
create mode 100644 RTCP/Cobalt/CoInterface/test/tRSPRawTransfer.cc
create mode 100644 RTCP/Cobalt/GPUProc/etc/parset-additions.d/rspraw-enable.parset.OBSID
create mode 100644 RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc
create mode 100644 RTCP/Cobalt/OutputProc/src/RSPRawWriter.h
diff --git a/LCS/Common/include/Common/Thread/Semaphore.h b/LCS/Common/include/Common/Thread/Semaphore.h
index 3bb01ecadee..a9571e2e22b 100644
--- a/LCS/Common/include/Common/Thread/Semaphore.h
+++ b/LCS/Common/include/Common/Thread/Semaphore.h
@@ -42,6 +42,7 @@ class Semaphore
bool tryDown(unsigned count = 1);
bool tryDown(unsigned count, const struct timespec ×pec);
+ unsigned getValue();
void noMore();
private:
@@ -114,6 +115,13 @@ inline bool Semaphore::tryDown(unsigned count, const struct timespec ×pec)
}
+inline unsigned Semaphore::getValue()
+{
+ ScopedLock lock(mutex); // w/ C++11, we can use memory_order_relaxed instead
+ return level;
+}
+
+
inline void Semaphore::noMore()
{
ScopedLock lock(mutex);
diff --git a/LCS/Stream/include/Stream/FileDescriptorBasedStream.h b/LCS/Stream/include/Stream/FileDescriptorBasedStream.h
index 8b46090e947..1682651b1a0 100644
--- a/LCS/Stream/include/Stream/FileDescriptorBasedStream.h
+++ b/LCS/Stream/include/Stream/FileDescriptorBasedStream.h
@@ -1,6 +1,6 @@
//# FileDescriptorBasedStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -37,8 +37,16 @@ class FileDescriptorBasedStream : public Stream
virtual size_t tryRead(void *ptr, size_t size);
virtual size_t tryWrite(const void *ptr, size_t size);
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt);
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt);
+
virtual void sync();
+ // Apart from int, fcntl can also be called with an arg of type struct flock *, or struct f_owner_ex *
+ int fcntl(int cmd);
+ int fcntl(int cmd, int arg);
+
+
int fd;
};
diff --git a/LCS/Stream/include/Stream/FileStream.h b/LCS/Stream/include/Stream/FileStream.h
index ee2cb7f3eeb..db905d9e8ac 100644
--- a/LCS/Stream/include/Stream/FileStream.h
+++ b/LCS/Stream/include/Stream/FileStream.h
@@ -1,6 +1,6 @@
//# FileStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2015-2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -40,7 +40,7 @@ class FileStream : public FileDescriptorBasedStream
virtual void skip( size_t bytes ); // seek ahead
- virtual size_t size(); // return file size
+ size_t size(); // return file size
};
} // namespace LOFAR
diff --git a/LCS/Stream/include/Stream/FixedBufferStream.h b/LCS/Stream/include/Stream/FixedBufferStream.h
index 40d0763aeff..1116043f876 100644
--- a/LCS/Stream/include/Stream/FixedBufferStream.h
+++ b/LCS/Stream/include/Stream/FixedBufferStream.h
@@ -1,6 +1,6 @@
//# FixedBufferStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -32,6 +32,10 @@ namespace LOFAR {
// no wrap-around support!
// not thread safe!
+// Note that an object represents a reader xor writer, not the buffer.
+// which is passed in. To use it, reader and writer each create an object passing the same buffer.
+// The reader may (at its own peril) overtake the writer. (I don't get what's the use of this semantic.)
+
class FixedBufferStream : public Stream
{
public:
@@ -41,6 +45,9 @@ class FixedBufferStream : public Stream
virtual size_t tryRead(void *ptr, size_t size);
virtual size_t tryWrite(const void *ptr, size_t size);
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt);
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt);
+
private:
char * const itsEnd;
diff --git a/LCS/Stream/include/Stream/NamedPipeStream.h b/LCS/Stream/include/Stream/NamedPipeStream.h
index 2980859e041..4e820936b22 100644
--- a/LCS/Stream/include/Stream/NamedPipeStream.h
+++ b/LCS/Stream/include/Stream/NamedPipeStream.h
@@ -1,6 +1,6 @@
//# NamedPipeStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -36,7 +36,11 @@ class NamedPipeStream : public Stream
NamedPipeStream(const char *name, bool serverSide);
virtual ~NamedPipeStream();
- virtual size_t tryRead(void *, size_t), tryWrite(const void *, size_t);
+ virtual size_t tryRead(void *ptr, size_t size);
+ virtual size_t tryWrite(const void *ptr, size_t size);
+
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt);
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt);
virtual void sync();
diff --git a/LCS/Stream/include/Stream/NullStream.h b/LCS/Stream/include/Stream/NullStream.h
index 96e5086bd21..5e373facb29 100644
--- a/LCS/Stream/include/Stream/NullStream.h
+++ b/LCS/Stream/include/Stream/NullStream.h
@@ -1,6 +1,6 @@
//# NullStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -25,8 +25,6 @@
#include <Stream/Stream.h>
-#include <errno.h>
-
namespace LOFAR {
@@ -37,6 +35,9 @@ class NullStream : public Stream
virtual size_t tryRead(void *ptr, size_t size);
virtual size_t tryWrite(const void *ptr, size_t size);
+
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt);
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt);
};
} // namespace LOFAR
diff --git a/LCS/Stream/include/Stream/SharedMemoryStream.h b/LCS/Stream/include/Stream/SharedMemoryStream.h
index 90af7b83201..cf259f57921 100644
--- a/LCS/Stream/include/Stream/SharedMemoryStream.h
+++ b/LCS/Stream/include/Stream/SharedMemoryStream.h
@@ -1,6 +1,6 @@
//# SharedMemoryStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -39,6 +39,10 @@ class SharedMemoryStream : public Stream
virtual size_t tryRead(void *ptr, size_t size);
virtual size_t tryWrite(const void *ptr, size_t size);
+ // these 2 throw a NotImplemented exception
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt);
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt);
+
private:
Mutex readLock, writeLock;
Semaphore readDone, writePosted;
diff --git a/LCS/Stream/include/Stream/SocketStream.h b/LCS/Stream/include/Stream/SocketStream.h
index d26495c1337..730552a65d3 100644
--- a/LCS/Stream/include/Stream/SocketStream.h
+++ b/LCS/Stream/include/Stream/SocketStream.h
@@ -1,6 +1,6 @@
//# SocketStream.h:
//#
-//# Copyright (C) 2008, 2015
+//# Copyright (C) 2008, 2015-2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -59,20 +59,34 @@ class SocketStream : public FileDescriptorBasedStream
const Protocol protocol;
const Mode mode;
+ /*
+ * Send message(s). Note: only for UDP client socket!
+ * @bufBase contains all messages to be sent. It may have gaps, but all of the same size.
+ * @msgSize indicates the (common) size of each individual message.
+ * @sentMsgSizes length indicates the (maximum) number of messages to send.
+ * Note: the Linux sendmmsg(2) man page indicates this is capped to UIO_MAXIOV (1024).
+ * The actually sent sizes per message will be written into this argument.
+ * @flags: passed to sendmsg(2)/sendmmsg(2). For some of our cases, we want to always pass MSG_CONFIRM.
+ * Returns the number of messages sent if ok, or throws on syscall error.
+ */
+ unsigned sendmmsg( void *bufBase, size_t msgSize,
+ std::vector<unsigned> &sentMsgSizes, int flags ) const;
+
/*
* Receive message(s). Note: only for UDP server socket!
* @bufBase is large enough to store all to be received messages
- * @maxMsgSize indicates the max size of _each_ (i.e. 1) message
- * @recvdMsgSizes is passed in with a size indicating the max number of
- * messages to receive. Actually received sizes will be written therein.
+ * @maxMsgSize indicates the max size of each individual message
+ * @recvdMsgSizes length indicates the maximum number of messages to receive.
+ * The actually received sizes per message will be written into this argument.
* Returns the number of messages received if ok, or throws on syscall error
*/
- unsigned recvmmsg( void *bufBase, unsigned maxMsgSize,
+ unsigned recvmmsg( void *bufBase, size_t maxMsgSize,
std::vector<unsigned> &recvdMsgSizes ) const;
- // Allow individual recv()/send() calls to last for 'timeout' seconds before returning EWOULDBLOCK
+ // Allow individual recv()/send() calls to last for 'timeout' seconds before returning EAGAIN (or EWOULDBLOCK)
void setTimeout(double timeout);
+ std::string getHostname() const { return hostname; }
int getPort() const { return port; }
private:
diff --git a/LCS/Stream/include/Stream/Stream.h b/LCS/Stream/include/Stream/Stream.h
index 49cbf2465d2..93b7ff703c2 100644
--- a/LCS/Stream/include/Stream/Stream.h
+++ b/LCS/Stream/include/Stream/Stream.h
@@ -1,6 +1,6 @@
//# Stream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -24,6 +24,7 @@
#define LOFAR_LCS_STREAM_STREAM_H
#include <cstddef>
+#include <sys/uio.h>
#include <string>
#include <Common/Exception.h>
@@ -42,6 +43,9 @@ class Stream
virtual size_t tryWrite(const void *ptr, size_t size) = 0;
void write(const void *ptr, size_t size); // does not return until all bytes are written
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt) = 0;
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt) = 0;
+
std::string readLine(); // excludes '\n'
virtual void sync();
diff --git a/LCS/Stream/include/Stream/StringStream.h b/LCS/Stream/include/Stream/StringStream.h
index c305d56a640..ba43c9150da 100644
--- a/LCS/Stream/include/Stream/StringStream.h
+++ b/LCS/Stream/include/Stream/StringStream.h
@@ -1,6 +1,6 @@
//# StringStream.h:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -39,6 +39,9 @@ class StringStream : public Stream
virtual size_t tryRead(void *ptr, size_t size);
virtual size_t tryWrite(const void *ptr, size_t size);
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt);
+ virtual size_t tryWritev(const struct iovec *iov, int iovcnt);
+
void close();
private:
diff --git a/LCS/Stream/src/FileDescriptorBasedStream.cc b/LCS/Stream/src/FileDescriptorBasedStream.cc
index ebd7a2c9074..5b6f3380533 100644
--- a/LCS/Stream/src/FileDescriptorBasedStream.cc
+++ b/LCS/Stream/src/FileDescriptorBasedStream.cc
@@ -1,6 +1,6 @@
//# FileDescriptorBasedStream.cc:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -25,6 +25,7 @@
#include <Stream/FileDescriptorBasedStream.h>
#include <unistd.h>
+#include <fcntl.h>
#include <Common/SystemCallException.h>
#include <Common/Thread/Cancellation.h>
@@ -82,10 +83,54 @@ size_t FileDescriptorBasedStream::tryWrite(const void *ptr, size_t size)
}
+size_t FileDescriptorBasedStream::tryReadv(const struct iovec *iov, int iovcnt)
+{
+ ssize_t bytes = ::readv(fd, iov, iovcnt);
+
+ if (bytes < 0)
+ THROW_SYSCALL("readv");
+
+ return bytes;
+}
+
+
+size_t FileDescriptorBasedStream::tryWritev(const struct iovec *iov, int iovcnt)
+{
+ ssize_t bytes = ::writev(fd, iov, iovcnt);
+
+ if (bytes < 0)
+ THROW_SYSCALL("writev");
+
+ return bytes;
+}
+
+
void FileDescriptorBasedStream::sync()
{
if (::fsync(fd) < 0)
THROW_SYSCALL("fsync");
}
+
+int FileDescriptorBasedStream::fcntl(int cmd)
+{
+ int rv = ::fcntl(fd, cmd);
+
+ if (rv < 0)
+ THROW_SYSCALL("fcntl");
+
+ return rv;
+}
+
+
+int FileDescriptorBasedStream::fcntl(int cmd, int arg)
+{
+ int rv = ::fcntl(fd, cmd, arg);
+
+ if (rv < 0)
+ THROW_SYSCALL("fcntl");
+
+ return rv;
+}
+
} // namespace LOFAR
diff --git a/LCS/Stream/src/FixedBufferStream.cc b/LCS/Stream/src/FixedBufferStream.cc
index 1578ba4042b..43630a2e44a 100644
--- a/LCS/Stream/src/FixedBufferStream.cc
+++ b/LCS/Stream/src/FixedBufferStream.cc
@@ -1,6 +1,6 @@
//# FixedBufferStream.cc:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -48,6 +48,9 @@ size_t FixedBufferStream::tryRead(void *ptr, size_t size)
{
Cancellation::point(); // keep behaviour consistent with real I/O streams
+ if (size == 0)
+ return 0;
+
size_t numBytes = std::min<size_t>(size, itsEnd - itsHead);
if (numBytes == 0)
@@ -64,6 +67,9 @@ size_t FixedBufferStream::tryWrite(const void *ptr, size_t size)
{
Cancellation::point(); // keep behaviour consistent with real I/O streams
+ if (size == 0)
+ return 0;
+
size_t numBytes = std::min<size_t>(size, itsEnd - itsHead);
if (numBytes == 0)
@@ -75,4 +81,61 @@ size_t FixedBufferStream::tryWrite(const void *ptr, size_t size)
return numBytes;
}
+
+size_t FixedBufferStream::tryReadv(const struct iovec *iov, int iovcnt)
+{
+ Cancellation::point(); // keep behaviour consistent with real I/O streams
+
+ size_t nread = 0;
+
+ for (int i = 0; i < iovcnt; i++) {
+ if (iov[i].iov_len <= (size_t)(itsEnd - itsHead)) {
+ memcpy(iov[i].iov_base, itsHead, iov[i].iov_len);
+ itsHead += iov[i].iov_len;
+ nread += iov[i].iov_len;
+ } else {
+ if (itsEnd - itsHead == 0) {
+ if (nread == 0) // to be read > 0
+ THROW(EndOfStreamException, "No space left in buffer");
+ } else {
+ memcpy(iov[i].iov_base, itsHead, itsEnd - itsHead);
+ itsHead = itsEnd;
+ nread += itsEnd - itsHead;
+ }
+ break;
+ }
+ }
+
+ return nread;
+}
+
+
+size_t FixedBufferStream::tryWritev(const struct iovec *iov, int iovcnt)
+{
+ Cancellation::point(); // keep behaviour consistent with real I/O streams
+
+ size_t nwritten = 0;
+
+ for (int i = 0; i < iovcnt; i++) {
+ if (iov[i].iov_len <= (size_t)(itsEnd - itsHead)) {
+ memcpy(itsHead, iov[i].iov_base, iov[i].iov_len);
+ itsHead += iov[i].iov_len;
+ nwritten += iov[i].iov_len;
+ } else {
+ if (itsEnd - itsHead == 0) {
+ if (nwritten == 0) // to be written > 0
+ THROW(EndOfStreamException, "No space left in buffer");
+ } else {
+ memcpy(itsHead, iov[i].iov_base, itsEnd - itsHead);
+ itsHead = itsEnd;
+ nwritten += itsEnd - itsHead;
+ }
+ break;
+ }
+ }
+
+ return nwritten;
+}
+
+
} // namespace LOFAR
diff --git a/LCS/Stream/src/NamedPipeStream.cc b/LCS/Stream/src/NamedPipeStream.cc
index 20d7d961a22..2feca603820 100644
--- a/LCS/Stream/src/NamedPipeStream.cc
+++ b/LCS/Stream/src/NamedPipeStream.cc
@@ -1,6 +1,6 @@
//# NamedPipeStream.cc:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -86,6 +86,18 @@ size_t NamedPipeStream::tryWrite(const void *ptr, size_t size)
}
+size_t NamedPipeStream::tryReadv(const struct iovec *iov, int iovcnt)
+{
+ return itsReadStream->tryReadv(iov, iovcnt);
+}
+
+
+size_t NamedPipeStream::tryWritev(const struct iovec *iov, int iovcnt)
+{
+ return itsWriteStream->tryWritev(iov, iovcnt);
+}
+
+
void NamedPipeStream::sync()
{
itsWriteStream->sync();
diff --git a/LCS/Stream/src/NullStream.cc b/LCS/Stream/src/NullStream.cc
index 1b6c54b3e84..24e29d5e481 100644
--- a/LCS/Stream/src/NullStream.cc
+++ b/LCS/Stream/src/NullStream.cc
@@ -1,6 +1,6 @@
//# NullStream.cc:
//#
-//# Copyright (C) 2008
+//# Copyright (C) 2008, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -23,10 +23,9 @@
#include <lofar_config.h>
#include <Stream/NullStream.h>
-#include <Common/Thread/Cancellation.h>
#include <cstring>
-
+#include <Common/Thread/Cancellation.h>
namespace LOFAR {
@@ -51,4 +50,34 @@ size_t NullStream::tryWrite(const void *, size_t size)
return size;
}
+
+size_t NullStream::tryReadv(const struct iovec *iov, int iovcnt)
+{
+ Cancellation::point(); // keep behaviour consistent with non-null streams
+
+ size_t size = 0;
+
+ for (int i = 0; i < iovcnt; i++) {
+ memset(iov[i].iov_base, 0, iov[i].iov_len);
+ size += iov[i].iov_len;
+ }
+
+ return size;
+}
+
+
+size_t NullStream::tryWritev(const struct iovec *iov, int iovcnt)
+{
+ Cancellation::point(); // keep behaviour consistent with non-null streams
+
+ size_t size = 0;
+
+ for (int i = 0; i < iovcnt; i++) {
+ size += iov[i].iov_len;
+ }
+
+ return size;
+}
+
+
} // namespace LOFAR
diff --git a/LCS/Stream/src/SharedMemoryStream.cc b/LCS/Stream/src/SharedMemoryStream.cc
index e589c61da9a..921efd1d450 100644
--- a/LCS/Stream/src/SharedMemoryStream.cc
+++ b/LCS/Stream/src/SharedMemoryStream.cc
@@ -1,6 +1,6 @@
//# SharedMemoryStream.cc:
//#
-//# Copyright (C) 2012
+//# Copyright (C) 2012, 2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -61,6 +61,18 @@ size_t SharedMemoryStream::tryWrite(const void *ptr, size_t size)
return readSize;
}
+
+size_t SharedMemoryStream::tryReadv(const struct iovec * /*iov*/, int /*iovcnt*/)
+{
+ THROW(NotImplemented, "SharedMemoryStream::tryReadv()");
+}
+
+
+size_t SharedMemoryStream::tryWritev(const struct iovec * /*iov*/, int /*iovcnt*/)
+{
+ THROW(NotImplemented, "SharedMemoryStream::tryWritev()");
+}
+
} // namespace LOFAR
#endif
diff --git a/LCS/Stream/src/SocketStream.cc b/LCS/Stream/src/SocketStream.cc
index 8afb6968e39..a08f7c5dd5c 100644
--- a/LCS/Stream/src/SocketStream.cc
+++ b/LCS/Stream/src/SocketStream.cc
@@ -1,6 +1,6 @@
//# SocketStream.cc:
//#
-//# Copyright (C) 2008, 2015
+//# Copyright (C) 2008, 2015-2017
//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
//#
@@ -29,6 +29,7 @@
#include <cstdio>
#include <ctime>
#include <cerrno>
+#include <cassert>
#include <sys/types.h>
#include <unistd.h>
#include <sys/select.h>
@@ -282,8 +283,8 @@ void SocketStream::setTimeout(double timeout)
#if defined __linux__ && __GLIBC_PREREQ(2,12)
-// Actually, recvmmsg is supported by Linux 2.6.32+ using glibc 2.12+
-#define HAVE_RECVMMSG
+// Actually, sendmmsg and recvmmsg are supported by Linux 2.6.32+ using glibc 2.12+
+#define HAVE_MMSG
#else
// Equalize data structures to share more code. Declared within LOFAR namespace.
// Need it as vector template arg, so cannot be locally declared (ok w/ C++11).
@@ -292,17 +293,71 @@ struct mmsghdr {
};
#endif
-unsigned SocketStream::recvmmsg( void *bufBase, unsigned maxMsgSize,
+unsigned SocketStream::sendmmsg( void *bufBase, size_t msgSize,
+ std::vector<unsigned> &sentMsgSizes, int flags ) const
+{
+ ASSERT(protocol == UDP);
+ ASSERT(mode == Client);
+
+ // If sendmmsg() is not available, then use sendmsg() (1 call) as fall-back.
+#ifdef HAVE_MMSG
+ const unsigned numBufs = sentMsgSizes.size();
+#else
+ const unsigned numBufs = 1;
+ if (sentMsgSizes.empty()) {
+ return 0;
+ }
+#endif
+
+ // register our send buffer(s)
+ std::vector<struct iovec> iov(numBufs);
+ for (unsigned i = 0; i < numBufs; i++) {
+ iov[i].iov_base = (char*)bufBase + i * msgSize;
+ iov[i].iov_len = msgSize;
+ }
+
+ std::vector<struct mmsghdr> msgs(numBufs);
+ for (unsigned i = 0; i < numBufs; ++i) {
+ msgs[i].msg_hdr.msg_name = NULL; // dst address already set on socket via connect(2)
+ msgs[i].msg_hdr.msg_iov = &iov[i];
+ msgs[i].msg_hdr.msg_iovlen = 1;
+ msgs[i].msg_hdr.msg_control = NULL; // we're not interested in sending OoB data
+ }
+
+ int numSent;
+#ifdef HAVE_MMSG
+ numSent = ::sendmmsg(fd, &msgs[0], numBufs, flags);
+ if (numSent < 0)
+ THROW_SYSCALL("sendmmsg");
+
+ for (int i = 0; i < numSent; ++i) {
+ sentMsgSizes[i] = msgs[i].msg_len; // num bytes sent is stored in msg_len by sendmmsg()
+ }
+#else
+ numSent = ::sendmsg(fd, &msgs[0].msg_hdr, flags);
+ if (numSent < 0)
+ THROW_SYSCALL("sendmsg");
+
+ sentMsgSizes[0] = static_cast<unsigned>(numSent); // num bytes sent is returned by sendmsg()
+ numSent = 1; // equalize return val semantics to num msgs sent
+#endif
+ return static_cast<unsigned>(numSent);
+}
+
+unsigned SocketStream::recvmmsg( void *bufBase, size_t maxMsgSize,
std::vector<unsigned> &recvdMsgSizes ) const
{
ASSERT(protocol == UDP);
ASSERT(mode == Server);
// If recvmmsg() is not available, then use recvmsg() (1 call) as fall-back.
-#ifdef HAVE_RECVMMSG
+#ifdef HAVE_MMSG
const unsigned numBufs = recvdMsgSizes.size();
#else
const unsigned numBufs = 1;
+ if (recvdMsgSizes.empty()) {
+ return 0;
+ }
#endif
// register our receive buffer(s)
@@ -321,7 +376,7 @@ unsigned SocketStream::recvmmsg( void *bufBase, unsigned maxMsgSize,
}
int numRead;
-#ifdef HAVE_RECVMMSG
+#ifdef HAVE_MMSG
// Note: the timeout parameter doesn't work as expected: only between datagrams
// is the timeout checked, not when waiting for one (i.e. numBufs=1 or MSG_WAITFORONE).
numRead = ::recvmmsg(fd, &msgs[0], numBufs, 0, NULL);
diff --git a/LCS/Stream/src/StringStream.cc b/LCS/Stream/src/StringStream.cc
index d629d634687..b213a18a5a5 100644
--- a/LCS/Stream/src/StringStream.cc
+++ b/LCS/Stream/src/StringStream.cc
@@ -40,13 +40,20 @@ size_t StringStream::tryRead(void *ptr, size_t size)
{
Cancellation::point(); // keep behaviour consistent with real I/O streams
+ {
+ ScopedLock sl(itsMutex);
+// still wrong for !USE_THREADS wrt stringstream exception vs EndOfStreamException, but !USE_THREADS is obsolete anyway
#ifdef USE_THREADS
- if (!dataWritten.down(size))
- THROW(EndOfStreamException, "Stream has been closed");
+ if (!dataWritten.down(size)) {
+ size_t avail = dataWritten.getValue();
+ if (avail == 0) // size > 0
+ THROW(EndOfStreamException, "Stream has been closed");
+
+ size = avail;
+ dataWritten.down(size);
+ }
#endif
- {
- ScopedLock sl(itsMutex);
itsBuffer.read(static_cast<char*>(ptr), size);
}
@@ -71,6 +78,69 @@ size_t StringStream::tryWrite(const void *ptr, size_t size)
}
+size_t StringStream::tryReadv(const struct iovec *iov, int iovcnt)
+{
+ Cancellation::point(); // keep behaviour consistent with non-null streams
+
+ size_t size = 0;
+
+ for (int i = 0; i < iovcnt; i++) {
+ size += iov[i].iov_len;
+ }
+
+ {
+ ScopedLock sl(itsMutex);
+// still wrong for !USE_THREADS wrt stringstream exception vs EndOfStreamException, but !USE_THREADS is obsolete anyway
+#ifdef USE_THREADS
+ if (!dataWritten.down(size)) {
+ size_t avail = dataWritten.getValue();
+ if (avail == 0) // size > 0
+ THROW(EndOfStreamException, "Stream has been closed");
+
+ size = avail;
+ dataWritten.down(size);
+
+ for (int i = 0; i < iovcnt && avail > 0; i++) {
+ size_t len = avail < iov[i].iov_len ? avail : iov[i].iov_len;
+ itsBuffer.read(static_cast<char*>(iov[i].iov_base), len);
+ avail -= len;
+ }
+
+ return size;
+ }
+#endif
+
+ for (int i = 0; i < iovcnt; i++) {
+ itsBuffer.read(static_cast<char*>(iov[i].iov_base), iov[i].iov_len);
+ }
+ }
+
+ return size;
+}
+
+
+size_t StringStream::tryWritev(const struct iovec *iov, int iovcnt)
+{
+ Cancellation::point(); // keep behaviour consistent with non-null streams
+
+ size_t size = 0;
+
+ {
+ ScopedLock sl(itsMutex);
+ for (int i = 0; i < iovcnt; i++) {
+ itsBuffer.write(static_cast<const char*>(iov[i].iov_base), iov[i].iov_len);
+ size += iov[i].iov_len;
+ }
+ }
+
+#ifdef USE_THREADS
+ dataWritten.up(size);
+#endif
+
+ return size;
+}
+
+
void StringStream::close()
{
#ifdef USE_THREADS
diff --git a/RTCP/Cobalt/CoInterface/src/CMakeLists.txt b/RTCP/Cobalt/CoInterface/src/CMakeLists.txt
index 5820f8c93da..0a9300bbc94 100644
--- a/RTCP/Cobalt/CoInterface/src/CMakeLists.txt
+++ b/RTCP/Cobalt/CoInterface/src/CMakeLists.txt
@@ -15,6 +15,7 @@ lofar_add_library(cointerface
BudgetTimer.cc
FinalMetaData.cc
LTAFeedback.cc
+ RSPRawTransfer.cc
Stream.cc
Parset.cc
RunningStatistics.cc
diff --git a/RTCP/Cobalt/CoInterface/src/CorrelatedData.h b/RTCP/Cobalt/CoInterface/src/CorrelatedData.h
index 0efe2c42eee..b191945a19c 100644
--- a/RTCP/Cobalt/CoInterface/src/CorrelatedData.h
+++ b/RTCP/Cobalt/CoInterface/src/CorrelatedData.h
@@ -29,7 +29,6 @@
#include <CoInterface/Config.h>
#include <CoInterface/StreamableData.h>
#include <CoInterface/MultiDimArray.h>
-#include <CoInterface/OutputTypes.h>
namespace LOFAR
diff --git a/RTCP/Cobalt/CoInterface/src/OutputTypes.h b/RTCP/Cobalt/CoInterface/src/OutputTypes.h
index a2280b6f80f..032701e3eac 100644
--- a/RTCP/Cobalt/CoInterface/src/OutputTypes.h
+++ b/RTCP/Cobalt/CoInterface/src/OutputTypes.h
@@ -1,5 +1,6 @@
//# OutputTypes.h
-//# Copyright (C) 2011-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2011-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -32,7 +33,8 @@ namespace LOFAR
enum OutputType
{
CORRELATED_DATA = 1,
- BEAM_FORMED_DATA
+ BEAM_FORMED_DATA,
+ RSP_RAW_DATA
};
} // namespace Cobalt
diff --git a/RTCP/Cobalt/CoInterface/src/Parset.cc b/RTCP/Cobalt/CoInterface/src/Parset.cc
index afbc2653015..f83a97e3ce3 100644
--- a/RTCP/Cobalt/CoInterface/src/Parset.cc
+++ b/RTCP/Cobalt/CoInterface/src/Parset.cc
@@ -1,5 +1,5 @@
//# Parset.cc
-//# Copyright (C) 2008-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2008-2017 ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -319,10 +319,11 @@ namespace LOFAR
// NOTE: Make sure that all keys have defaults, to make test parsets
// a lot shorter.
+ // --Update: No, instead use tParsetDefault in tests and reject parsets with missing required values.
- vector<string> emptyVectorString;
- vector<unsigned> emptyVectorUnsigned;
- vector<double> emptyVectorDouble;
+ const vector<string> emptyVectorString;
+ const vector<unsigned> emptyVectorUnsigned;
+ const vector<double> emptyVectorDouble;
// Generic information
settings.realTime = getBool("Cobalt.realTime", false);
@@ -498,7 +499,7 @@ namespace LOFAR
string prefix = str(format("PIC.Core.Cobalt.%s.") % node.name);
node.hostName = getString(prefix + "host", "localhost");
- node.cpu = getUint32(prefix + "cpu", 0);
+ node.cpu = getInt(prefix + "cpu", -1);
node.mpi_nic = getString(prefix + "mpi_nic", "");
node.gpus = getUint32Vector(prefix + "gpus", vector<unsigned>(1,0)); // default to [0]
@@ -635,35 +636,7 @@ namespace LOFAR
settings.beamFormer.nrDelayCompensationChannels = nrDelayCompCh;
// Derive antennaFields to use for beam forming
- vector<string> beamFormerStations = getStringVector("Cobalt.BeamFormer.stationList", emptyVectorString, true);
- if (beamFormerStations.empty()) {
- // if [], default to all stations
- beamFormerStations = stations;
- } else {
- // filter out stations not in the observation. This allows basic defaults ([CS002..7] etc) to be used, even if
- // they contain stations that are not available.
- vector<string> filteredList;
-
- for (size_t i = 0; i < beamFormerStations.size(); i++) {
- if (std::find(stations.begin(),
- stations.end(),
- beamFormerStations[i]) != stations.end()) {
- filteredList.push_back(beamFormerStations[i]);
- } else {
- LOG_WARN_STR("Removing station " << beamFormerStations[i] << " from the beam-former, as it is not participating in the observation: " << settings.rawStationList);
- }
- }
-
- beamFormerStations = filteredList;
- }
-
- // Note that this could happen by accident if too many stations are taken out of the observation, leaving none to beam form.
- ASSERTSTR(!beamFormerStations.empty(), "No stations left to beam form! Aborting.");
-
- // Sort stations (CS, RS, int'l), to get a consistent and predictable order.
- std::sort(beamFormerStations.begin(), beamFormerStations.end(), compareStationNames);
-
- settings.beamFormer.antennaFieldNames = ObservationSettings::expandAntennaFieldNames(beamFormerStations, settings.antennaSet);
+ settings.beamFormer.antennaFieldNames = getOutputTypeAntennaFieldNames("Cobalt.BeamFormer.stationList", stations);
LOG_DEBUG_STR("Beamforming " << settings.beamFormer.antennaFieldNames.size() << " fields: " << settings.beamFormer.antennaFieldNames);
ObservationSettings::BeamFormer::StokesSettings
@@ -902,6 +875,113 @@ namespace LOFAR
settings.beamFormer.dedispersionFFTsize = getUint32("Cobalt.BeamFormer.dedispersionFFTsize", settings.blockSize);
}
+ /* ===============================
+ * RSPRaw collection information
+ * ===============================
+ */
+
+ settings.rspRaw.enabled = getBool("Observation.DataProducts.Output_RSPRaw.enabled", false);
+ if (settings.rspRaw.enabled) {
+ if (isDefined("Cobalt.RSPRaw.startTime")) {
+ settings.rspRaw.startTime = getTime("Cobalt.RSPRaw.startTime", "");
+ } else {
+ settings.rspRaw.startTime = (time_t)settings.startTime;
+ }
+ if (isDefined("Cobalt.RSPRaw.stopTime")) {
+ settings.rspRaw.stopTime = getTime("Cobalt.RSPRaw.stopTime", "");
+ } else {
+ settings.rspRaw.stopTime = (time_t)settings.stopTime;
+ }
+
+ // Read antenna field names (via stationList key) to use for RSP raw output
+ settings.rspRaw.antennaFieldNames = getOutputTypeAntennaFieldNames("Cobalt.RSPRaw.stationList", stations);
+ if (settings.rspRaw.antennaFieldNames.empty()) {
+ settings.rspRaw.antennaFieldNames = settings.antennaFieldNames; // default
+ }
+ LOG_INFO_STR("RSP raw: " << settings.rspRaw.antennaFieldNames.size() << " fields: " << settings.rspRaw.antennaFieldNames);
+
+ /*
+ * RSPRaw is not a supported output type in the rest of the system (it is COBALT-only).
+ * The Observatory specifies a correlation or beamforming observation, then copies a
+ * parset override in place. Hence, all RSPRaw settings must be auto-detectable (sane defaults),
+ * possibly from correlation or beamforming settings, even if those are not enabled (anymore)!
+ */
+ vector<ObservationSettings::FileLocation> locations = getFileLocations("RSPRaw");
+ const bool locationsAutoDetected = locations.empty();
+ if (locationsAutoDetected) {
+ const vector<ObservationSettings::FileLocation> correlated_locations = getFileLocations("Correlated");
+ const vector<ObservationSettings::FileLocation> coherent_locations = getFileLocations("CoherentStokes");
+ const vector<ObservationSettings::FileLocation> incoherent_locations = getFileLocations("IncoherentStokes");
+
+ locations.insert(locations.end(), correlated_locations.begin(), correlated_locations.end());
+ locations.insert(locations.end(), coherent_locations.begin(), coherent_locations.end());
+ locations.insert(locations.end(), incoherent_locations.begin(), incoherent_locations.end());
+ if (locations.empty()) {
+ THROW(CoInterfaceException, "No RSP raw locations specified and could not derive any location(s) from correlated, coherent, or incoherent file locations (even if not enabled).");
+ }
+
+ // Purge duplicate hostname locations. Clearing filenames first makes this easier.
+ for (size_t i = 0; i < locations.size(); ++i) {
+ locations[i].filename.clear();
+ }
+ std::sort(locations.begin(), locations.end());
+ std::unique(locations.begin(), locations.end());
+ }
+
+ // Assign output file locations
+ unsigned maxNrAntFieldStreams = 0; // normally we have 4 sending RSP boards per antenna field
+ unsigned locationIdx = 0;
+ for (size_t i = 0; i < settings.rspRaw.antennaFieldNames.size(); ++i) {
+ const ObservationSettings::AntennaFieldName& afName = settings.rspRaw.antennaFieldNames[i];
+ vector<ObservationSettings::AntennaFieldName>::iterator nameIt = std::find(settings.antennaFieldNames.begin(),
+ settings.antennaFieldNames.end(), afName);
+ ASSERTSTR(nameIt == settings.antennaFieldNames.end(), "RSP raw antenna field name " << afName.fullName() <<
+ " missing in observation antenna field list"); // getOutputTypeAntennaFieldNames() must have avoided this
+ vector<ObservationSettings::AntennaField>::iterator afIt = settings.antennaFields.begin() +
+ std::distance(settings.antennaFieldNames.begin(), nameIt);
+ size_t nrStreams = afIt->inputStreams.size();
+ if (maxNrAntFieldStreams < nrStreams) {
+ maxNrAntFieldStreams = nrStreams;
+ }
+
+ for (unsigned s = 0; s < nrStreams; ++s) {
+ if (locationIdx >= locations.size()) {
+ if (locationsAutoDetected) {
+ locationIdx = 0; // if auto-detected, assign locations in round-robin order
+ } else { // Either you specify no locations and we do it all, or you specify a complete list. No half-baked RSP raw settings!
+ THROW(CoInterfaceException, "No RSP raw filename or location specified for antenna field " <<
+ afName.fullName() << " stream " << s);
+ }
+ }
+
+ ObservationSettings::RSPRaw::File file;
+ file.antennaFieldNameIdx = i;
+ file.streamNr = s;
+ locations[locationIdx].filename = str(format("L%u_%s_%u_rsp.raw") % settings.observationID % afName.fullName().c_str() % s);
+ file.location = locations[locationIdx];
+ settings.rspRaw.files.push_back(file);
+
+ outputProcHosts.insert(file.location.host);
+
+ locationIdx += 1;
+ }
+ }
+
+ // Read nrBeamletsPerBoardList and apply sane defaults
+ settings.rspRaw.nrBeamletsPerBoardList = getUint32Vector("Cobalt.RSPRaw.nrBeamletsPerBoardList",
+ emptyVectorUnsigned, true);
+ if (settings.rspRaw.nrBeamletsPerBoardList.size() < maxNrAntFieldStreams) {
+ settings.rspRaw.nrBeamletsPerBoardList.resize(maxNrAntFieldStreams);
+ }
+ const unsigned maxNrPayloadBeamlets = 61 * 16 / settings.nrBitsPerSample; // see InputProc/src/Station/RSP.h, which we don't want to depend on here (better move hardware specs to CoInterface or LCS)
+ for (size_t i = 0; i < settings.rspRaw.nrBeamletsPerBoardList.size(); ++i) {
+ if (settings.rspRaw.nrBeamletsPerBoardList[i] > maxNrPayloadBeamlets) {
+ settings.rspRaw.nrBeamletsPerBoardList[i] = maxNrPayloadBeamlets;
+ }
+ }
+ }
+
+
// set output hosts
settings.outputProcHosts.clear();
for (set<string>::const_iterator i = outputProcHosts.begin(); i != outputProcHosts.end(); ++i) {
@@ -915,6 +995,41 @@ namespace LOFAR
return settings;
}
+ vector<struct ObservationSettings::AntennaFieldName>
+ Parset::getOutputTypeAntennaFieldNames(const string& stationListKey,
+ const vector<string>& stations) const
+ {
+ vector<string> newStations = getStringVector(stationListKey, vector<string>(), true);
+ if (newStations.empty()) {
+ // if [], default to all stations
+ newStations = stations;
+ } else {
+ // filter out stations not in the observation. This allows basic defaults ([CS002..7] etc) to be used, even if
+ // they contain stations that are not available.
+ vector<string> filteredList;
+
+ for (size_t i = 0; i < newStations.size(); i++) {
+ if (std::find(stations.begin(), stations.end(),
+ newStations[i]) != stations.end()) {
+ filteredList.push_back(newStations[i]);
+ } else {
+ LOG_WARN_STR("Removing station " << newStations[i] << " from interpreted " << stationListKey <<
+ ", as it is not participating in the observation: " << settings.rawStationList);
+ }
+ }
+
+ newStations = filteredList;
+ }
+
+ // Note that this could happen by accident if too many stations are taken out of the observation, leaving none
+ ASSERTSTR(!newStations.empty(), "No stations left! Aborting.");
+
+ // Sort stations (CS, RS, int'l), to get a consistent and predictable order.
+ std::sort(newStations.begin(), newStations.end(), compareStationNames);
+
+ return ObservationSettings::expandAntennaFieldNames(newStations, settings.antennaSet);
+ }
+
bool Parset::nodeReadsAntennaFieldData(const struct ObservationSettings& settings,
const string& nodeName) const {
for (size_t i = 0; i < settings.antennaFields.size(); ++i) {
@@ -1156,15 +1271,6 @@ namespace LOFAR
}
- void Parset::checkVectorLength(const std::string &key, unsigned expectedSize) const
- {
- unsigned actualSize = getStringVector(key, true).size();
-
- if (actualSize != expectedSize)
- THROW(CoInterfaceException, "Key \"" << string(key) << "\" contains wrong number of entries (expected: " << expectedSize << ", actual: " << actualSize << ')');
- }
-
-
void Parset::checkInputConsistency() const
{
}
@@ -1185,10 +1291,13 @@ namespace LOFAR
std::string Parset::getHostName(OutputType outputType, unsigned streamNr) const
{
if (outputType == CORRELATED_DATA)
- return settings.correlator.files[streamNr].location.host;
+ return settings.correlator.files.at(streamNr).location.host;
if (outputType == BEAM_FORMED_DATA)
- return settings.beamFormer.files[streamNr].location.host;
+ return settings.beamFormer.files.at(streamNr).location.host;
+
+ if (outputType == RSP_RAW_DATA)
+ return settings.rspRaw.files.at(streamNr).location.host;
return "unknown";
}
@@ -1197,10 +1306,13 @@ namespace LOFAR
std::string Parset::getFileName(OutputType outputType, unsigned streamNr) const
{
if (outputType == CORRELATED_DATA)
- return settings.correlator.files[streamNr].location.filename;
+ return settings.correlator.files.at(streamNr).location.filename;
if (outputType == BEAM_FORMED_DATA)
- return settings.beamFormer.files[streamNr].location.filename;
+ return settings.beamFormer.files.at(streamNr).location.filename;
+
+ if (outputType == RSP_RAW_DATA)
+ return settings.rspRaw.files.at(streamNr).location.filename;
return "unknown";
}
@@ -1209,10 +1321,13 @@ namespace LOFAR
std::string Parset::getDirectoryName(OutputType outputType, unsigned streamNr) const
{
if (outputType == CORRELATED_DATA)
- return settings.correlator.files[streamNr].location.directory;
+ return settings.correlator.files.at(streamNr).location.directory;
if (outputType == BEAM_FORMED_DATA)
- return settings.beamFormer.files[streamNr].location.directory;
+ return settings.beamFormer.files.at(streamNr).location.directory;
+
+ if (outputType == RSP_RAW_DATA)
+ return settings.rspRaw.files.at(streamNr).location.directory;
return "unknown";
}
@@ -1226,10 +1341,26 @@ namespace LOFAR
switch (outputType) {
case CORRELATED_DATA: return settings.correlator.files.size();
case BEAM_FORMED_DATA: return settings.beamFormer.files.size();
+ case RSP_RAW_DATA: return settings.rspRaw.files.size();
default: THROW(CoInterfaceException, "Unknown output type");
}
}
+ unsigned Parset::getRSPRawOutputStreamIdx(const std::string& antennaFieldName,
+ unsigned boardNr) const
+ {
+ const std::vector<ObservationSettings::RSPRaw::File>& files = settings.rspRaw.files;
+ for (size_t i = 0; i < files.size(); ++i) {
+ if (settings.rspRaw.antennaFieldNames.at(files[i].antennaFieldNameIdx).fullName() == antennaFieldName &&
+ files[i].streamNr == boardNr) {
+ return i;
+ }
+ }
+
+ THROW(CoInterfaceException, "Unknown output stream for antenna field name " <<
+ antennaFieldName << " and RSP board nr " << boardNr);
+ }
+
size_t Parset::nrBytesPerComplexSample() const
{
return 2 * nrBitsPerSample() / 8;
@@ -1300,7 +1431,7 @@ namespace LOFAR
return list;
}
- double Parset::getTime(const std::string &name, const std::string &defaultValue) const
+ time_t Parset::getTime(const std::string &name, const std::string &defaultValue) const
{
return LOFAR::to_time_t(boost::posix_time::time_from_string(getString(name, defaultValue)));
}
@@ -1372,7 +1503,7 @@ namespace LOFAR
return settings.nrBitsPerSample;
}
- unsigned Parset::nrObsOutputTypes() const
+ unsigned Parset::nrProcessedOutputTypes() const
{
unsigned nr = 0;
@@ -1385,6 +1516,7 @@ namespace LOFAR
if (settings.beamFormer.anyIncoherentTABs()) {
nr += 1;
}
+ // RSP raw is not a processed output type, so not counted: see caller(s) for impact if changing this
return nr;
}
@@ -1394,6 +1526,7 @@ namespace LOFAR
switch (outputType) {
case CORRELATED_DATA: return settings.correlator.enabled;
case BEAM_FORMED_DATA: return settings.beamFormer.enabled;
+ case RSP_RAW_DATA: return settings.rspRaw.enabled;
default: THROW(CoInterfaceException, "Unknown output type");
}
}
diff --git a/RTCP/Cobalt/CoInterface/src/Parset.h b/RTCP/Cobalt/CoInterface/src/Parset.h
index 9686bf00f34..7c9066f02c9 100644
--- a/RTCP/Cobalt/CoInterface/src/Parset.h
+++ b/RTCP/Cobalt/CoInterface/src/Parset.h
@@ -1,5 +1,5 @@
//# Parset.h: class/struct that holds the Parset information
-//# Copyright (C) 2008-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2008-2017 ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -191,8 +191,13 @@ namespace LOFAR
antennaField == other.antennaField;
}
+ bool operator==(const std::string &other) const {
+ return fullName() == other;
+ }
+
// Returns the indices in fullSet for each element of subSet.
- static std::vector<unsigned> indices(const std::vector<AntennaFieldName> &subSet, const std::vector<AntennaFieldName> &fullSet);
+ static std::vector<unsigned> indices(const std::vector<AntennaFieldName> &subSet,
+ const std::vector<AntennaFieldName> &fullSet);
// Returns a list of the station names.
static std::vector<std::string> names( const std::vector<AntennaFieldName> &list );
@@ -290,8 +295,8 @@ namespace LOFAR
// Host name
std::string hostName;
- // CPU number to bind to
- size_t cpu;
+ // CPU socket number to bind to, or -1 to not bind to any
+ int cpu;
// CUDA GPU numbers to bind to
std::vector<unsigned> gpus;
@@ -306,6 +311,7 @@ namespace LOFAR
std::vector<struct Node> nodes;
// Cluster where data writers run
+ // TODO: move to per output type and read Output_XXX.storageClusterName keys
std::string outputCluster;
/*
@@ -408,6 +414,28 @@ namespace LOFAR
string host;
string directory;
string filename;
+
+ // Make FileLocation objects comparable and equality testable,
+ // so we can prune dups in a vector using sort + unique.
+ bool operator<(const FileLocation& rhs) const {
+ int cmp = cluster.compare(rhs.cluster);
+ if (cmp < 0) return true;
+ if (cmp > 0) return false;
+ cmp = host.compare(rhs.host);
+ if (cmp < 0) return true;
+ if (cmp > 0) return false;
+ cmp = directory.compare(rhs.directory);
+ if (cmp < 0) return true;
+ if (cmp > 0) return false;
+ cmp = filename.compare(rhs.filename);
+ if (cmp < 0) return true;
+ return false;
+ }
+
+ bool operator==(const FileLocation& rhs) const {
+ return filename == rhs.filename && host == rhs.host &&
+ directory == rhs.directory && cluster == rhs.cluster;
+ }
};
/* ===============================
@@ -583,6 +611,9 @@ namespace LOFAR
// Antenna fields to use for beam forming.
// Must be a subset of the observation antenna-field list
+ //
+ // Derived from key: Cobalt.Beamformer.stationList
+ // TODO: split up in CoherentStokes and IncoherentStokes antenna field name lists
std::vector<AntennaFieldName> antennaFieldNames;
// All SAPs, with information about the TABs to form.
@@ -654,6 +685,64 @@ namespace LOFAR
struct BeamFormer beamFormer;
+ /* ===============================
+ * RSPRaw collection information
+ * ===============================
+ */
+
+ struct RSPRaw {
+ // Whether RSP raw data collection was requested.
+ //
+ // key: Observation.DataProducts.Output_RSPRaw.enabled
+ // Default: false
+ bool enabled;
+
+ // Specified RSP raw data dump / piggy backing start time (inclusive)
+ // in seconds since 1970.
+ //
+ // key: Cobalt.RSPRaw.startTime
+ // Default: value of Observation.startTime
+ time_t startTime;
+
+ // Specified raw RSP data dump / piggy backing stop time (exclusive)
+ // in seconds since 1970.
+ //
+ // key: Cobalt.RSPRaw.stopTime
+ // Default: value of Observation.stopTime
+ time_t stopTime;
+
+ // All antenna fields specified for RSP raw output
+ // Must be a subset of the observation antenna-field list
+ //
+ // Derived from key: Cobalt.RSPRaw.stationList
+ // Default value: antennaFieldNames (observation wide)
+ std::vector<AntennaFieldName> antennaFieldNames;
+
+ // A vector with per RSP board (for all stations) the number of beamlets,
+ // to send out from each station RSP frame.
+ // Vector size must be the (max) number of antenna field input streams (typically 4).
+ // (This is a cheap variant of full (beam, subband) selection per (station, board).)
+ //
+ // key: Cobalt.RSPRaw.nrBeamletsPerBoardList
+ // An absent key, missing values (vector too small), or too high value means all.
+ std::vector<unsigned> nrBeamletsPerBoardList;
+
+ struct File {
+ // Index into *RSP raw* antennaFieldNames
+ unsigned antennaFieldNameIdx;
+
+ // Stream number, aka sender RSP board nr (typically 0-3)
+ unsigned streamNr;
+
+ struct FileLocation location;
+ };
+
+ // The list of files to write
+ std::vector<struct File> files;
+ };
+
+ struct RSPRaw rspRaw;
+
// Returns the Nyquist zone number based on bandFilter.
unsigned nyquistZone() const;
@@ -710,13 +799,14 @@ namespace LOFAR
std::string positionType() const;
unsigned dedispersionFFTsize() const;
- unsigned nrObsOutputTypes() const;
+ unsigned nrProcessedOutputTypes() const;
bool outputThisType(OutputType) const;
unsigned nrStreams(OutputType, bool force = false) const;
std::string getHostName(OutputType, unsigned streamNr) const;
std::string getFileName(OutputType, unsigned streamNr) const;
std::string getDirectoryName(OutputType, unsigned streamNr) const;
+ unsigned getRSPRawOutputStreamIdx(const std::string& antennaFieldName, unsigned boardNr) const;
double channel0Frequency( size_t subband, size_t nrChannels ) const;
@@ -729,14 +819,17 @@ namespace LOFAR
mutable std::string itsWriteCache;
- void checkVectorLength(const std::string &key, unsigned expectedSize) const;
void checkInputConsistency() const;
void addPosition(string stName);
- double getTime(const std::string &name, const std::string &defaultValue) const;
+ time_t getTime(const std::string &name, const std::string &defaultValue) const;
std::vector<double> centroidPos(const string &stations) const;
+ std::vector<struct ObservationSettings::AntennaFieldName>
+ getOutputTypeAntennaFieldNames(const std::string& stationListKey,
+ const std::vector<std::string>& stations) const;
+
std::vector<struct ObservationSettings::FileLocation> getFileLocations(const std::string outputType) const;
// Returns whether nodeName has to participate in the observation
diff --git a/RTCP/Cobalt/InputProc/src/Station/RSP.h b/RTCP/Cobalt/CoInterface/src/RSP.h
similarity index 91%
rename from RTCP/Cobalt/InputProc/src/Station/RSP.h
rename to RTCP/Cobalt/CoInterface/src/RSP.h
index ef602ae268d..1f3ca85708f 100644
--- a/RTCP/Cobalt/InputProc/src/Station/RSP.h
+++ b/RTCP/Cobalt/CoInterface/src/RSP.h
@@ -1,5 +1,6 @@
//# RSP.h: RSP data format
-//# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2012-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -18,8 +19,8 @@
//#
//# $Id$
-#ifndef LOFAR_INPUT_PROC_RSP_H
-#define LOFAR_INPUT_PROC_RSP_H
+#ifndef LOFAR_INTERFACE_RSP_H
+#define LOFAR_INTERFACE_RSP_H
#include <cstddef>
#include <complex>
@@ -51,7 +52,7 @@ namespace LOFAR
// ----------------------------------------------------------------------
struct Header {
- // 2: Beamlet Data CoInterface 5.0
+ // 2: Beamlet Data CoInterface 5.0: no longer supported
// 3: Beamlet Data CoInterface 6.0 (8- and 4-bit mode support)
uint8 version;
@@ -98,7 +99,8 @@ namespace LOFAR
// Payload, allocated for maximum size.
// Actual size depends on the header (nrBeamlets, nrBlocks). It changed in
- // the past (61 vs 60) and may be less for tests and old pre-recorded data
+ // the past (61 vs 60) and may be less for tests, old pre-recorded data, and
+ // RSP raw output for offline reprocessing or piggy-backing (selectable beamlet subset).
union Payload {
char data[8130];
@@ -167,15 +169,9 @@ namespace LOFAR
unsigned bitMode() const
{
- if (header.version < 3)
- return 16;
-
- switch (header.sourceInfo2 & 0x3) {
- default:
- case 0x0: return 16;
- case 0x1: return 8;
- case 0x2: return 4;
- }
+ //if (header.version < 3) // disabled: Beamlet Data CoInterface 5.0 is too old to care and in a hot path
+ // return 16;
+ return 16 >> header.sourceInfo2; // 0x0: 16, 0x1: 8, 0x2: 4
}
void bitMode(unsigned mode)
@@ -206,6 +202,11 @@ namespace LOFAR
header.blockSequenceNumber = ts.getBlockId();
}
+ void timeStampError()
+ {
+ header.timestamp = 0xFFFFFFFF; // clock not initialised
+ }
+
size_t packetSize() const
{
return sizeof(RSP::Header) + header.nrBlocks * header.nrBeamlets * 2 * 2 * bitMode() / 8;
diff --git a/RTCP/Cobalt/CoInterface/src/RSPRawTransfer.cc b/RTCP/Cobalt/CoInterface/src/RSPRawTransfer.cc
new file mode 100644
index 00000000000..7d6e35e37f8
--- /dev/null
+++ b/RTCP/Cobalt/CoInterface/src/RSPRawTransfer.cc
@@ -0,0 +1,207 @@
+//# RSPRawTransfer.cc: Sender and receiver for RSP raw output
+//# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include "RSPRawTransfer.h"
+
+#include <cstring>
+#include <fcntl.h>
+#include <sys/socket.h>
+#include <Common/LofarLogger.h>
+#include <Common/SystemCallException.h>
+#include <Stream/StreamFactory.h>
+
+using namespace std;
+
+namespace LOFAR {
+ namespace Cobalt {
+
+ /* RSPRawSender */
+
+ RSPRawSender::RSPRawSender() :
+ itsStream(NULL),
+ itsSentMsgSizes(0),
+ itsNrBeamletsToSend(0),
+ itsNrDroppedPackets(0)
+ {
+ }
+
+ RSPRawSender::RSPRawSender(unsigned maxNrPacketsPerSend, unsigned nrBeamletsToSend,
+ const string& streamDesc, time_t deadline) :
+ itsStream(createStream(streamDesc, false, deadline)),
+ itsSentMsgSizes(maxNrPacketsPerSend),
+ itsNrBeamletsToSend(nrBeamletsToSend),
+ itsNrDroppedPackets(0)
+ {
+ ASSERTSTR(maxNrPacketsPerSend > 0, "maxNrPacketsPerSend must be > 0");
+
+ // In RT mode InputProc threads may not block. Prefer not sent (i.e. dropped) packets.
+ if (deadline != 0) {
+ FileDescriptorBasedStream *fdStream = dynamic_cast<FileDescriptorBasedStream *>(itsStream.get());
+ if (fdStream != NULL) {
+ fdStream->fcntl(F_SETFL, fdStream->fcntl(F_GETFL) | O_NONBLOCK);
+ }
+ }
+ }
+
+ RSPRawSender::~RSPRawSender()
+ {
+ try {
+ trySendPending(); // try, but it may be way too late
+ } catch (SystemCallException& ) {
+ itsNrDroppedPackets += 1; // count partially sent as dropped now
+ }
+
+ if (itsNrDroppedPackets > 0) {
+ SocketStream *sockStream = dynamic_cast<SocketStream *>(itsStream.get());
+ if (sockStream == NULL) {
+ LOG_WARN_STR("RSPRawSender: number of RSP packets not sent: " << itsNrDroppedPackets);
+ } else {
+ LOG_WARN_STR("RSPRawSender " << sockStream->getHostname() << ':' << sockStream->getPort() <<
+ " number of RSP packets not sent: " << itsNrDroppedPackets);
+ }
+ }
+ }
+
+ bool RSPRawSender::initialized() const
+ {
+ return itsStream != NULL;
+ }
+
+ unsigned RSPRawSender::getNrDroppedPackets() const
+ {
+ return itsNrDroppedPackets;
+ }
+
+ void RSPRawSender::trySend(struct RSP *packets, unsigned nrPackets)
+ {
+ ASSERTSTR(nrPackets <= itsSentMsgSizes.size(), "nrPackets > max indicated when sender was constructed");
+
+ /*
+ * Patch packet headers if we need to send only the 1st N beamlets.
+ * Note: we assume all packets (per RSP board) have the same nrBeamlets and size.
+ * If not in nrBeamlets, we may incorrectly restore some of the packet headers below...
+ * If not in size, we may append bogus (or uninit data), but struct RSP is large enough.
+ */
+ const uint8 packetNrBeamlets = packets[0].header.nrBeamlets; // save to restore
+ if (itsNrBeamletsToSend < packetNrBeamlets) {
+ for (unsigned i = 0; i < nrPackets; i++) {
+ packets[i].header.nrBeamlets = itsNrBeamletsToSend;
+ }
+ }
+
+ size_t packetSize = packets[0].packetSize();
+ SocketStream *sockStream = dynamic_cast<SocketStream *>(itsStream.get());
+ try {
+ if (sockStream != NULL && sockStream->protocol == SocketStream::UDP) {
+ /*
+ * MSG_CONFIRM: Inform link-layer to just send the data without periodic ARP probing.
+ * We haven't seen any replies (network peer doesn't send any in this case, with its downsides),
+ * but we cannot afford stalls.
+ */
+ unsigned nrSent = sockStream->sendmmsg(packets, packetSize, itsSentMsgSizes, MSG_CONFIRM);
+ if (nrSent < nrPackets) { // don't check itsSentMsgSizes: resending remainders won't help with UDP (message oriented)
+ itsNrDroppedPackets += nrPackets - nrSent;
+ LOG_WARN("RSPRawSender::trySend(): fewer sent to avoid blocking"); // not retried, not even in non-RT...
+ }
+ } else { // no SocketStream or SocketStream::TCP
+ // With TCP we must avoid partial RSP frame transfer. Try sending any remaining data of 1 RSP packet first.
+ trySendPending(); // may throw
+
+ // Prepare writev(2)
+ vector<struct iovec> iov(nrPackets);
+ for (unsigned i = 0; i < nrPackets; i++) {
+ iov[i].iov_base = (char *)packets + i * packetSize;
+ iov[i].iov_len = packetSize;
+ }
+
+ size_t bytesSent = itsStream->tryWritev(&iov[0], nrPackets); // may throw
+ if (bytesSent < nrPackets * packetSize) {
+ // Drop, except for unsent data of partially sent packet (if so). Stash that to retry later.
+ unsigned nrSent = bytesSent / packetSize;
+ unsigned partialPacketSent = bytesSent % packetSize;
+ if (partialPacketSent != 0) {
+ LOG_WARN("RSPRawSender::trySend(): partial packet sent: will retry remainder later");
+ const char *data = (char *)iov[nrSent].iov_base + partialPacketSent;
+ size_t size = packetSize - partialPacketSent;
+ itsPendingData.resize(size);
+ std::memcpy(&itsPendingData[0], data, size);
+ nrSent += 1; // partially sent and stashed
+ }
+
+ itsNrDroppedPackets += nrPackets - nrSent;
+ LOG_WARN("RSPRawSender::trySend(): fewer sent to avoid blocking"); // not retried, not even in non-RT...
+ }
+ }
+ } catch (SystemCallException& exc) {
+ itsNrDroppedPackets += nrPackets;
+ if (exc.error == EAGAIN || exc.error == EWOULDBLOCK) {
+ LOG_WARN("RSPRawSender: sent fewer packets than requested to avoid blocking");
+ } else {
+ static bool errorSeen;
+ if (!errorSeen) {
+ LOG_ERROR_STR("RSPRawSender: " << exc.what()); // backtrace not useful here
+ errorSeen = true;
+ }
+ }
+ }
+
+ if (itsNrBeamletsToSend < packetNrBeamlets) {
+ for (unsigned i = 0; i < nrPackets; i++) {
+ packets[i].header.nrBeamlets = packetNrBeamlets; // restore
+ }
+ }
+ }
+
+ void RSPRawSender::trySendPending()
+ {
+ if (!itsPendingData.empty()) {
+ size_t bytesSent = itsStream->tryWrite(&itsPendingData[0], itsPendingData.size()); // may throw
+ size_t newSize = itsPendingData.size() - bytesSent;
+ std::memmove(&itsPendingData[0], &itsPendingData[bytesSent], newSize); // at most 8k
+ itsPendingData.resize(newSize);
+ }
+ }
+
+
+ /* RSPRawReceiver */
+
+ RSPRawReceiver::RSPRawReceiver() :
+ itsStream(NULL)
+ {
+ }
+
+ RSPRawReceiver::RSPRawReceiver(const string& streamDesc) :
+ itsStream(createStream(streamDesc, true))
+ {
+ }
+
+ RSPRawReceiver::~RSPRawReceiver()
+ {
+ }
+
+ void RSPRawReceiver::receive() const
+ {
+ }
+
+ } // namespace Cobalt
+} // namespace LOFAR
+
diff --git a/RTCP/Cobalt/CoInterface/src/RSPRawTransfer.h b/RTCP/Cobalt/CoInterface/src/RSPRawTransfer.h
new file mode 100644
index 00000000000..34c117cefbd
--- /dev/null
+++ b/RTCP/Cobalt/CoInterface/src/RSPRawTransfer.h
@@ -0,0 +1,84 @@
+//# RSPRawTransfer.h: Sender and receiver for RSP raw output
+//# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_COINTERFACE_RSPRAWTRANSFER_H
+#define LOFAR_COINTERFACE_RSPRAWTRANSFER_H
+
+// \file
+// Sender and receiver for RSP raw output. Sender is also used for COBALT RSP data piggy-backing.
+
+#include <ctime> // time_t
+#include <string>
+#include <Stream/SocketStream.h>
+#include "RSP.h"
+#include "SmartPtr.h"
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+
+class RSPRawSender {
+public:
+ RSPRawSender();
+
+ // deadline is an absolute timestamp or 0 for no connection timeout (blocking).
+ RSPRawSender(unsigned maxNrPacketsPerSend, unsigned nrBeamletsToSend,
+ const std::string& streamDesc, time_t deadline = 0);
+
+ ~RSPRawSender();
+
+ bool initialized() const;
+
+ // Ensure nrPackets <= maxNrPacketsPerSend (passed upon object construction).
+ // If deadline (passed upon object construction) was non-zero, trySend() may drop packets to avoid blocking.
+ void trySend(struct RSP *packets, unsigned nrPackets);
+
+ unsigned getNrDroppedPackets() const;
+
+private:
+ void trySendPending();
+
+ SmartPtr<Stream> itsStream;
+ std::vector<unsigned> itsSentMsgSizes;
+ unsigned itsNrBeamletsToSend;
+ unsigned itsNrDroppedPackets;
+ std::vector<unsigned char> itsPendingData; // w/ TCP to retry sending the remainder of a partially sent RSP packet
+};
+
+class RSPRawReceiver {
+public:
+ RSPRawReceiver();
+
+ RSPRawReceiver(const std::string& streamDesc);
+
+ ~RSPRawReceiver();
+
+ void receive() const;
+
+private:
+ SmartPtr<Stream> itsStream;
+};
+
+ } // namespace Cobalt
+} // namespace LOFAR
+
+#endif
+
diff --git a/RTCP/Cobalt/CoInterface/src/Stream.cc b/RTCP/Cobalt/CoInterface/src/Stream.cc
index 24d4a3385f2..f83c8bae797 100644
--- a/RTCP/Cobalt/CoInterface/src/Stream.cc
+++ b/RTCP/Cobalt/CoInterface/src/Stream.cc
@@ -61,6 +61,7 @@ namespace LOFAR
// The returned descriptor can be supplied to LCS/Stream StreamFactory.h
+ // For RSP raw impl convenience this function must remain thread-safe.
string getStreamDescriptorBetweenIONandStorage(const Parset &parset, OutputType outputType, unsigned streamNr, const std::string &bind_local_iface)
{
string host = parset.getHostName(outputType, streamNr);
diff --git a/RTCP/Cobalt/CoInterface/src/StreamableData.h b/RTCP/Cobalt/CoInterface/src/StreamableData.h
index 5fe7ec3e223..0237ecd0b08 100644
--- a/RTCP/Cobalt/CoInterface/src/StreamableData.h
+++ b/RTCP/Cobalt/CoInterface/src/StreamableData.h
@@ -1,5 +1,6 @@
//# StreamableData.h
-//# Copyright (C) 2008-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2008-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -25,11 +26,12 @@
#include <Common/LofarTypes.h>
#include <Common/DataConvert.h>
-#include <CoInterface/Parset.h>
-#include <CoInterface/MultiDimArray.h>
-#include <CoInterface/SparseSet.h>
-#include <CoInterface/Allocator.h>
#include <Stream/Stream.h>
+#include "Parset.h"
+#include "MultiDimArray.h"
+#include "SparseSet.h"
+#include "Allocator.h"
+#include "RSP.h"
namespace LOFAR
{
@@ -50,15 +52,20 @@ namespace LOFAR
// the CPU which fills the datastructure sets the peerMagicNumber,
// because other CPUs will overwrite it with a read(s,true) call from
// either disk or network.
- StreamableData() : peerMagicNumber(magic), rawSequenceNumber(0)
+ StreamableData(bool readWithSeqNr = true, bool writeWithSeqNr = true)
+ : peerMagicNumber(magic),
+ readWithSequenceNumber(readWithSeqNr),
+ writeWithSequenceNumber(writeWithSeqNr),
+ rawSequenceNumber(0)
{
}
+
virtual ~StreamableData()
{
}
- void read(Stream *, bool withSequenceNumber, unsigned align = 1);
- void write(Stream *, bool withSequenceNumber, unsigned align = 1);
+ void read(Stream *, unsigned align);
+ void write(Stream *, unsigned align);
bool shouldByteSwap() const
{
@@ -98,6 +105,8 @@ namespace LOFAR
virtual void writeData(Stream *, unsigned) = 0;
private:
+ bool readWithSequenceNumber;
+ bool writeWithSequenceNumber;
uint32_t rawSequenceNumber; /// possibly needs byte swapping
};
@@ -121,9 +130,9 @@ namespace LOFAR
};
- inline void StreamableData::read(Stream *str, bool withSequenceNumber, unsigned alignment)
+ inline void StreamableData::read(Stream *str, unsigned alignment)
{
- if (withSequenceNumber) {
+ if (readWithSequenceNumber) {
std::vector<char> header(alignment > 2 * sizeof(uint32_t) ? alignment : 2 * sizeof(uint32_t));
uint32_t &magicValue = *reinterpret_cast<uint32_t *>(&header[0]);
uint32_t &seqNo = *reinterpret_cast<uint32_t *>(&header[sizeof(uint32_t)]);
@@ -138,10 +147,10 @@ namespace LOFAR
}
- inline void StreamableData::write(Stream *str, bool withSequenceNumber, unsigned alignment)
+ inline void StreamableData::write(Stream *str, unsigned alignment)
{
- if (withSequenceNumber) {
+ if (writeWithSequenceNumber) {
/* std::vector<char> header(alignment > sizeof(uint32_t) ? alignment : sizeof(uint32_t)); */
std::vector<char> header(alignment > 2 * sizeof(uint32_t) ? alignment : 2 * sizeof(uint32_t));
uint32_t &magicValue = *reinterpret_cast<uint32_t *>(&header[0]);
@@ -187,6 +196,40 @@ namespace LOFAR
str->write(samples.origin(), samples.num_elements() * sizeof(T));
}
+
+ class RSPRawData : public StreamableData
+ {
+ public:
+ RSPRawData()
+ : StreamableData(false, false), // raw: no seq nrs
+ buffer(bufferSize),
+ used(0)
+ {
+ }
+
+ protected:
+ virtual void readData(Stream *str, unsigned alignment)
+ {
+ (void)alignment;
+
+ used = str->tryRead(&buffer[0], bufferSize); // don't know what to expect, so read what is avail
+ }
+
+ virtual void writeData(Stream *str, unsigned alignment)
+ {
+ (void)alignment;
+
+ str->write(&buffer[0], used);
+ used = 0;
+ }
+
+ private:
+ static const unsigned bufferSize = 64 * sizeof(RSP);
+
+ vector<uint8_t> buffer; // vector<RSP> could have worked, but byte stream from TCP to storage
+ size_t used;
+ };
+
} // namespace Cobalt
} // namespace LOFAR
diff --git a/RTCP/Cobalt/CoInterface/test/CMakeLists.txt b/RTCP/Cobalt/CoInterface/test/CMakeLists.txt
index 007b8b7c288..d5f8b1bba3c 100644
--- a/RTCP/Cobalt/CoInterface/test/CMakeLists.txt
+++ b/RTCP/Cobalt/CoInterface/test/CMakeLists.txt
@@ -11,6 +11,7 @@ lofar_add_test(tpow2 tpow2.cc)
lofar_add_test(tSparseSet tSparseSet.cc)
lofar_add_test(tfpequals tfpequals.cc)
lofar_add_test(tcmpfloat DEPENDS cmpfloat)
+lofar_add_test(tRSPRawTransfer tRSPRawTransfer.cc)
if(UNITTEST++_FOUND)
diff --git a/RTCP/Cobalt/CoInterface/test/tRSPRawTransfer.cc b/RTCP/Cobalt/CoInterface/test/tRSPRawTransfer.cc
new file mode 100644
index 00000000000..aaebc93c793
--- /dev/null
+++ b/RTCP/Cobalt/CoInterface/test/tRSPRawTransfer.cc
@@ -0,0 +1,76 @@
+//# tRSPRawTransfer.cc
+//# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include <omp.h>
+
+#include <Common/LofarLogger.h>
+#include <CoInterface/RSPRawTransfer.h>
+
+using namespace LOFAR;
+using namespace LOFAR::Cobalt;
+using namespace std;
+
+static bool doSender(void)
+{
+//TODO RSPRawSender sender(...);
+
+ return true;
+}
+
+static bool doReceiver(void)
+{
+//TODO RSPRawReceiver receiver(...);
+
+ return true;
+}
+
+static bool runTest(void)
+{
+ bool senderStatus;
+ bool receiverStatus;
+
+ // create sender and receiver in separate threads for easy testing
+# pragma omp parallel sections num_threads(2)
+ {
+# pragma omp section
+ {
+ senderStatus = doSender();
+ }
+
+# pragma omp section
+ {
+ receiverStatus = doReceiver();
+ }
+ }
+
+ return senderStatus && receiverStatus;
+}
+
+int main(void)
+{
+ INIT_LOGGER("tRSPRawTransfer");
+
+ bool status = runTest();
+
+ return !status;
+}
+
diff --git a/RTCP/Cobalt/GPUProc/etc/CMakeLists.txt b/RTCP/Cobalt/GPUProc/etc/CMakeLists.txt
index aca4d465e8a..f43abc09e42 100644
--- a/RTCP/Cobalt/GPUProc/etc/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/etc/CMakeLists.txt
@@ -23,5 +23,9 @@ file(GLOB _config_parsets_bin "${CMAKE_CURRENT_BINARY_DIR}/parset-additions.d/de
install(FILES ${_config_parsets_bin} DESTINATION etc/parset-additions.d/default)
#lofar_add_sysconf_files(${_config_parsets_bin})
+# Enable RSP raw data output (and disable correlated and beamformed output) by copying this file into override/ replacing OBSID by the observation id.
+install(DIRECTORY DESTINATION etc/parset-additions.d/override) # pre-create directory, because we tell users to copy into it
+install(FILES "parset-additions.d/rspraw-enable.parset.OBSID" DESTINATION etc/parset-additions.d)
+
# Install script to generate StationStreams.parset for the live test system.
lofar_add_sbin_scripts(parset-additions.d/default/generateStationStreams.sh)
diff --git a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareList.parset b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareList.parset
index f5ac9d995bb..2cb8d95630e 100644
--- a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareList.parset
+++ b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareList.parset
@@ -14,7 +14,7 @@ PIC.Core.Cobalt.localhost.gpus=[0]
PIC.Core.Cobalt.gpu01_0.host=gpu01
PIC.Core.Cobalt.gpu01_0.cpu=0
PIC.Core.Cobalt.gpu01_0.mpi_nic=
-PIC.Core.Cobalt.gpu01_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.gpu01_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.gpu01_0.gpus=[0, 1]
PIC.Core.Cobalt.gpu01_1.host=gpu01
@@ -27,119 +27,260 @@ PIC.Core.Cobalt.gpu01_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt001_0.host=cbt001
PIC.Core.Cobalt.cbt001_0.cpu=0
PIC.Core.Cobalt.cbt001_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt001_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt001_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt001_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt001_1.host=cbt001
PIC.Core.Cobalt.cbt001_1.cpu=1
PIC.Core.Cobalt.cbt001_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt001_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt001_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt001_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt002_0.host=cbt002
PIC.Core.Cobalt.cbt002_0.cpu=0
PIC.Core.Cobalt.cbt002_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt002_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt002_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt002_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt002_1.host=cbt002
PIC.Core.Cobalt.cbt002_1.cpu=1
PIC.Core.Cobalt.cbt002_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt002_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt002_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt002_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt003_0.host=cbt003
PIC.Core.Cobalt.cbt003_0.cpu=0
PIC.Core.Cobalt.cbt003_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt003_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt003_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt003_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt003_1.host=cbt003
PIC.Core.Cobalt.cbt003_1.cpu=1
PIC.Core.Cobalt.cbt003_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt003_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt003_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt003_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt004_0.host=cbt004
PIC.Core.Cobalt.cbt004_0.cpu=0
PIC.Core.Cobalt.cbt004_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt004_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt004_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt004_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt004_1.host=cbt004
PIC.Core.Cobalt.cbt004_1.cpu=1
PIC.Core.Cobalt.cbt004_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt004_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt004_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt004_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt005_0.host=cbt005
PIC.Core.Cobalt.cbt005_0.cpu=0
PIC.Core.Cobalt.cbt005_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt005_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt005_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt005_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt005_1.host=cbt005
PIC.Core.Cobalt.cbt005_1.cpu=1
PIC.Core.Cobalt.cbt005_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt005_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt005_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt005_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt006_0.host=cbt006
PIC.Core.Cobalt.cbt006_0.cpu=0
PIC.Core.Cobalt.cbt006_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt006_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt006_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt006_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt006_1.host=cbt006
PIC.Core.Cobalt.cbt006_1.cpu=1
PIC.Core.Cobalt.cbt006_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt006_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt006_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt006_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt007_0.host=cbt007
PIC.Core.Cobalt.cbt007_0.cpu=0
PIC.Core.Cobalt.cbt007_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt007_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt007_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt007_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt007_1.host=cbt007
PIC.Core.Cobalt.cbt007_1.cpu=1
PIC.Core.Cobalt.cbt007_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt007_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt007_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt007_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt008_0.host=cbt008
PIC.Core.Cobalt.cbt008_0.cpu=0
PIC.Core.Cobalt.cbt008_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt008_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt008_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt008_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt008_1.host=cbt008
PIC.Core.Cobalt.cbt008_1.cpu=1
PIC.Core.Cobalt.cbt008_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt008_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt008_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt008_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt009_0.host=cbt009
PIC.Core.Cobalt.cbt009_0.cpu=0
PIC.Core.Cobalt.cbt009_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt009_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt009_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt009_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt009_1.host=cbt009
PIC.Core.Cobalt.cbt009_1.cpu=1
PIC.Core.Cobalt.cbt009_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt009_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt009_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt009_1.gpus=[2, 3]
PIC.Core.Cobalt.cbt010_0.host=cbt010
PIC.Core.Cobalt.cbt010_0.cpu=0
PIC.Core.Cobalt.cbt010_0.mpi_nic=mlx4_0
-PIC.Core.Cobalt.cbt010_0.out_nic=[CEP4:ib0,DragNet:ib0]
+PIC.Core.Cobalt.cbt010_0.out_nic=[CEP4:ib0,DRAGNET:ib0]
PIC.Core.Cobalt.cbt010_0.gpus=[0, 1]
PIC.Core.Cobalt.cbt010_1.host=cbt010
PIC.Core.Cobalt.cbt010_1.cpu=1
PIC.Core.Cobalt.cbt010_1.mpi_nic=mlx4_1
-PIC.Core.Cobalt.cbt010_1.out_nic=[CEP4:ib1,DragNet:ib1]
+PIC.Core.Cobalt.cbt010_1.out_nic=[CEP4:ib1,DRAGNET:ib1]
PIC.Core.Cobalt.cbt010_1.gpus=[2, 3]
+
+# The DRAGNET cluster (without dragnet head node and dragproc node)
+
+PIC.Core.DRAGNET.drg01.host=drg01
+PIC.Core.DRAGNET.drg01.cpu=-1
+PIC.Core.DRAGNET.drg01.mpi_nic=
+PIC.Core.DRAGNET.drg01.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg01.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg02.host=drg02
+PIC.Core.DRAGNET.drg02.cpu=-1
+PIC.Core.DRAGNET.drg02.mpi_nic=
+PIC.Core.DRAGNET.drg02.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg02.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg03.host=drg03
+PIC.Core.DRAGNET.drg03.cpu=-1
+PIC.Core.DRAGNET.drg03.mpi_nic=
+PIC.Core.DRAGNET.drg03.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg03.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg04.host=drg04
+PIC.Core.DRAGNET.drg04.cpu=-1
+PIC.Core.DRAGNET.drg04.mpi_nic=
+PIC.Core.DRAGNET.drg04.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg04.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg05.host=drg05
+PIC.Core.DRAGNET.drg05.cpu=-1
+PIC.Core.DRAGNET.drg05.mpi_nic=
+PIC.Core.DRAGNET.drg05.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg05.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg06.host=drg06
+PIC.Core.DRAGNET.drg06.cpu=-1
+PIC.Core.DRAGNET.drg06.mpi_nic=
+PIC.Core.DRAGNET.drg06.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg06.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg07.host=drg07
+PIC.Core.DRAGNET.drg07.cpu=-1
+PIC.Core.DRAGNET.drg07.mpi_nic=
+PIC.Core.DRAGNET.drg07.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg07.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg08.host=drg08
+PIC.Core.DRAGNET.drg08.cpu=-1
+PIC.Core.DRAGNET.drg08.mpi_nic=
+PIC.Core.DRAGNET.drg08.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg08.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg09.host=drg09
+PIC.Core.DRAGNET.drg09.cpu=-1
+PIC.Core.DRAGNET.drg09.mpi_nic=
+PIC.Core.DRAGNET.drg09.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg09.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg10.host=drg10
+PIC.Core.DRAGNET.drg10.cpu=-1
+PIC.Core.DRAGNET.drg10.mpi_nic=
+PIC.Core.DRAGNET.drg10.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg10.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg11.host=drg11
+PIC.Core.DRAGNET.drg11.cpu=-1
+PIC.Core.DRAGNET.drg11.mpi_nic=
+PIC.Core.DRAGNET.drg11.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg11.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg12.host=drg12
+PIC.Core.DRAGNET.drg12.cpu=-1
+PIC.Core.DRAGNET.drg12.mpi_nic=
+PIC.Core.DRAGNET.drg12.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg12.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg13.host=drg13
+PIC.Core.DRAGNET.drg13.cpu=-1
+PIC.Core.DRAGNET.drg13.mpi_nic=
+PIC.Core.DRAGNET.drg13.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg13.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg14.host=drg14
+PIC.Core.DRAGNET.drg14.cpu=-1
+PIC.Core.DRAGNET.drg14.mpi_nic=
+PIC.Core.DRAGNET.drg14.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg14.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg15.host=drg15
+PIC.Core.DRAGNET.drg15.cpu=-1
+PIC.Core.DRAGNET.drg15.mpi_nic=
+PIC.Core.DRAGNET.drg15.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg15.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg16.host=drg16
+PIC.Core.DRAGNET.drg16.cpu=-1
+PIC.Core.DRAGNET.drg16.mpi_nic=
+PIC.Core.DRAGNET.drg16.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg16.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg17.host=drg17
+PIC.Core.DRAGNET.drg17.cpu=-1
+PIC.Core.DRAGNET.drg17.mpi_nic=
+PIC.Core.DRAGNET.drg17.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg17.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg18.host=drg18
+PIC.Core.DRAGNET.drg18.cpu=-1
+PIC.Core.DRAGNET.drg18.mpi_nic=
+PIC.Core.DRAGNET.drg18.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg18.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg19.host=drg19
+PIC.Core.DRAGNET.drg19.cpu=-1
+PIC.Core.DRAGNET.drg19.mpi_nic=
+PIC.Core.DRAGNET.drg19.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg19.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg20.host=drg20
+PIC.Core.DRAGNET.drg20.cpu=-1
+PIC.Core.DRAGNET.drg20.mpi_nic=
+PIC.Core.DRAGNET.drg20.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg20.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg21.host=drg21
+PIC.Core.DRAGNET.drg21.cpu=-1
+PIC.Core.DRAGNET.drg21.mpi_nic=
+PIC.Core.DRAGNET.drg21.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg21.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg22.host=drg22
+PIC.Core.DRAGNET.drg22.cpu=-1
+PIC.Core.DRAGNET.drg22.mpi_nic=
+PIC.Core.DRAGNET.drg22.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg22.gpus=[0, 1, 2, 3]
+
+PIC.Core.DRAGNET.drg23.host=drg23
+PIC.Core.DRAGNET.drg23.cpu=-1
+PIC.Core.DRAGNET.drg23.mpi_nic=
+PIC.Core.DRAGNET.drg23.out_nic=[CEP4:ib0,DRAGNET:ib0]
+PIC.Core.DRAGNET.drg23.gpus=[0, 1, 2, 3]
+
diff --git a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareUsed.parset b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareUsed.parset
index ac5194cf5ea..5509586a494 100644
--- a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareUsed.parset
+++ b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/HardwareUsed.parset
@@ -23,3 +23,26 @@ Cobalt.Nodes = [
cbt008_1
]
+DRAGNET.Nodes = [
+ drg01,
+ drg02,
+ drg03,
+ drg04,
+ drg05,
+ drg06,
+ drg07,
+ drg08,
+ drg09,
+ drg10,
+ drg11,
+ drg12,
+ drg13,
+ drg14,
+ drg15,
+ drg16,
+ drg17,
+ drg18,
+ drg19,
+ drg20
+]
+
diff --git a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/rspraw-enable.parset.OBSID b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/rspraw-enable.parset.OBSID
new file mode 100644
index 00000000000..70b247ec737
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/rspraw-enable.parset.OBSID
@@ -0,0 +1,58 @@
+# rspraw-enable.parset.OBSID: override COBALT settings to enable RSP raw antenna field output
+#
+# Basic Usage: After preparing an observation with desired station and freq settings, but before the MACScheduler starts it up,
+# run as lofarsys on the COBALT head node:
+# cd $LOFARROOT/etc/parset-additions.d/override && cp ../rspraw-enable.parset.OBSID rspraw-enable.parset.123456
+# where 123456 is the observation ID.
+#
+# NOTE: This is a COBALT-only expert mode setting adjustment. While you really have to outstrip yourself to
+# screw up the system beyond this observation, little customization effort is needed to screw up this observation.
+#
+# $Id$
+
+# By default, disable correlated and beamformed data.
+# You can comment out or enable these to *also* get correlated and/or beamformed output;
+# assuming you specified all correlator and/or beamformer settings properly.
+Observation.DataProducts.Output_Correlated.enabled=false
+Observation.DataProducts.Output_Beamformed.enabled=false
+
+
+# Enable RSP raw output
+Observation.DataProducts.Output_RSPRaw.enabled=true
+
+# The following RSP raw settings are commented out by default, because then they are auto-detected from
+# correlated and/or beamformed settings, even if those are disabled just above. Feel free to customize.
+
+# Override start/stop times: Observation runs as specified, but RSP raw data is only written out for this interval.
+# The interval must be within the observation start/stop times interval.
+# Default: observation start/stop time
+#Cobalt.RSPRaw.startTime=2017-01-29 10:00:00
+#Cobalt.RSPRaw.stopTime=2017-01-29 11:00:00
+
+# Override station list: Observation uses stations as specified, but RSP raw data is only written for these stations.
+# The list must be a subset of the observation station set.
+# Default: the full set of observation stations
+#Cobalt.RSPRaw.stationList=[CS002, CS003]
+
+# Override beamlets: Observation uses beam and frequency settings as specified,
+# but RSP raw data is only written for the first N beamlets (whatever beams/subbands that corresponds to...).
+# The value is a list: one value per sending RSP board (stream) for all stations(!), thus typically a list of (at least) length 4.
+# Each value must be within the valid range for the bit mode:
+# 16 bit: [ 61, 61, 61, 61] (thus max 244)
+# 8 bit: [122, 122, 122, 122] (thus max 488)
+# 4 bit: [244, 244, 244, 244] (but the stations can handle a max total of 966 (not 976); unclear how it's spread over the RSP boards)
+# Default: all observation beamlets
+#Cobalt.RSPRaw.nrBeamletsPerBoardList=[122, 122, 122, 122]
+
+# Override filenames and locations where RSP raw data is written to.
+# Note that the hostnames must be on the COBALT/CEP4 infiniband network (10G may also work; you may need to also override the network interface bound to...).
+# Prefer fully qualified domain names (FQDN), or CEP4:/path (per file) and the system will assign CEP4 nodes.
+# Note that you must provide (at least) enough filenames and locations! Typically, 4x the number of antenna fields for RSP raw output.
+# WARNING: Auto-detection fills in the observation ID, but if you override you cannot:
+# another observation with the same customized override file will overwrite previously dumped data!
+# Default: Hostnames in locations: round robin over the set of correlated + beamformed hostnames.
+# Filenames format: Lxxx_yyy_zzz_rsp.raw where xxx=OBS_ID, yyy=ANT_FIELD_NAME, zzz=BOARD_NR (+ Lxxx_yyy_zzz_rsp.raw.parset files).
+#Observation.DataProducts.Output_RSPRaw.filenames=[L123456_CS002HBA0_0_rsp.raw, L123456_CS002HBA0_1_rsp.raw, L123456_CS002HBA0_2_rsp.raw, L123456_CS002HBA0_3_rsp.raw, L123456_CS002HBA1_0_rsp.raw, L123456_CS002HBA1_1_rsp.raw, L123456_CS002HBA1_2_rsp.raw, L123456_CS002HBA1_3_rsp.raw, L123456_CS003HBA0_0_rsp.raw, L123456_CS003HBA0_1_rsp.raw, L123456_CS003HBA0_2_rsp.raw, L123456_CS003HBA0_3_rsp.raw, L123456_CS003HBA1_0_rsp.raw, L123456_CS003HBA1_1_rsp.raw, L123456_CS003HBA1_2_rsp.raw, L123456_CS003HBA1_3_rsp.raw]
+#Observation.DataProducts.Output_RSPRaw.locations=[CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs,CEP4:/data/projects/2017LOFAROBS/L123456/cs]
+#Observation.DataProducts.Output_RSPRaw.storageClusterName=CEP4
+
diff --git a/RTCP/Cobalt/GPUProc/src/MPIReceiver.h b/RTCP/Cobalt/GPUProc/src/MPIReceiver.h
index 4ca0040b425..eeccb9516c7 100644
--- a/RTCP/Cobalt/GPUProc/src/MPIReceiver.h
+++ b/RTCP/Cobalt/GPUProc/src/MPIReceiver.h
@@ -21,8 +21,8 @@
// \file
// Include for processor optimalizetion functionality
-#ifndef LOFAR_GPUPROC_MPI_UTILS_H
-#define LOFAR_GPUPROC_MPI_UTILS_H
+#ifndef LOFAR_GPUPROC_MPI_RECEIVER_H
+#define LOFAR_GPUPROC_MPI_RECEIVER_H
#include <InputProc/Transpose/MPIUtil.h>
#include <InputProc/SampleType.h>
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
index 7d2775bec32..438b1254048 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
@@ -1,5 +1,5 @@
//# StationInput.cc: Routines to manage I/O from the stations.
-//# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2012-2013, 2017 ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -34,6 +34,8 @@
#include <map>
#include <vector>
#include <string>
+#include <algorithm>
+#include <memory>
#include <boost/format.hpp>
#include <Common/LofarLogger.h>
@@ -158,7 +160,8 @@ namespace LOFAR {
StationInput::StationInput( const Parset &ps, size_t stationIdx,
- const SubbandDistribution &subbandDistribution )
+ const SubbandDistribution &subbandDistribution,
+ unsigned hostID )
:
ps(ps),
stationIdx(stationIdx),
@@ -175,14 +178,45 @@ namespace LOFAR {
targetSubbands(values(subbandDistribution)),
beamletIndices(generateBeamletIndices())
{
- for (size_t i = 0; i < nrBoards; ++i) {
- rspDataPool.push_back(new Pool<RSPData>(str(format("StationInput::rspDataPool[%u] [station %s]") % i % stationID.name()), ps.settings.realTime));
- }
+ ASSERTSTR(nrBoards > 0, logPrefix << "No input streams");
// Log all input descriptions
LOG_INFO_STR(logPrefix << "Input streams: " << ps.settings.antennaFields.at(stationIdx).inputStreams);
- ASSERTSTR(nrBoards > 0, logPrefix << "No input streams");
+ for (unsigned i = 0; i < nrBoards; ++i) {
+ rspDataPool.push_back(new Pool<RSPData>(str(format("StationInput::rspDataPool[%u] [station %s]") % i % stationID.name()),
+ ps.settings.realTime));
+ }
+
+ if (ps.settings.rspRaw.enabled &&
+ std::find(ps.settings.rspRaw.antennaFieldNames.begin(), ps.settings.rspRaw.antennaFieldNames.end(),
+ stationID.name()) != ps.settings.rspRaw.antennaFieldNames.end()) {
+ unsigned maxNrPacketsToSend = NONRT_PACKET_BATCH_SIZE;
+ time_t connTimeout = 0;
+ if (ps.settings.realTime) {
+ maxNrPacketsToSend = RT_PACKET_BATCH_SIZE;
+ connTimeout = std::time(NULL) + 3; // a few secs from now
+ }
+
+ rspRawSenders.resize(nrBoards);
+ #pragma omp parallel for num_threads(nrBoards) // easy way to apply a single connTimeout to all
+ for (unsigned i = 0; i < nrBoards; ++i) {
+ unsigned fileIdx = ps.getRSPRawOutputStreamIdx(stationID.name(), i);
+// TODO: produce UDP stream descriptor for piggy backing instead of TCP for RSP raw storage
+ string desc = getStreamDescriptorBetweenIONandStorage(ps, RSP_RAW_DATA, fileIdx,
+ hostID < ps.settings.nodes.size() ? ps.settings.nodes[hostID].out_nic : "");
+ LOG_INFO_STR(logPrefix << "Opening RSP raw output stream for data from antenna field " <<
+ stationID.name() << " board " << i << " with stream descriptor " << desc);
+
+ try {
+ new (&rspRawSenders[i]) RSPRawSender(maxNrPacketsToSend, ps.settings.rspRaw.nrBeamletsPerBoardList.at(i),
+ desc, connTimeout); // pre-C++11 has no emplace_back(), so use placement new
+ } catch (Exception &exc) { // TimeOutException, SystemCallException
+ LOG_ERROR_STR(logPrefix << "RSPRawSender creation failure for stream " << desc << ": " << exc);
+ new (&rspRawSenders[i]) RSPRawSender;
+ }
+ }
+ }
}
@@ -211,8 +245,8 @@ namespace LOFAR {
// The corresponding (board,slot) combination for that subband,
// for this station.
- const size_t board = ps.settings.antennaFields[stationIdx].rspBoardMap[sb];
- const size_t slot = ps.settings.antennaFields[stationIdx].rspSlotMap[sb];
+ const unsigned board = ps.settings.antennaFields[stationIdx].rspBoardMap[sb];
+ const unsigned slot = ps.settings.antennaFields[stationIdx].rspSlotMap[sb];
ASSERT(board < nrBoards);
ASSERT(slot < mode.nrBeamletsPerBoard());
@@ -226,7 +260,7 @@ namespace LOFAR {
}
- SmartPtr<Stream> StationInput::inputStream(size_t board) const
+ SmartPtr<Stream> StationInput::inputStream(unsigned board) const
{
SmartPtr<Stream> stream;
@@ -267,7 +301,7 @@ namespace LOFAR {
}
- void StationInput::readRSPRealTime( size_t board, MACIO::RTmetadata &mdLogger,
+ void StationInput::readRSPRealTime( unsigned board, MACIO::RTmetadata &mdLogger,
const string &mdKeyPrefix )
{
/*
@@ -345,7 +379,7 @@ namespace LOFAR {
} else {
// One core can't handle the load, so use multiple
# pragma omp parallel for num_threads(nrBoards)
- for (size_t board = 0; board < nrBoards; board++) {
+ for (unsigned board = 0; board < nrBoards; board++) {
//NSTimer copyRSPTimer(str(format("%s [board %i] copy RSP -> block") % logPrefix % board), true, true);
OMPThread::ScopedName sn(str(format("%s wr %u") % ps.settings.antennaFields.at(stationIdx).name % board));
@@ -361,25 +395,44 @@ namespace LOFAR {
// Write valid packets to the current and/or next packet
//copyRSPTimer.start();
- for (size_t p = 0; p < RT_PACKET_BATCH_SIZE; ++p) {
- struct RSP &packet = rspData->packets[p];
+ if (ps.settings.correlator.enabled || ps.settings.beamFormer.enabled) {
+ for (unsigned p = 0; p < RT_PACKET_BATCH_SIZE; ++p) {
+ struct RSP &packet = rspData->packets[p];
- if (packet.payloadError())
- continue;
+ if (packet.payloadError())
+ continue;
- if (current.write(packet, beamletIndices, nrBeamletIndices)
- && next) {
- // We have data (potentially) spilling into `next'.
+ if (current.write(packet, beamletIndices, nrBeamletIndices) && next) {
+ // We have data (potentially) spilling into `next'.
- if (next->write(packet, beamletIndices, nrBeamletIndices)) {
- // Only emit log lines every minute seconds to prevent spam
- if (loggedSeenFutureData + mode.secondsToSamples(60) < now) {
- LOG_ERROR_STR(logPrefix << "Received data for several blocks into the future -- discarding.");
- loggedSeenFutureData = now;
+ if (next->write(packet, beamletIndices, nrBeamletIndices)) {
+ // Only emit log lines every minute seconds to prevent spam
+ if (loggedSeenFutureData + mode.secondsToSamples(60) < now) {
+ LOG_ERROR_STR(logPrefix << "Received data for several blocks into the future -- discarding.");
+ loggedSeenFutureData = now;
+ }
}
}
}
}
+
+ if (!rspRawSenders.empty() && rspRawSenders[board].initialized()) {
+ // Quick check on timestamp
+ // Don't cater for out-of-order input: the extra logic is not worth the benefit.
+ // We may not always get all RT_PACKET_BATCH_SIZE. PacketReader sets payloadError and
+ // timestamp as invalid in remaining entries. Such timestamps are filtered here too.
+ unsigned nrPackets = RT_PACKET_BATCH_SIZE;
+ for (unsigned p = 0; p < nrPackets; p++) {
+ if (rspData->packets[p].header.timestamp >= ps.settings.rspRaw.stopTime ||
+ rspData->packets[p].header.timestamp < ps.settings.rspRaw.startTime) {
+ nrPackets = p;
+ break;
+ }
+ }
+ if (nrPackets > 0) {
+ rspRawSenders[board].trySend(&rspData->packets[0], nrPackets);
+ }
+ }
//copyRSPTimer.stop();
outputQueue.append(rspData);
@@ -415,7 +468,7 @@ namespace LOFAR {
vector<bool> read_next_packet(nrBoards, true);
for(;;) {
- for(size_t board = 0; board < nrBoards; board++) {
+ for (unsigned board = 0; board < nrBoards; board++) {
if (!read_next_packet[board] || !readers[board])
continue;
@@ -433,18 +486,18 @@ namespace LOFAR {
}
// Determine which board provided the youngest packet
- int youngest = -1;
+ unsigned youngest = nrBoards; // init to special (invalid) value
- for (size_t board = 0; board < nrBoards; board++) {
+ for (unsigned board = 0; board < nrBoards; board++) {
if (!readers[board])
continue;
- if (youngest == -1 || last_packets[youngest].timeStamp() > last_packets[board].timeStamp())
+ if (youngest == nrBoards || last_packets[youngest].timeStamp() > last_packets[board].timeStamp())
youngest = board;
}
// Break if all streams turned out to be inactive
- if (youngest == -1)
+ if (youngest == nrBoards)
break;
// Emit youngest packet
@@ -463,8 +516,8 @@ namespace LOFAR {
// Next packet should only be read from the stream we
// emitted from
- for(size_t board = 0; board < nrBoards; board++)
- read_next_packet[board] = (board == (size_t)youngest);
+ for (unsigned board = 0; board < nrBoards; board++)
+ read_next_packet[board] = (board == youngest);
}
// Signal EOD by inserting a packet beyond obs end
@@ -509,13 +562,22 @@ namespace LOFAR {
// Only packet 0 is used in non-rt mode
- if (current.write(data->packets[0], beamletIndices, nrBeamletIndices)) {
- // We have data (potentially) spilling into `next'.
- if (!next || next->write(data->packets[0], beamletIndices, nrBeamletIndices)) {
- // Data is even later than next? Put this data back for a future block.
- rspDataPool[0]->filled.prepend(data);
- ASSERT(!data);
- return;
+ if (ps.settings.correlator.enabled || ps.settings.beamFormer.enabled) {
+ if (current.write(data->packets[0], beamletIndices, nrBeamletIndices)) {
+ // We have data (potentially) spilling into `next'.
+ if (!next || next->write(data->packets[0], beamletIndices, nrBeamletIndices)) {
+ // Data is even later than next? Put this data back for a future block.
+ rspDataPool[0]->filled.prepend(data);
+ ASSERT(!data);
+ return;
+ }
+ }
+ }
+
+ if (!rspRawSenders.empty() && rspRawSenders[data->board].initialized()) {
+ if (data->packets[0].header.timestamp < ps.settings.rspRaw.stopTime &&
+ data->packets[0].header.timestamp >= ps.settings.rspRaw.startTime) {
+ rspRawSenders[data->board].trySend(&data->packets[0], NONRT_PACKET_BATCH_SIZE);
}
}
@@ -573,7 +635,7 @@ namespace LOFAR {
if (ps.settings.realTime) {
#pragma omp parallel for num_threads(nrBoards)
- for(size_t board = 0; board < nrBoards; board++) {
+ for (unsigned board = 0; board < nrBoards; board++) {
try {
OMPThreadSet::ScopedRun sr(packetReaderThreads);
OMPThread::ScopedName sn(str(format("%s rd %u") % ps.settings.antennaFields.at(stationIdx).name % board));
@@ -651,13 +713,13 @@ namespace LOFAR {
template<typename SampleT> void sendInputToPipeline(const Parset &ps,
- size_t stationIdx, const SubbandDistribution &subbandDistribution,
+ size_t stationIdx, const SubbandDistribution &subbandDistribution, unsigned hostID,
MACIO::RTmetadata &mdLogger, const string &mdKeyPrefix, Trigger *stopSwitch)
{
// sanity check: Find out if we should actual start working here.
StationMetaData<SampleT> sm(ps, stationIdx, subbandDistribution);
- StationInput si(ps, stationIdx, subbandDistribution);
+ StationInput si(ps, stationIdx, subbandDistribution, hostID);
const struct StationID stationID(StationID::parseFullFieldName(
ps.settings.antennaFields.at(stationIdx).name));
@@ -721,30 +783,29 @@ namespace LOFAR {
}
void sendInputToPipeline(const Parset &ps, size_t stationIdx,
- const SubbandDistribution &subbandDistribution,
+ const SubbandDistribution &subbandDistribution, unsigned hostID,
MACIO::RTmetadata &mdLogger, const string &mdKeyPrefix, Trigger *stopSwitch)
{
switch (ps.nrBitsPerSample()) {
default:
case 16:
sendInputToPipeline< SampleType<i16complex> >(ps, stationIdx,
- subbandDistribution,
+ subbandDistribution, hostID,
mdLogger, mdKeyPrefix, stopSwitch);
break;
case 8:
sendInputToPipeline< SampleType< i8complex> >(ps, stationIdx,
- subbandDistribution,
+ subbandDistribution, hostID,
mdLogger, mdKeyPrefix, stopSwitch);
break;
case 4:
sendInputToPipeline< SampleType< i4complex> >(ps, stationIdx,
- subbandDistribution,
+ subbandDistribution, hostID,
mdLogger, mdKeyPrefix, stopSwitch);
break;
}
}
}
}
-
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationInput.h b/RTCP/Cobalt/GPUProc/src/Station/StationInput.h
index b3f655b4dde..45a2b88ef09 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationInput.h
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationInput.h
@@ -1,5 +1,5 @@
//# StationInput.h: Routines to manage I/O from the stations.
-//# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2012-2013, 2017 ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -34,10 +34,11 @@
#include <CoInterface/SubbandMetaData.h>
#include <CoInterface/Queue.h>
#include <CoInterface/BestEffortQueue.h>
+#include <CoInterface/RSP.h>
+#include <CoInterface/RSPRawTransfer.h>
#include <InputProc/Buffer/StationID.h>
#include <InputProc/Buffer/BoardMode.h>
#include <InputProc/RSPTimeStamp.h>
-#include <InputProc/Station/RSP.h>
#include "StationTranspose.h"
@@ -76,7 +77,8 @@ namespace LOFAR {
class StationInput {
public:
StationInput( const Parset &ps, size_t stationIdx,
- const SubbandDistribution &subbandDistribution );
+ const SubbandDistribution &subbandDistribution,
+ unsigned hostID = 0 );
template <typename SampleT>
void processInput( Queue< SmartPtr< MPIData<SampleT> > > &inputQueue,
@@ -86,14 +88,14 @@ namespace LOFAR {
private:
// Each packet is expected to have 16 samples per subband, i.e. ~80 us worth of data @ 200 MHz.
// So 512 packets is ~40 ms of data.
- static const unsigned RT_PACKET_BATCH_SIZE = 512;
+ static const unsigned RT_PACKET_BATCH_SIZE = 512; // RSP raw sendmmsg(2) needs this to be <= UIO_MAXIOV (1024)
static const unsigned NONRT_PACKET_BATCH_SIZE = 1;
// Data received from an RSP board
struct RSPData {
std::vector<struct RSP> packets;
- size_t board; // annotation used in non-rt mode
+ unsigned board; // annotation used in non-rt mode
RSPData(size_t numPackets):
packets(numPackets)
@@ -109,8 +111,9 @@ namespace LOFAR {
const std::string logPrefix;
const BoardMode mode;
- const size_t nrBoards;
- std::vector< SmartPtr< Pool< RSPData > > > rspDataPool; // [nrboards]
+ const unsigned nrBoards;
+ std::vector< SmartPtr< Pool< RSPData > > > rspDataPool; // [nrBoards]
+ std::vector<RSPRawSender> rspRawSenders; // [nrBoards] if RSP raw enabled and the antenna field is selected, else []
// Whether we emitted certain errors (to prevent log spam)
TimeStamp loggedSeenFutureData;
@@ -125,7 +128,7 @@ namespace LOFAR {
MultiDimArray<ssize_t, 2> generateBeamletIndices();
- SmartPtr<Stream> inputStream(size_t board) const;
+ SmartPtr<Stream> inputStream(unsigned board) const;
/*
* Reads data from all the station input streams, and puts their packets in rspDataPool.
@@ -145,7 +148,7 @@ namespace LOFAR {
*
* Read data from one board in real-time mode.
*/
- void readRSPRealTime( size_t board, MACIO::RTmetadata &mdLogger,
+ void readRSPRealTime( unsigned board, MACIO::RTmetadata &mdLogger,
const std::string &mdKeyPrefix );
/*
@@ -166,6 +169,7 @@ namespace LOFAR {
void sendInputToPipeline(const Parset &ps, size_t stationIdx,
const SubbandDistribution &subbandDistribution,
+ unsigned hostID, // mpi rank for this node
MACIO::RTmetadata &mdLogger,
const std::string &mdKeyPrefix,
Trigger *stopSwitch = NULL);
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h
index 6df17f21c03..16211ae2b93 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h
@@ -29,8 +29,8 @@
#include <CoInterface/Parset.h>
#include <CoInterface/Queue.h>
#include <CoInterface/SubbandMetaData.h>
+#include <CoInterface/RSP.h>
#include <InputProc/RSPTimeStamp.h>
-#include <InputProc/Station/RSP.h>
#include <InputProc/Transpose/MPIProtocol.h>
#include <InputProc/Transpose/MPIUtil.h>
diff --git a/RTCP/Cobalt/GPUProc/src/cuda/Pipelines/Pipeline.cc b/RTCP/Cobalt/GPUProc/src/cuda/Pipelines/Pipeline.cc
index 67e24f2d7b5..51370ec4a41 100644
--- a/RTCP/Cobalt/GPUProc/src/cuda/Pipelines/Pipeline.cc
+++ b/RTCP/Cobalt/GPUProc/src/cuda/Pipelines/Pipeline.cc
@@ -146,7 +146,8 @@ namespace LOFAR
// be in bulk: if processing is cheap, all subbands will be output right after they have been received.
//
// Allow queue to drop items older than 3 seconds.
- multiSender(hostMap(ps, subbandIndices, hostID), ps, 3.0, hostID < ps.settings.nodes.size() ? ps.settings.nodes.at(hostID).out_nic : ""),
+ multiSender(hostMap(ps, subbandIndices, hostID), ps, 3.0,
+ hostID < ps.settings.nodes.size() ? ps.settings.nodes[hostID].out_nic : ""),
hostID(hostID)
{
ASSERTSTR(!devices.empty(), "Not bound to any GPU!");
@@ -830,7 +831,7 @@ namespace LOFAR
if (ps.settings.correlator.enabled) {
const string desc = getStreamDescriptorBetweenIONandStorage(ps, CORRELATED_DATA, globalSubbandIdx,
- hostID < ps.settings.nodes.size() ? ps.settings.nodes.at(hostID).out_nic : "");
+ hostID < ps.settings.nodes.size() ? ps.settings.nodes[hostID].out_nic : "");
try {
outputStream = createStream(desc, false, 0);
diff --git a/RTCP/Cobalt/GPUProc/src/rtcp.cc b/RTCP/Cobalt/GPUProc/src/rtcp.cc
index 5195a218163..20ee60d73c7 100644
--- a/RTCP/Cobalt/GPUProc/src/rtcp.cc
+++ b/RTCP/Cobalt/GPUProc/src/rtcp.cc
@@ -285,11 +285,13 @@ int main(int argc, char **argv)
if(mpi.rank() >= 0 && (size_t)mpi.rank() < ps.settings.nodes.size()) {
struct ObservationSettings::Node mynode = ps.settings.nodes.at(mpi.rank());
- // set the processor affinity before any threads are created
- setProcessorAffinity(mynode.cpu);
+ if (mynode.cpu != -1) {
+ // set the processor affinity before any threads are created
+ setProcessorAffinity(mynode.cpu);
+ }
#ifdef HAVE_LIBNUMA
- if (numa_available() != -1) {
+ if (mynode.cpu != -1 && numa_available() != -1) {
// force node + memory binding for future allocations
struct bitmask *numa_node = numa_allocate_nodemask();
numa_bitmask_clearall(numa_node);
@@ -315,7 +317,7 @@ int main(int argc, char **argv)
LOG_DEBUG_STR("Bound to memory on nodes " << nodestrs);
} else {
- LOG_WARN("Cannot bind memory (libnuma says there is no numa available)");
+ LOG_INFO("Cannot bind memory: cpu nr to bind to is set to -1 or libnuma reports NUMA is not available");
}
#else
LOG_WARN("Cannot bind memory (no libnuma support)");
@@ -388,12 +390,13 @@ int main(int argc, char **argv)
// Creation of pipelines cause fork/exec, which we need to
// do before we start doing anything fancy with libraries and threads.
- if (subbandIndices.empty()) {
- // no operation -- don't even create a pipeline!
- pipeline = NULL;
- } else {
+ if ((ps.settings.correlator.enabled || ps.settings.beamFormer.enabled) &&
+ !subbandIndices.empty()) {
pipeline = new Pipeline(ps, subbandIndices, devices,
MPI_receive_pool, mdLogger, mdKeyPrefix, mpi.rank());
+ } else {
+ // RSP raw data output or piggy-backing only, or software test without pipeline
+ pipeline = NULL;
}
// After pipeline creation (post-fork()), allow creation of a thread to send
@@ -520,7 +523,7 @@ int main(int argc, char **argv)
mdLogger.log(mdKeyPrefixInputProc + PN_CSI_CPU, cpuNr);
- sendInputToPipeline(ps, stat, subbandDistribution,
+ sendInputToPipeline(ps, stat, subbandDistribution, mpi.rank(),
mdLogger, mdKeyPrefixInputProc, &stopSwitch);
}
}
@@ -538,8 +541,9 @@ int main(int argc, char **argv)
{
OMPThread::ScopedName sn("obs process");
- // Process station data
- if (!subbandDistribution[mpi.rank()].empty()) {
+ // Process station data if needed and if any
+ if ((ps.settings.correlator.enabled || ps.settings.beamFormer.enabled) &&
+ !subbandDistribution[mpi.rank()].empty()) {
pipeline->processObservation();
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/scripts/cobalt_functions.sh b/RTCP/Cobalt/GPUProc/src/scripts/cobalt_functions.sh
index 263ec2aef3b..a7a79ecfcd8 100755
--- a/RTCP/Cobalt/GPUProc/src/scripts/cobalt_functions.sh
+++ b/RTCP/Cobalt/GPUProc/src/scripts/cobalt_functions.sh
@@ -36,7 +36,7 @@ function setkey {
echo "$KEY = $VAL" >> "$PARSET"
}
-COBALT_DATAPRODUCTS="Correlated CoherentStokes IncoherentStokes"
+COBALT_DATAPRODUCTS="Correlated CoherentStokes IncoherentStokes RSPRaw"
function read_cluster_model {
# HACK: Search for first cluster, and assume they're all the same. We support only output
diff --git a/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc b/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc
index a5ae57a1491..88673b74b50 100644
--- a/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc
+++ b/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc
@@ -106,7 +106,7 @@ int main(int argc, char **argv)
}
MACIO::RTmetadata rtmd(ps.settings.observationID, "", "");
- sendInputToPipeline(ps, stat, subbandDistribution,
+ sendInputToPipeline(ps, stat, subbandDistribution, mpi.rank(),
rtmd, "rtmd key prefix");
cout << "First ended" << endl;
}
diff --git a/RTCP/Cobalt/InputProc/src/Station/Generator.h b/RTCP/Cobalt/InputProc/src/Station/Generator.h
index 22fcdcde5cb..e3d4831cae2 100644
--- a/RTCP/Cobalt/InputProc/src/Station/Generator.h
+++ b/RTCP/Cobalt/InputProc/src/Station/Generator.h
@@ -26,13 +26,13 @@
#include <Stream/Stream.h>
#include <CoInterface/SmartPtr.h>
+#include <CoInterface/RSP.h>
#include <InputProc/RSPBoards.h>
#include <InputProc/Buffer/StationID.h>
#include <InputProc/RSPTimeStamp.h>
#include "PacketFactory.h"
-#include "RSP.h"
namespace LOFAR
{
diff --git a/RTCP/Cobalt/InputProc/src/Station/PacketFactory.h b/RTCP/Cobalt/InputProc/src/Station/PacketFactory.h
index 14d0166febc..900f7ec31cc 100644
--- a/RTCP/Cobalt/InputProc/src/Station/PacketFactory.h
+++ b/RTCP/Cobalt/InputProc/src/Station/PacketFactory.h
@@ -22,11 +22,10 @@
#ifndef LOFAR_INPUT_PROC_PACKETFACTORY_H
#define LOFAR_INPUT_PROC_PACKETFACTORY_H
+#include <CoInterface/RSP.h>
#include <InputProc/RSPTimeStamp.h>
#include <InputProc/Buffer/BoardMode.h>
-#include "RSP.h"
-
namespace LOFAR
{
namespace Cobalt
diff --git a/RTCP/Cobalt/InputProc/src/Station/PacketReader.cc b/RTCP/Cobalt/InputProc/src/Station/PacketReader.cc
index 1047c1fa2d4..8cad4fddd52 100644
--- a/RTCP/Cobalt/InputProc/src/Station/PacketReader.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/PacketReader.cc
@@ -40,18 +40,17 @@ namespace LOFAR
PacketReader::PacketReader( const std::string &logPrefix, Stream &inputStream,
const BoardMode &mode )
- :
- logPrefix(str(boost::format("%s [PacketReader] ") % logPrefix)),
+ : mode(mode),
inputStream(inputStream),
- mode(mode),
+ logPrefix(str(boost::format("%s [PacketReader] ") % logPrefix)),
+ hadSizeError(false),
nrReceived(0),
nrBadMode(0),
nrBadTime(0),
nrBadData(0),
nrBadOther(0),
- hadSizeError(false),
- lastLogTime(0)
+ lastLogTime(0.0)
{
// Partial reads are not supported on UDP streams, because each read()
// will consume a full packet.
@@ -74,7 +73,9 @@ namespace LOFAR
if (inputIsUDP) {
SocketStream &sstream = dynamic_cast<SocketStream&>(inputStream);
- vector<unsigned> recvdSizes(packets.size());
+ if (recvdSizes.size() != packets.size()) {
+ recvdSizes.resize(packets.size());
+ }
numRead = sstream.recvmmsg(&packets[0], sizeof(struct RSP), recvdSizes);
nrReceived += numRead;
@@ -93,6 +94,7 @@ namespace LOFAR
// mark unused packet buffers as invalid
for (size_t i = numRead; i < packets.size(); ++i) {
packets[i].payloadError(true);
+ packets[i].timeStampError(); // easier for RSP raw code to filter
}
}
@@ -134,7 +136,7 @@ namespace LOFAR
}
// illegal version means illegal packet
- if (packet.header.version < 2) {
+ if (packet.header.version < 3) {
// This mainly catches packets that are all zero (f.e. /dev/zero or
// null: streams).
++nrBadOther;
diff --git a/RTCP/Cobalt/InputProc/src/Station/PacketReader.h b/RTCP/Cobalt/InputProc/src/Station/PacketReader.h
index 2140a4c2717..323a31047ed 100644
--- a/RTCP/Cobalt/InputProc/src/Station/PacketReader.h
+++ b/RTCP/Cobalt/InputProc/src/Station/PacketReader.h
@@ -22,13 +22,13 @@
#define LOFAR_INPUT_PROC_PACKETREADER_H
#include <string>
+#include <vector>
#include <Common/Exception.h>
#include <Stream/SocketStream.h>
#include <MACIO/RTmetadata.h>
-#include <InputProc/Buffer/BoardMode.h>
-
-#include "RSP.h"
+#include <CoInterface/RSP.h>
+#include "../Buffer/BoardMode.h"
namespace LOFAR
{
@@ -62,29 +62,32 @@ namespace LOFAR
const std::string &mdKeyPrefix);
private:
- const std::string logPrefix;
+ // The mode against which to validate (ignored if mode == MODE_ANY)
+ const BoardMode mode;
// The stream from which packets are read.
Stream &inputStream;
- // The mode against which to validate (ignored if mode == MODE_ANY)
- const BoardMode mode;
+ // For SocketStream recvmmsg() to indicate max nr packets to receive and to return bytes sent.
+ std::vector<unsigned> recvdSizes;
+
+ const std::string logPrefix;
// Whether inputStream is an UDP stream
// UDP streams do not allow partial reads and can use recvmmsg(2) (Linux).
bool inputIsUDP;
// Statistics covering the packets read so far
+ bool hadSizeError; // already reported about wrongly sized packets since last logStatistics()
size_t nrReceived; // nr. of packets received
size_t nrBadMode; // nr. of packets with wrong mode (clock, bit mode)
size_t nrBadTime; // nr. of packets with an illegal time stamp
size_t nrBadData; // nr. of packets with payload errors
size_t nrBadOther; // nr. of packets that are bad in another fashion (illegal header, packet size, etc)
- bool hadSizeError; // already reported about wrongly sized packets since last logStatistics()
-
double lastLogTime; // time since last log print, to monitor data rates
+
// numbytes is the actually received size, as indicated by the kernel
bool validatePacket(const struct RSP &packet, size_t numbytes);
};
diff --git a/RTCP/Cobalt/InputProc/src/Station/PacketStream.h b/RTCP/Cobalt/InputProc/src/Station/PacketStream.h
index 0e61ba0eb32..38c5baf1179 100644
--- a/RTCP/Cobalt/InputProc/src/Station/PacketStream.h
+++ b/RTCP/Cobalt/InputProc/src/Station/PacketStream.h
@@ -24,9 +24,9 @@
#include <Stream/Stream.h>
#include <Common/Thread/Cancellation.h>
-#include <InputProc/RSPTimeStamp.h>
+#include <CoInterface/RSP.h>
+#include "../RSPTimeStamp.h"
#include "PacketFactory.h"
-#include "RSP.h"
namespace LOFAR
{
@@ -53,23 +53,26 @@ namespace LOFAR
{
Cancellation::point();
- if (current >= to)
+ if (size == 0) {
+ return 0;
+ }
+
+ if (current >= to) {
THROW(EndOfStreamException, "No data beyond " << to);
+ }
if (offset == 0) {
// generate new packet
factory.makePacket(packet, current, boardNr);
-
current += packet.header.nrBlocks;
}
- size_t numBytes = std::min(packet.packetSize() - offset, size);
-
+ size_t pktSize = packet.packetSize();
+ size_t numBytes = std::min(pktSize - offset, size);
memcpy(ptr, reinterpret_cast<char*>(&packet) + offset, numBytes);
offset += numBytes;
-
- if (offset == packet.packetSize()) {
+ if (offset == pktSize) {
// written full packet, so we'll need a new one on next read
offset = 0;
}
@@ -77,13 +80,60 @@ namespace LOFAR
return numBytes;
}
- virtual size_t tryWrite(const void *ptr, size_t size)
+ virtual size_t tryWrite(const void * /*ptr*/, size_t /*size*/)
{
- // not supported
- (void)ptr;
- (void)size;
+ THROW(NotImplemented, "Writing to PacketStream is not supported");
+ }
- THROW(EndOfStreamException, "Writing to PacketStream is not supported");
+ virtual size_t tryReadv(const struct iovec *iov, int iovcnt)
+ {
+ Cancellation::point();
+
+ size_t nread = 0;
+ for (int i = 0; i < iovcnt; i++) {
+ if (iov[i].iov_len == 0) {
+ continue;
+ }
+
+ if (current >= to) {
+ if (nread == 0) {
+ THROW(EndOfStreamException, "No data beyond " << to);
+ } else {
+ break;
+ }
+ }
+
+ if (offset == 0) {
+ // generate new packet
+ factory.makePacket(packet, current, boardNr);
+ current += packet.header.nrBlocks;
+ }
+
+ size_t pktSize = packet.packetSize();
+ size_t numBytes = std::min(pktSize - offset, iov[i].iov_len);
+ memcpy(iov[i].iov_base, reinterpret_cast<char*>(&packet) + offset, numBytes);
+
+ offset += numBytes;
+ if (offset == pktSize) {
+ // written full packet, so we'll need a new one on next read
+ offset = 0;
+ }
+
+ nread += numBytes;
+
+ // Mimic tryRead() impl above: max 1 (partial) packet per buffer.
+ // Then we can only use the next iov if we could exactly fill the previous, else our retval is ambiguous.
+ if (numBytes < pktSize) {
+ break;
+ }
+ }
+
+ return nread;
+ }
+
+ virtual size_t tryWritev(const struct iovec * /*iov*/, int /*iovcnt*/)
+ {
+ THROW(NotImplemented, "Writing to PacketStream is not supported");
}
private:
diff --git a/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc b/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc
index 80dacfdf124..a08a7c694b8 100644
--- a/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc
@@ -33,7 +33,7 @@
#include <Stream/StreamFactory.h>
#include <ApplCommon/PosixTime.h>
#include <CoInterface/SmartPtr.h>
-#include "RSP.h"
+#include <CoInterface/RSP.h>
#include "PacketReader.h"
using namespace LOFAR;
diff --git a/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc b/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc
index 26c453f790e..a22df4b097f 100644
--- a/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc
@@ -33,9 +33,9 @@
#include <ApplCommon/PosixTime.h>
#include <Stream/StreamFactory.h>
#include <CoInterface/SmartPtr.h>
+#include <CoInterface/RSP.h>
#include <InputProc/Buffer/BoardMode.h>
#include <InputProc/RSPTimeStamp.h>
-#include <InputProc/Station/RSP.h>
#include <InputProc/Station/RSPPacketFactory.h>
using namespace std;
diff --git a/RTCP/Cobalt/InputProc/src/Station/printRSP.cc b/RTCP/Cobalt/InputProc/src/Station/printRSP.cc
index 4ad2daeddcc..1cecf4aabef 100644
--- a/RTCP/Cobalt/InputProc/src/Station/printRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/printRSP.cc
@@ -21,8 +21,6 @@
//# Always #include <lofar_config.h> first!
#include <lofar_config.h>
-#include "RSP.h"
-
#include <ctime>
#include <cstring>
#include <cstdlib>
@@ -33,6 +31,7 @@
#include <Common/LofarLogger.h>
#include <Common/DataConvert.h>
#include <Stream/FileStream.h>
+#include <CoInterface/RSP.h>
#include <InputProc/RSPTimeStamp.h>
using namespace LOFAR;
diff --git a/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc b/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc
index d45caa76b51..26c5d28e608 100644
--- a/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc
@@ -33,7 +33,7 @@
#include <ApplCommon/PosixTime.h>
#include <Stream/StreamFactory.h>
#include <CoInterface/SmartPtr.h>
-#include "RSP.h"
+#include <CoInterface/RSP.h>
#include "PacketReader.h"
using namespace LOFAR;
diff --git a/RTCP/Cobalt/InputProc/test/tPacketReader.cc b/RTCP/Cobalt/InputProc/test/tPacketReader.cc
index a689800de0a..3f5676bcdea 100644
--- a/RTCP/Cobalt/InputProc/test/tPacketReader.cc
+++ b/RTCP/Cobalt/InputProc/test/tPacketReader.cc
@@ -25,8 +25,8 @@
#include <Common/LofarLogger.h>
#include <Stream/FileStream.h>
+#include <CoInterface/RSP.h>
#include <InputProc/Station/PacketReader.h>
-#include <InputProc/Station/RSP.h>
using namespace LOFAR;
using namespace Cobalt;
diff --git a/RTCP/Cobalt/InputProc/test/tRSP.cc b/RTCP/Cobalt/InputProc/test/tRSP.cc
index 9c7ade0a028..91a68fefd4f 100644
--- a/RTCP/Cobalt/InputProc/test/tRSP.cc
+++ b/RTCP/Cobalt/InputProc/test/tRSP.cc
@@ -28,8 +28,8 @@
#include <Common/DataConvert.h>
#include <Stream/FileStream.h>
+#include <CoInterface/RSP.h>
#include <InputProc/RSPTimeStamp.h>
-#include <InputProc/Station/RSP.h>
using namespace LOFAR;
diff --git a/RTCP/Cobalt/OutputProc/src/CMakeLists.txt b/RTCP/Cobalt/OutputProc/src/CMakeLists.txt
index 1646bd68ba4..656663906c0 100644
--- a/RTCP/Cobalt/OutputProc/src/CMakeLists.txt
+++ b/RTCP/Cobalt/OutputProc/src/CMakeLists.txt
@@ -23,6 +23,7 @@ lofar_add_library(outputproc
CommonLofarAttributes.cc
OutputThread.cc
SubbandWriter.cc
+ RSPRawWriter.cc
TBB_StaticMapping.cc
)
diff --git a/RTCP/Cobalt/OutputProc/src/CommonLofarAttributes.cc b/RTCP/Cobalt/OutputProc/src/CommonLofarAttributes.cc
index 1c1d9832a7a..51b0c229835 100644
--- a/RTCP/Cobalt/OutputProc/src/CommonLofarAttributes.cc
+++ b/RTCP/Cobalt/OutputProc/src/CommonLofarAttributes.cc
@@ -135,7 +135,7 @@ namespace LOFAR
}
// max freq is shifted if 2nd PPF is active for all types in obs
// min freq is shifted if 2nd PPF is active for any type in obs
- bool applyPPFtoMax = nrAppliedPPFs == parset.nrObsOutputTypes();
+ bool applyPPFtoMax = nrAppliedPPFs == parset.nrProcessedOutputTypes();
bool applyPPFtoMin = nrAppliedPPFs > 0;
vector<double> subbandCenterFrequencies(parset.settings.subbands.size());
diff --git a/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc b/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
index 649be8f7002..330ab5abb29 100644
--- a/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
+++ b/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
@@ -1,5 +1,6 @@
//# GPUProcIO.cc: Routines for communicating with GPUProc
-//# Copyright (C) 2008-2014 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2008-2014, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -51,6 +52,7 @@
#include <CoInterface/SmartPtr.h>
#include <CoInterface/SelfDestructTimer.h>
#include "SubbandWriter.h"
+#include "RSPRawWriter.h"
#include "OutputThread.h"
using namespace LOFAR;
@@ -154,6 +156,7 @@ bool process(Stream &controlStream)
vector<SmartPtr<SubbandWriter> > subbandWriters;
vector<SmartPtr<TABOutputThread> > tabWriters;
+ vector<SmartPtr<RSPRawWriter> > rspRawWriters;
/*
* Construct writers
@@ -198,7 +201,7 @@ bool process(Stream &controlStream)
continue;
}
- const unsigned allFileIdx = fileIdx + parset.settings.correlator.files.size();
+ const unsigned allFileIdx = fileIdx + (parset.settings.correlator.enabled ? parset.settings.correlator.files.size() : 0);
mdLogger.log(mdKeyPrefix + PN_COP_STORAGE_HOST + '[' + lexical_cast<string>(allFileIdx) + ']', myHostName);
LOG_INFO_STR("Allocating transpose buffers for " << file.location.filename);
@@ -244,6 +247,34 @@ bool process(Stream &controlStream)
}
}
+ if (parset.settings.rspRaw.enabled) {
+ for (size_t fileIdx = 0; fileIdx < parset.settings.rspRaw.files.size(); ++fileIdx)
+ {
+ struct ObservationSettings::RSPRaw::File &file = parset.settings.rspRaw.files[fileIdx];
+
+ if (myHostNames.find(file.location.host) == myHostNames.end()) {
+ continue;
+ }
+
+ LOG_INFO_STR("starting with RSP raw fileIdx " << fileIdx);
+
+ // The rest of the system doesn't know about RSP raw data output, but if monitoring did, enable this:
+ /*
+ const unsigned allFileIdx = fileIdx + (parset.settings.correlator.enabled ? parset.settings.correlator.files.size() : 0) +
+ (parset.settings.beamFormer.enabled ? parset.settings.beamFormer.files.size() : 0);
+ mdLogger.log(mdKeyPrefix + PN_COP_STORAGE_HOST + '[' + lexical_cast<string>(allFileIdx) + ']', myHostName);
+ */
+
+ string logPrefix = str(format("[obs %u RSP raw stream %3u] ")
+ % parset.settings.observationID % fileIdx);
+
+ RSPRawWriter *writer = new RSPRawWriter(parset, fileIdx, mdLogger, mdKeyPrefix, logPrefix);
+ rspRawWriters.push_back(writer);
+
+ LOG_INFO_STR("done with RSP raw fileIdx " << fileIdx);
+ }
+ }
+
LOG_INFO_STR("Finished setting up writers");
/*
@@ -256,7 +287,7 @@ bool process(Stream &controlStream)
TABTranspose::MultiReceiver mr("2nd-transpose-", collectors);
-# pragma omp parallel sections num_threads(3)
+# pragma omp parallel sections num_threads(4)
{
// Done signal from controller, by sending the final meta data
# pragma omp section
@@ -312,6 +343,18 @@ bool process(Stream &controlStream)
tabWriters[i]->process();
}
}
+
+ // RSPRawWriters
+# pragma omp section
+ {
+ OMPThread::ScopedName sn("rspRawWr");
+
+# pragma omp parallel for num_threads(rspRawWriters.size())
+ for (int i = 0; i < (int)rspRawWriters.size(); ++i) {
+ OMPThread::ScopedName sn(str(format("rspRawWr %u") % rspRawWriters[i]->streamNr()));
+ rspRawWriters[i]->process();
+ }
+ }
}
/*
@@ -361,6 +404,21 @@ bool process(Stream &controlStream)
bus.send(msg);
}
+ // The rest of the system doesn't know about RSP raw data output, but if feedback did, enable this:
+ /*
+ for (size_t i = 0; i < rspRawWriters.size(); ++i) {
+ Protocols::TaskFeedbackDataproducts msg(
+ myName,
+ "",
+ str(boost::format("Feedback for RSP Raw Data, file nr %s") % rspRawWriters[i]->streamNr()),
+ str(format("%s") % parset.settings.momID),
+ str(format("%s") % parset.settings.observationID),
+ rspRawWriters[i]->feedbackLTA());
+
+ bus.send(msg);
+ }
+ */
+
/*
* SIGN OFF
*/
diff --git a/RTCP/Cobalt/OutputProc/src/GPUProcIO.h b/RTCP/Cobalt/OutputProc/src/GPUProcIO.h
index ba1ec282829..dbab9c828e5 100644
--- a/RTCP/Cobalt/OutputProc/src/GPUProcIO.h
+++ b/RTCP/Cobalt/OutputProc/src/GPUProcIO.h
@@ -1,5 +1,6 @@
//# GPUProcIO.h
-//# Copyright (C) 2009-2014 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2009-2014, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -34,7 +35,7 @@ namespace LOFAR
// Receive and process a full observation. Will:
// * Receive a Parset over the controlStream
// * Fulfill roles for parset.settings.outputProcHosts[myRank]:
- // - Start SubbandWriters/TABWriters
+ // - Start SubbandWriters/TABWriters/RSPRawWriters
// - Receive input and write output for all of them
// * Call readFinalMetaData to obtain the final metadata from GPUProc,
// and send it to all writers.
diff --git a/RTCP/Cobalt/OutputProc/src/InputThread.cc b/RTCP/Cobalt/OutputProc/src/InputThread.cc
index d53c81bac9d..b9cd5d40fab 100644
--- a/RTCP/Cobalt/OutputProc/src/InputThread.cc
+++ b/RTCP/Cobalt/OutputProc/src/InputThread.cc
@@ -1,5 +1,6 @@
//# InputThread.cc
-//# Copyright (C) 2008-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2008-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -34,18 +35,19 @@ namespace LOFAR
{
namespace Cobalt
{
- InputThread::InputThread(const Parset &parset,
+ InputThread::InputThread(const Parset &parset, OutputType outputType,
unsigned streamNr, Pool<StreamableData> &outputPool,
const std::string &logPrefix)
:
itsLogPrefix(logPrefix + "[InputThread] "),
itsNrIntegrationsReceived(0),
itsNrIntegrations(parset.settings.correlator.nrIntegrations),
- itsInputDescriptor(getStreamDescriptorBetweenIONandStorage(parset, CORRELATED_DATA, streamNr)),
+ itsInputDescriptor(getStreamDescriptorBetweenIONandStorage(parset, outputType, streamNr)),
itsOutputPool(outputPool),
itsDeadline(parset.settings.realTime ? parset.settings.stopTime : 0)
{
- ASSERT(parset.settings.correlator.enabled);
+ ASSERT((parset.settings.correlator.enabled && outputType == CORRELATED_DATA) ||
+ (parset.settings.rspRaw.enabled && outputType == RSP_RAW_DATA));
}
@@ -57,10 +59,10 @@ namespace LOFAR
LOG_INFO_STR(itsLogPrefix << "Creating connection from " << itsInputDescriptor << ": done" );
for(SmartPtr<StreamableData> data; (data = itsOutputPool.free.remove()) != NULL; itsOutputPool.filled.append(data)) {
- data->read(streamFromION, true, 1); // Cobalt writes with an alignment of 1
+ data->read(streamFromION, 1); // Cobalt writes with an alignment of 1
++itsNrIntegrationsReceived;
- LOG_DEBUG_STR(itsLogPrefix << "Received integration " << data->sequenceNumber());
+ LOG_DEBUG_STR(itsLogPrefix << "Received data block with seq nr " << data->sequenceNumber());
}
} catch (TimeOutException &) {
LOG_WARN_STR(itsLogPrefix << "Connection from " << itsInputDescriptor << " timed out");
diff --git a/RTCP/Cobalt/OutputProc/src/InputThread.h b/RTCP/Cobalt/OutputProc/src/InputThread.h
index 85b561240d4..515aa20dcf8 100644
--- a/RTCP/Cobalt/OutputProc/src/InputThread.h
+++ b/RTCP/Cobalt/OutputProc/src/InputThread.h
@@ -1,5 +1,6 @@
//# InputThread.h
-//# Copyright (C) 2008-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2008-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -28,6 +29,7 @@
#include <CoInterface/Parset.h>
#include <CoInterface/Pool.h>
#include <CoInterface/StreamableData.h>
+#include <CoInterface/OutputTypes.h>
namespace LOFAR
@@ -42,12 +44,13 @@ namespace LOFAR
* The Stream is created from the parset through using
* getStreamDescriptorBetweenIONandStorage.
*
- * This class is designed to handle visibilities only.
+ * This class is designed to handle visibility (UV) and RSP raw data.
*/
class InputThread
{
public:
InputThread(const Parset &parset,
+ OutputType outputType,
unsigned streamNr,
Pool<StreamableData> &outputPool,
const std::string &logPrefix);
@@ -57,7 +60,8 @@ namespace LOFAR
private:
const std::string itsLogPrefix;
- // we receive integration "blocks"
+ // We receive integration "blocks" (UV);
+ // for RSP raw it is a stream of (up to) some block size bytes.
size_t itsNrIntegrationsReceived;
const size_t itsNrIntegrations;
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterFile.cc b/RTCP/Cobalt/OutputProc/src/MSWriterFile.cc
index ebbea6c74b6..e789a08069c 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterFile.cc
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterFile.cc
@@ -1,5 +1,6 @@
//# MSWriterFile.cc: a raw file writer
-//# Copyright (C) 2009-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2009-2013, 2016
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -90,7 +91,7 @@ namespace LOFAR
void MSWriterFile::write(StreamableData *data)
{
- data->write(&itsFile, true, 512);
+ data->write(&itsFile, 512);
}
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc b/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc
index 4260d4c06a8..f1043edc4e5 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc
@@ -51,7 +51,7 @@ namespace LOFAR
void MSWriterNull::write(StreamableData *data)
{
- // We do not know why the creation of the propper writer failed.
+ // We do not know why the creation of the proper writer failed.
// Assume nothing and only report that we did not write anything
itsConfiguration.replace("percentageWritten", str(format("%u") % 0));
itsConfiguration.replace("size", str(format("%u") % getDataSize()));
diff --git a/RTCP/Cobalt/OutputProc/src/OutputThread.cc b/RTCP/Cobalt/OutputProc/src/OutputThread.cc
index 5dc58114d21..0b96d5a7e29 100644
--- a/RTCP/Cobalt/OutputProc/src/OutputThread.cc
+++ b/RTCP/Cobalt/OutputProc/src/OutputThread.cc
@@ -1,5 +1,6 @@
//# OutputThread.cc:
-//# Copyright (C) 2009-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2009-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -33,6 +34,7 @@
#include <Common/Thread/Cancellation.h>
#include <ApplCommon/PVSSDatapointDefs.h>
+#include <CoInterface/Parset.h>
#include <CoInterface/OutputTypes.h>
#include <CoInterface/Exceptions.h>
#include <CoInterface/LTAFeedback.h>
@@ -248,7 +250,9 @@ namespace LOFAR
}
# pragma omp section
- init();
+ {
+ init();
+ }
}
}
@@ -287,33 +291,27 @@ namespace LOFAR
const std::string fileName = itsParset.getFileName(CORRELATED_DATA, itsStreamNr);
const std::string path = directoryName + "/" + fileName;
+ LOG_INFO_STR(itsLogPrefix << "Writing correlated data to " << path);
- try
- {
- LOG_INFO_STR(itsLogPrefix << "Writing to " << path);
-
- itsWriter = new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr);
+ if (itsParset.settings.realTime) {
+ try {
+ itsWriter = new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr);
- logInitialStreamMetadataEvents("Correlated", fileName, directoryName);
- }
- catch (Exception &ex)
- {
- LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex);
- if (!itsParset.settings.realTime)
- THROW(StorageException, ex);
+ logInitialStreamMetadataEvents("Correlated", fileName, directoryName);
+ } catch (Exception &ex) {
+ LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex);
+ itsWriter = new MSWriterNull(itsParset);
- itsWriter = new MSWriterNull(itsParset);
#if defined HAVE_AIPSPP
- }
- catch (casa::AipsError &ex)
- {
- LOG_ERROR_STR(itsLogPrefix << "Caught AipsError: " << ex.what());
-
- if (!itsParset.settings.realTime)
- THROW(StorageException, ex.what());
-
- itsWriter = new MSWriterNull(itsParset);
+ } catch (casa::AipsError &ex) {
+ LOG_ERROR_STR(itsLogPrefix << "Caught AipsError: " << ex.what());
+ itsWriter = new MSWriterNull(itsParset);
#endif
+ }
+ } else { // don't handle exception in non-RT: it is fatal: avoid rethrow for a clean stracktrace
+ itsWriter = new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr);
+
+ logInitialStreamMetadataEvents("Correlated", fileName, directoryName);
}
itsNrExpectedBlocks = itsParset.settings.correlator.nrIntegrations;
@@ -351,39 +349,98 @@ namespace LOFAR
const std::string fileName = itsParset.getFileName(BEAM_FORMED_DATA, itsStreamNr);
const std::string path = directoryName + "/" + fileName;
+ LOG_INFO_STR(itsLogPrefix << "Writing beamformed data to " << path);
- try
- {
- LOG_INFO_STR(itsLogPrefix << "Writing to " << path);
+ if (itsParset.settings.realTime) {
+ try {
+#ifdef HAVE_DAL
+ itsWriter = new MSWriterDAL<float,3>(path, itsParset, itsStreamNr);
+#else
+ itsWriter = new MSWriterFile(path);
+#endif
+ logInitialStreamMetadataEvents("Beamformed", fileName, directoryName);
+ } catch (Exception &ex) {
+ LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex);
+ itsWriter = new MSWriterNull(itsParset);
+
+#if defined HAVE_AIPSPP
+ } catch (casa::AipsError &ex) {
+ LOG_ERROR_STR(itsLogPrefix << "Caught AipsError: " << ex.what());
+ itsWriter = new MSWriterNull(itsParset);
+#endif
+ }
+ } else { // don't handle exception in non-RT: it is fatal: avoid rethrow for a clean stracktrace
#ifdef HAVE_DAL
itsWriter = new MSWriterDAL<float,3>(path, itsParset, itsStreamNr);
#else
itsWriter = new MSWriterFile(path);
#endif
-
logInitialStreamMetadataEvents("Beamformed", fileName, directoryName);
}
- catch (Exception &ex)
- {
- LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex);
- if (!itsParset.settings.realTime)
- THROW(StorageException, ex);
- itsWriter = new MSWriterNull(itsParset);
-#if defined HAVE_AIPSPP
- }
- catch (casa::AipsError &ex)
- {
- LOG_ERROR_STR(itsLogPrefix << "Caught AipsError: " << ex.what());
- if ( !itsParset.settings.realTime)
- THROW(StorageException, ex.what());
+ itsNrExpectedBlocks = itsParset.settings.nrBlocks();
+ }
- itsWriter = new MSWriterNull(itsParset);
-#endif
+
+ RSPRawOutputThread::RSPRawOutputThread(const Parset &parset,
+ unsigned streamNr, Pool<StreamableData> &outputPool,
+ RTmetadata &mdLogger, const std::string &mdKeyPrefix,
+ const std::string &logPrefix, const std::string &targetDirectory)
+ :
+ OutputThread<StreamableData>(
+ parset,
+ streamNr,
+ outputPool,
+ mdLogger,
+ mdKeyPrefix,
+ logPrefix + "[RSPRawOutputThread] ",
+ targetDirectory)
+ {
+ }
+
+ void RSPRawOutputThread::createMS()
+ {
+ // Unlike the other output types, there is no need to grab casacoreMutex
+ // or delay cancellation, because the RSP raw writer does not use casacore or libhdf5.
+
+ const std::string directoryName =
+ itsTargetDirectory == ""
+ ? itsParset.getDirectoryName(RSP_RAW_DATA, itsStreamNr)
+ : itsTargetDirectory;
+ const std::string fileName = itsParset.getFileName(CORRELATED_DATA, itsStreamNr);
+
+ const std::string path = directoryName + "/" + fileName;
+ LOG_INFO_STR(itsLogPrefix << "Writing RSP raw data to " << path);
+
+ // Write parset as observation metadata. We end up with many duplicates, but at least we know all storage nodes used have it.
+ // Also patch antenna field stream locations (originals in StationStreams.parset) for easy offline reprocessing.
+ Parset rspRawParset = itsParset;
+//TODO: patch locations, like
+//PIC.Core.CS006HBA.RSP.ports = [udp:cbt004-10GB01:10060, udp:cbt004-10GB01:10061, udp:cbt004-10GB01:10062, udp:cbt004-10GB01:10063]
+//PIC.Core.CS006HBA.RSP.receiver = cbt004_0
+
+ if (itsParset.settings.realTime) {
+ try {
+ rspRawParset.writeFile(fileName + ".parset");
+ itsWriter = new MSWriterFile(path);
+
+ // The rest of the system doesn't know about RSP raw data output, but if monitoring did, enable this:
+ //logInitialStreamMetadataEvents("RSPRaw", fileName, directoryName);
+ } catch (Exception& ex) {
+ LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex);
+ itsWriter = new MSWriterNull(itsParset);
+ }
+ } else { // don't handle exception in non-RT: it is fatal: avoid rethrow for a clean stracktrace
+ rspRawParset.writeFile(fileName + ".parset");
+ itsWriter = new MSWriterFile(path);
+
+ // The rest of the system doesn't know about RSP raw data output, but if monitoring did, enable this:
+ //logInitialStreamMetadataEvents("RSPRaw", fileName, directoryName);
}
- itsNrExpectedBlocks = itsParset.settings.nrBlocks();
+//TODO: see if we can get the data dropping stats in a useful state for RSP raw
+ itsNrExpectedBlocks = itsParset.settings.nrBlocks() * itsParset.settings.blockSize;
}
} // namespace Cobalt
} // namespace LOFAR
diff --git a/RTCP/Cobalt/OutputProc/src/OutputThread.h b/RTCP/Cobalt/OutputProc/src/OutputThread.h
index b272680ba0c..93e8395960b 100644
--- a/RTCP/Cobalt/OutputProc/src/OutputThread.h
+++ b/RTCP/Cobalt/OutputProc/src/OutputThread.h
@@ -1,5 +1,6 @@
//# OutputThread.h
-//# Copyright (C) 2009-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Copyright (C) 2009-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -140,6 +141,22 @@ namespace LOFAR
};
+ /*
+ * RSPRawOutputThread specialises in creating *_rsp.raw
+ * files for raw RSP antenna field data.
+ */
+ class RSPRawOutputThread: public OutputThread<StreamableData>
+ {
+ public:
+ RSPRawOutputThread(const Parset &parset, unsigned streamNr,
+ Pool<StreamableData> &outputPool,
+ RTmetadata &mdLogger, const std::string &mdKeyPrefix,
+ const std::string &logPrefix,
+ const std::string &targetDirectory = "");
+
+ virtual void createMS();
+ };
+
} // namespace Cobalt
} // namespace LOFAR
diff --git a/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc b/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc
new file mode 100644
index 00000000000..b42aeec6273
--- /dev/null
+++ b/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc
@@ -0,0 +1,93 @@
+//# RSPRawWriter.cc: Write raw data stream of an RSP board to storage
+//# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include "RSPRawWriter.h"
+
+//#include <CoInterface/CorrelatedData.h>
+#include <CoInterface/OMPThread.h>
+#include <Common/Timer.h>
+
+#include <boost/format.hpp>
+using boost::format;
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+ RSPRawWriter::RSPRawWriter(const Parset &parset, unsigned streamNr,
+ RTmetadata &mdLogger, const std::string &mdKeyPrefix,
+ const std::string &logPrefix)
+ :
+ itsStreamNr(streamNr),
+ itsOutputPool(str(format("RSPRawWriter::itsOutputPool [stream %u]") % streamNr), parset.settings.realTime),
+ itsInputThread(parset, RSP_RAW_DATA, streamNr, itsOutputPool, logPrefix),
+ itsOutputThread(parset, streamNr, itsOutputPool, mdLogger, mdKeyPrefix, logPrefix)
+ {
+ NSTimer timer(str(format("preallocator %u") % itsStreamNr), true, true);
+
+ for (unsigned i = 0; i < preAllocateReceiveQueue; i++) {
+ timer.start();
+
+ LOG_DEBUG_STR(str(format("[stream %u] Allocating element %u") % itsStreamNr % i));
+ RSPRawData *data = new RSPRawData();
+ LOG_DEBUG_STR(str(format("[stream %u] Appending element %u") % itsStreamNr % i));
+ itsOutputPool.free.append(data);
+
+ timer.stop();
+ }
+ }
+
+
+ void RSPRawWriter::process()
+ {
+# pragma omp parallel sections num_threads(2)
+ {
+# pragma omp section
+ {
+ OMPThread::ScopedName sn(str(format("RSPRaw input %u") % itsStreamNr));
+
+ itsInputThread.process();
+ }
+
+# pragma omp section
+ {
+ OMPThread::ScopedName sn(str(format("RSPRaw output %u") % itsStreamNr));
+
+ itsOutputThread.process();
+ }
+ }
+ }
+
+
+ void RSPRawWriter::fini( const FinalMetaData &finalMetaData )
+ {
+ itsOutputThread.fini(finalMetaData);
+ }
+
+
+ ParameterSet RSPRawWriter::feedbackLTA() const
+ {
+ return itsOutputThread.feedbackLTA();
+ }
+ } // namespace Cobalt
+} // namespace LOFAR
+
diff --git a/RTCP/Cobalt/OutputProc/src/RSPRawWriter.h b/RTCP/Cobalt/OutputProc/src/RSPRawWriter.h
new file mode 100644
index 00000000000..2f1b45245d0
--- /dev/null
+++ b/RTCP/Cobalt/OutputProc/src/RSPRawWriter.h
@@ -0,0 +1,75 @@
+//# RSPRawWriter.h: Write raw data stream of an RSP board to storage
+//# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_STORAGE_RSPBOARDWRITER_H
+#define LOFAR_STORAGE_RSPBOARDWRITER_H
+
+#include <string>
+
+#include <CoInterface/Parset.h>
+#include <CoInterface/Pool.h>
+#include <CoInterface/StreamableData.h>
+#include <CoInterface/FinalMetaData.h>
+#include "InputThread.h"
+#include "OutputThread.h"
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+ /*
+ * RSPRawWriter is responsible for completely handling the reception
+ * and writing of one stream of data from an RSP board (via InputProc/GPUProc).
+ *
+ * It maintains an InputThread and RSPRawOutputThread, connected by
+ * an internal Pool<> of data blocks.
+ */
+ class RSPRawWriter
+ {
+ public:
+ RSPRawWriter(const Parset &parset,
+ unsigned streamNr,
+ RTmetadata &mdLogger,
+ const std::string &mdKeyPrefix,
+ const std::string &logPrefix);
+
+ void process();
+
+ void fini(const FinalMetaData &finalMetaData);
+
+ ParameterSet feedbackLTA() const;
+
+ unsigned streamNr() const { return itsStreamNr; }
+
+ private:
+ static const unsigned preAllocateReceiveQueue = 32; // number of elements to construct before starting
+
+ const unsigned itsStreamNr;
+
+ Pool<StreamableData> itsOutputPool;
+
+ InputThread itsInputThread;
+ RSPRawOutputThread itsOutputThread;
+ };
+ } // namespace Cobalt
+} // namespace LOFAR
+
+#endif
+
diff --git a/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc b/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc
index 20a616788a3..10e20410033 100644
--- a/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc
+++ b/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc
@@ -1,5 +1,6 @@
-//# SubbandWriter.cc: Writes visibilities and beam-formed data
-//# Copyright (C) 2008-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# SubbandWriter.cc: Write a subband of visibility data (UV) to storage
+//# Copyright (C) 2008-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -42,7 +43,7 @@ namespace LOFAR
itsArena(0),
itsAllocator(0),
itsOutputPool(str(format("SubbandWriter::itsOutputPool [stream %u]") % streamNr), parset.settings.realTime),
- itsInputThread(parset, streamNr, itsOutputPool, logPrefix),
+ itsInputThread(parset, CORRELATED_DATA, streamNr, itsOutputPool, logPrefix),
itsOutputThread(parset, streamNr, itsOutputPool, mdLogger, mdKeyPrefix, logPrefix),
itsAlignment(512),
itsNrStations(parset.settings.correlator.stations.size()),
@@ -94,14 +95,14 @@ namespace LOFAR
# pragma omp section
{
- OMPThread::ScopedName sn(str(format("input %u") % itsStreamNr));
+ OMPThread::ScopedName sn(str(format("uv input %u") % itsStreamNr));
itsInputThread.process();
}
# pragma omp section
{
- OMPThread::ScopedName sn(str(format("output %u") % itsStreamNr));
+ OMPThread::ScopedName sn(str(format("uv output %u") % itsStreamNr));
itsOutputThread.process();
}
diff --git a/RTCP/Cobalt/OutputProc/src/SubbandWriter.h b/RTCP/Cobalt/OutputProc/src/SubbandWriter.h
index 10eaa727098..9127148ec04 100644
--- a/RTCP/Cobalt/OutputProc/src/SubbandWriter.h
+++ b/RTCP/Cobalt/OutputProc/src/SubbandWriter.h
@@ -1,5 +1,6 @@
-//# SubbandWriter.h: Write visibilites and beam-formed data
-//# Copyright (C) 2008-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# SubbandWriter.h: Write a subband of visibility data (UV) to storage
+//# Copyright (C) 2008-2013, 2017
+//# ASTRON (Netherlands Institute for Radio Astronomy)
//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
//#
//# This file is part of the LOFAR software suite.
@@ -23,7 +24,6 @@
#include <string>
-#include <CoInterface/OutputTypes.h>
#include <CoInterface/Parset.h>
#include <CoInterface/Pool.h>
#include <CoInterface/Allocator.h>
diff --git a/RTCP/Cobalt/OutputProc/src/plotMS.cc b/RTCP/Cobalt/OutputProc/src/plotMS.cc
index e98d90c5cde..ca1e47ceebd 100644
--- a/RTCP/Cobalt/OutputProc/src/plotMS.cc
+++ b/RTCP/Cobalt/OutputProc/src/plotMS.cc
@@ -186,7 +186,7 @@ int main(int argc, char *argv[])
for(;; ) {
try {
- data->read(&datafile, true, 512);
+ data->read(&datafile, 512);
} catch (EndOfStreamException &) {
break;
}
diff --git a/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc b/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc
index d51f882ab71..23afabfd4e9 100644
--- a/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc
+++ b/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc
@@ -118,7 +118,7 @@ SUITE(SubbandWriter)
*(data.visibilities.origin() + i) = complex<float>(i, 2*i);
}
- data.write(inputStream, true, 1);
+ data.write(inputStream, 1);
}
}
@@ -128,7 +128,7 @@ SUITE(SubbandWriter)
CorrelatedData data(ps.nrMergedStations(), ps.settings.correlator.nrChannels, ps.settings.correlator.nrSamplesPerIntegration(), heapAllocator, 512);
- data.read(&f, true, 512);
+ data.read(&f, 512);
for (size_t i = 0; i < data.visibilities.num_elements(); ++i) {
CHECK_EQUAL(complex<float>(i, 2*i), *(data.visibilities.origin() + i));
--
GitLab