diff --git a/.gitattributes b/.gitattributes
index ef7a92a854780828ce249c76142171cfad299a42..b7f643a561a073604bf84e1dc23eb9d5aae18827 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3607,6 +3607,7 @@ RTCP/Storage/src/OutputThread.cc -text
RTCP/Storage/src/TBB_StaticMapping.cc -text
RTCP/Storage/src/TBB_Writer.cc -text
RTCP/Storage/src/TBB_Writer_main.cc -text
+RTCP/Storage/src/createHeaders.cc -text
RTCP/Storage/src/gnuplotMS.sh -text
RTCP/Storage/src/plotMS.cc -text
RTCP/Storage/test/tDAL.cc -text
diff --git a/RTCP/Interface/test/CMakeLists.txt b/RTCP/Interface/test/CMakeLists.txt
index 391e1a16d323a55db0ecee4751f9b3fa7735eef0..8edef60161e31e3d0b15acbf21cf030d2caaca60 100644
--- a/RTCP/Interface/test/CMakeLists.txt
+++ b/RTCP/Interface/test/CMakeLists.txt
@@ -4,3 +4,4 @@ include(LofarCTest)
lofar_add_test(tSparseSet tSparseSet.cc)
lofar_add_test(tRSPTimeStamp tRSPTimeStamp.cc)
+lofar_add_test(tMultiDimArray tMultiDimArray.cc)
diff --git a/RTCP/Run/src/RTCP.parset b/RTCP/Run/src/RTCP.parset
index 2584988ca000c2843f949fed33c56023cb80ad5c..ddd47f3c915b758ebd12afdbce56c01bad6eacf0 100644
--- a/RTCP/Run/src/RTCP.parset
+++ b/RTCP/Run/src/RTCP.parset
@@ -8,7 +8,7 @@ Observation.Campaign.name = "Development"
Observation.Campaign.title = "Development"
# ----- Input streams
-Observation.VirtualInstrument.stationList = [S9,S10]
+Observation.VirtualInstrument.stationList = [CS302]
#Observation.antennaSet=LBA_INNER
#Observation.antennaSet=LBA_OUTER
#Observation.antennaSet=LBA_X
@@ -33,7 +33,7 @@ OLAP.nrTimesInFrame = 16
# [ 3*(0;1;2;3) ] --> [ 0,1,2,3,0,1,2,3,0,1,2,3 ]
# [ 3*(300..303) ] --> [ 300,301,302,303,300,301,302,303,300,301,302,303 ]
# [ 2*(5*0) ] --> [ 0,0,0,0,0,0,0,0,0,0 ]
-Observation.subbandList = [100..102]
+Observation.subbandList = [100..159]
Observation.sampleClock = 200
OLAP.nrBitsPerSample = 16
@@ -49,12 +49,12 @@ Observation.Beam[0].angle1 = 5.2336866848083394 # Cygnus
Observation.Beam[0].angle2 = 0.71094251447010637
Observation.Beam[0].directionType = J2000
-Observation.Beam[0].nrTiedArrayBeams = 1
+Observation.Beam[0].nrTiedArrayBeams = 0
Observation.Beam[0].TiedArrayBeam[0].angle1 = 0
Observation.Beam[0].TiedArrayBeam[0].angle2 = 0
Observation.Beam[0].TiedArrayBeam[0].directionType = J2000
Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure = 0
-Observation.Beam[0].TiedArrayBeam[0].coherent = F
+Observation.Beam[0].TiedArrayBeam[0].coherent = T
#Observation.Beam[0].TiedArrayBeam[1].angle1 = 0
#Observation.Beam[0].TiedArrayBeam[1].angle2 = 0
#Observation.Beam[0].TiedArrayBeam[1].directionType = J2000
@@ -76,12 +76,12 @@ Observation.DataProducts.Output_Correlated.namemask = L${OBSID}_SB${SUBBAND}_uv.
Observation.DataProducts.Output_Beamformed.namemask = L${OBSID}_B${BEAM}_S${STOKES}_P${PART}_bf.raw
Observation.DataProducts.Output_Trigger.namemask = L${OBSID}_B${BEAM}_S${STOKES}_P${PART}_bf.trigger
-Observation.channelsPerSubband = 64
+Observation.channelsPerSubband = 1
-OLAP.delayCompensation = T
-OLAP.coherentDedisperseChannels = T
+OLAP.delayCompensation = F
+OLAP.coherentDedisperseChannels = F
-OLAP.correctBandPass = T
+OLAP.correctBandPass = F
#Observation.bandFilter = LBA_10_90
Observation.bandFilter = LBA_30_90
#Observation.bandFilter = HBA_110_190
@@ -92,8 +92,8 @@ OLAP.Correlator.integrationTime = 1.0
#OLAP.IONProc.integrationSteps = 1 # (generated)
#OLAP.CNProc.integrationSteps = 768 # (generated, multiple of 128 for pencil beams)
-OLAP.CNProc_CoherentStokes.subbandsPerFile = 244
-OLAP.CNProc_CoherentStokes.which = I # I or IQUV or XXYY
+OLAP.CNProc_CoherentStokes.subbandsPerFile = 15
+OLAP.CNProc_CoherentStokes.which = XXYY # I or IQUV or XXYY
OLAP.CNProc_CoherentStokes.timeIntegrationFactor = 1 # time integration
OLAP.CNProc_CoherentStokes.channelsPerSubband = 0 # 0 = same as observation
@@ -103,7 +103,7 @@ OLAP.CNProc_IncoherentStokes.timeIntegrationFactor = 1 # time integration
OLAP.CNProc_IncoherentStokes.channelsPerSubband = 0 # 0 = same as observation
# ----- Beam forming
-OLAP.PencilInfo.flysEye = F # don't form pencil beams, but turn stations
+OLAP.PencilInfo.flysEye = T # don't form pencil beams, but turn stations
# into pencil beams
OLAP.PencilInfo.nrRings = 0 # 0 = just the central beam
@@ -119,7 +119,7 @@ OLAP.PencilInfo.ringSize = 0.01
OLAP.OLAP_Conn.IONProc_CNProc_Transport = FCNP
# generate fake station input, overriding any real data
-OLAP.CNProc.fakeInputData = F
+OLAP.CNProc.fakeInputData = T
OLAP.CNProc.checkFakeInputData = F
OLAP.CNProc.nrPPFTaps = 16
diff --git a/RTCP/Storage/include/Storage/OutputThread.h b/RTCP/Storage/include/Storage/OutputThread.h
index 53c543d2e1c2b5032da8adf5aee23194606b9143..4650197bb2ae5df8e6964b4cda9c46f615ff50a3 100644
--- a/RTCP/Storage/include/Storage/OutputThread.h
+++ b/RTCP/Storage/include/Storage/OutputThread.h
@@ -44,12 +44,13 @@ namespace RTCP {
class OutputThread
{
public:
- OutputThread(const Parset &, OutputType, unsigned streamNr, Queue<SmartPtr<StreamableData> > &freeQueue, Queue<SmartPtr<StreamableData> > &receiveQueue, const std::string &logPrefix, bool isBigEndian);
+ OutputThread(const Parset &, OutputType, unsigned streamNr, Queue<SmartPtr<StreamableData> > &freeQueue, Queue<SmartPtr<StreamableData> > &receiveQueue, const std::string &logPrefix, bool isBigEndian, const std::string &targetDirectory = "");
void start();
- private:
void createMS();
+
+ private:
void flushSequenceNumbers();
void writeSequenceNumber(StreamableData *);
void checkForDroppedData(StreamableData *);
@@ -63,6 +64,7 @@ class OutputThread
const bool itsIsBigEndian;
const std::string itsLogPrefix;
const bool itsCheckFakeData;
+ const std::string itsTargetDirectory;
Queue<SmartPtr<StreamableData> > &itsFreeQueue, &itsReceiveQueue;
diff --git a/RTCP/Storage/src/CMakeLists.txt b/RTCP/Storage/src/CMakeLists.txt
index 7e23bc9f5c8defa135d9ec1168a6313c5569b271..c4d7d09fac061eec08484e123b2099bc6c741e61 100644
--- a/RTCP/Storage/src/CMakeLists.txt
+++ b/RTCP/Storage/src/CMakeLists.txt
@@ -28,6 +28,7 @@ install(FILES
DESTINATION etc)
lofar_add_bin_program(Storage_main Storage_main.cc)
+lofar_add_bin_program(createHeaders createHeaders.cc)
lofar_add_bin_program(plotMS plotMS.cc)
lofar_add_bin_program(versionstorage versionstorage.cc)
#lofar_add_bin_program(TBB_Writer_main TBB_Writer_main.cc)
diff --git a/RTCP/Storage/src/OutputThread.cc b/RTCP/Storage/src/OutputThread.cc
index 818fc50af44e62900b6ffab852d765540c34056c..a150cca71547e778da885ebed98ea279c1f3160d 100644
--- a/RTCP/Storage/src/OutputThread.cc
+++ b/RTCP/Storage/src/OutputThread.cc
@@ -95,7 +95,7 @@ static void recursiveMakeDir(const string &dirname, const string &logPrefix)
}
-OutputThread::OutputThread(const Parset &parset, OutputType outputType, unsigned streamNr, Queue<SmartPtr<StreamableData> > &freeQueue, Queue<SmartPtr<StreamableData> > &receiveQueue, const std::string &logPrefix, bool isBigEndian)
+OutputThread::OutputThread(const Parset &parset, OutputType outputType, unsigned streamNr, Queue<SmartPtr<StreamableData> > &freeQueue, Queue<SmartPtr<StreamableData> > &receiveQueue, const std::string &logPrefix, bool isBigEndian, const std::string &targetDirectory)
:
itsParset(parset),
itsOutputType(outputType),
@@ -103,6 +103,7 @@ OutputThread::OutputThread(const Parset &parset, OutputType outputType, unsigned
itsIsBigEndian(isBigEndian),
itsLogPrefix(logPrefix + "[OutputThread] "),
itsCheckFakeData(parset.checkFakeInputData()),
+ itsTargetDirectory(targetDirectory),
itsFreeQueue(freeQueue),
itsReceiveQueue(receiveQueue),
itsBlocksWritten(0),
@@ -124,7 +125,7 @@ void OutputThread::createMS()
ScopedLock sl(casacoreMutex);
ScopedDelayCancellation dc; // don't cancel casacore calls
- std::string directoryName = itsParset.getDirectoryName(itsOutputType, itsStreamNr);
+ std::string directoryName = itsTargetDirectory == "" ? itsParset.getDirectoryName(itsOutputType, itsStreamNr) : itsTargetDirectory;
std::string fileName = itsParset.getFileName(itsOutputType, itsStreamNr);
std::string path = directoryName + "/" + fileName;
diff --git a/RTCP/Storage/src/createHeaders.cc b/RTCP/Storage/src/createHeaders.cc
new file mode 100644
index 0000000000000000000000000000000000000000..adb1547b12c417b39a804f4f2f2456ee3a9b563c
--- /dev/null
+++ b/RTCP/Storage/src/createHeaders.cc
@@ -0,0 +1,85 @@
+//# createHeaders.cc: Generates all .h5/.MS files given a (OLAP) parset
+//#
+//# Copyright (C) 2002-2004
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# $Id: createHeaders.cc 21558 2012-07-12 09:35:39Z mol $
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+
+#include <Common/LofarLogger.h>
+#include <Common/CasaLogSink.h>
+#include <Common/Exceptions.h>
+#include <Interface/Exceptions.h>
+#include <Interface/Parset.h>
+#include <Storage/Package__Version.h>
+#include <Storage/OutputThread.h>
+
+#include <string>
+
+#include <boost/format.hpp>
+
+using namespace LOFAR;
+using namespace LOFAR::RTCP;
+using namespace std;
+
+// Use a terminate handler that can produce a backtrace.
+Exception::TerminateHandler t(Exception::terminate);
+
+int main(int argc, char *argv[])
+{
+ bool isBigEndian = true;
+
+ if (argc < 2 || argc > 3) {
+ cout << str(boost::format("usage: %s parset [is_bigendian]") % argv[0]) << endl;
+ cout << endl;
+ cout << "parset: the filename of the parset to convert (parset must have been produced by RTCP/Run/src/LOFAR/Parset.py, aka an 'OLAP parset')." << endl;
+ cout << "is_bigendian: 1 if data is written big endian (f.e. data comes from the BlueGene/P), 0 otherwise. Default: " << (int)isBigEndian << endl;
+ return 1;
+ }
+
+#if defined HAVE_LOG4CPLUS
+ INIT_LOGGER(string(getenv("LOFARROOT") ? : ".") + "/etc/createHeaders.log_prop");
+#elif defined HAVE_LOG4CXX
+ #error LOG4CXX support is broken (nonsensical?) -- please fix this code if you want to use it
+ Context::initialize();
+ setLevel("Global",8);
+#else
+ INIT_LOGGER_WITH_SYSINFO("createHeaders");
+#endif
+
+ CasaLogSink::attach();
+
+ try {
+ Parset parset(argv[1]);
+ if (argc > 2) isBigEndian = boost::lexical_cast<bool>(argv[2]);
+
+ for (OutputType outputType = FIRST_OUTPUT_TYPE; outputType < LAST_OUTPUT_TYPE; outputType ++) {
+ const unsigned nrStreams = parset.nrStreams(outputType);
+
+ for (unsigned streamNr = 0; streamNr < nrStreams; streamNr ++) {
+ const string logPrefix = str(boost::format("[obs %u type %u stream %3u] ") % parset.observationID() % outputType % streamNr);
+
+ try {
+ // a dummy queue
+ Queue<SmartPtr<StreamableData> > queue;
+
+ OutputThread ot(parset, outputType, streamNr, queue, queue, logPrefix, isBigEndian, ".");
+ ot.createMS();
+ } catch (Exception &ex) {
+ LOG_WARN_STR(logPrefix << "Could not create header: " << ex);
+ } catch (exception &ex) {
+ LOG_WARN_STR(logPrefix << "Could not create header: " << ex.what());
+ }
+ }
+ }
+ } catch (Exception &ex) {
+ LOG_FATAL_STR("[obs unknown] Caught Exception: " << ex);
+ return 1;
+ }
+
+ LOG_INFO_STR("[obs unknown] Program end");
+ return 0;
+}