diff --git a/Docker/lofar-outputproc/Dockerfile.tmpl b/Docker/lofar-outputproc/Dockerfile.tmpl index 6c28e01e454a16b23de0768ae8967b71685c1619..ee7b9dc1c4a06d4a157b5e920d4c03d7a6653407 100644 --- a/Docker/lofar-outputproc/Dockerfile.tmpl +++ b/Docker/lofar-outputproc/Dockerfile.tmpl @@ -3,6 +3,46 @@ # FROM lofar-base:${LOFAR_TAG} +# +# ******************* +# Blitz +# ******************* +# + +#RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + +RUN apt-get update && apt-get install -y git python && \ + mkdir /src && cd /src && git clone --branch=1.0.1 https://github.com/blitzpp/blitz.git && \ + cd /src/blitz && ./configure && make install && \ + rm -rf /src/blitz && \ + apt-get purge git python + +# +# ******************* +# DAL +# ******************* +# + +ENV DAL_VERSION=v3.3.1 + +# Run-time dependencies +RUN aptitude install -y libhdf5-${LIBHDF5_VERSION} python3 && \ + aptitude clean && \ + aptitude autoclean + +RUN export BUILD_PACKAGES="git cmake g++ swig3.0 python3-setuptools python3-dev libhdf5-dev" && \ + aptitude install -y ${BUILD_PACKAGES} && \ + mkdir -p ${INSTALLDIR}/DAL/build && \ + git clone --branch ${DAL_VERSION//latest/master} https://github.com/nextgen-astrodata/DAL.git ${INSTALLDIR}/DAL/DAL.src && \ + cd ${INSTALLDIR}/DAL/build && \ + cmake -DPYTHON_INCLUDE_DIR=/usr/include/python${PYTHON_VERSION} -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython${PYTHON_VERSION}m.so -DBUILD_TESTING=OFF -DCMAKE_CXX_FLAGS="${CXX_FLAGS} -fpermissive" -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/DAL ${INSTALLDIR}/DAL/DAL.src && \ + make -j ${J} && \ + make install && \ + bash -c "rm -rf ${INSTALLDIR}/DAL/{DAL.src,build}" && \ + aptitude purge -y ${BUILD_PACKAGES} && \ + aptitude clean && \ + aptitude autoclean + # # ******************* # LOFAR @@ -10,7 +50,7 @@ FROM lofar-base:${LOFAR_TAG} # # Run-time dependencies -RUN apt-get update && apt-get install -y binutils liblog4cplus-1.1-9 libxml2 libboost-thread${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0 libboost-date-time${BOOST_VERSION}.0 libpng12-0 libsigc++-2.0-dev libxml++2.6-2v5 libboost-regex${BOOST_VERSION}.0 +RUN apt-get update && apt-get install -y binutils liblog4cplus-1.1-9 libxml2 libboost-thread${BOOST_VERSION}.1 libboost-filesystem${BOOST_VERSION}.1 libboost-date-time${BOOST_VERSION}.1 libpng16-16 libsigc++-2.0-dev libxml++2.6-2v5 libboost-regex${BOOST_VERSION}.1 # Tell image build information ENV LOFAR_BRANCH=${LOFAR_BRANCH} \ @@ -18,7 +58,7 @@ ENV LOFAR_BRANCH=${LOFAR_BRANCH} \ LOFAR_BUILDVARIANT=gnucxx11_2018_optarch # Install -RUN apt-get update && apt-get install -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libboost-regex${BOOST_VERSION} binutils-dev libopenblas-dev libcfitsio3-dev wcslib-dev && \ +RUN apt-get update && apt-get install -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libboost-dev boost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libboost-regex${BOOST_VERSION}-dev binutils-dev libopenblas-dev libcfitsio-dev wcslib-dev && \ mkdir -p ${INSTALLDIR}/lofar/build/${LOFAR_BUILDVARIANT} && \ cd ${INSTALLDIR}/lofar && \ svn --non-interactive -q co -r ${LOFAR_REVISION} -N ${LOFAR_BRANCH_URL} src; \ @@ -33,6 +73,6 @@ RUN apt-get update && apt-get install -y subversion cmake g++ gfortran bison fle bash -c "rm -rf ${INSTALLDIR}/lofar/{build,src}" && \ setcap cap_sys_nice,cap_sys_admin=ep ${INSTALLDIR}/lofar/bin/outputProc && \ setcap cap_sys_nice,cap_sys_admin=ep ${INSTALLDIR}/lofar/bin/TBB_Writer && \ - apt-get purge -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev binutils-dev libcfitsio3-dev wcslib-dev libopenblas-dev && \ + apt-get purge -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev binutils-dev libcfitsio3-dev wcslib-dev libopenblas-dev && \ apt-get autoremove -y --purge diff --git a/LCS/MessageBus/CMakeLists.txt b/LCS/MessageBus/CMakeLists.txt index 41113fb9e24557eb4f044f12f7be3e99f53068a2..ab4763a103c483e07ea9ac494962501061ae1874 100644 --- a/LCS/MessageBus/CMakeLists.txt +++ b/LCS/MessageBus/CMakeLists.txt @@ -1,6 +1,11 @@ # $Id$ -lofar_package(MessageBus 1.0 DEPENDS Common pyparameterset) +# Python can be explicitly disabled, if only the C++ libraries are needed +if(USE_PYTHON) + lofar_package(MessageBus 1.0 DEPENDS Common pyparameterset) +else(USE_PYTHON) + lofar_package(MessageBus 1.0 DEPENDS Common) +endif(USE_PYTHON) include(LofarFindPackage) lofar_find_package(QPID) diff --git a/RTCP/Cobalt/CoInterface/src/Parset.cc b/RTCP/Cobalt/CoInterface/src/Parset.cc index 7b437d15f0acb978910ed4dc50550e4faddd9fed..6473cded1f5a09b560116131f00512718748bc19 100644 --- a/RTCP/Cobalt/CoInterface/src/Parset.cc +++ b/RTCP/Cobalt/CoInterface/src/Parset.cc @@ -345,6 +345,8 @@ namespace LOFAR settings.corrections.bandPass = getBool("Cobalt.correctBandPass", true); settings.corrections.clock = getBool("Cobalt.correctClocks", true); settings.corrections.dedisperse = getBool("Cobalt.BeamFormer.coherentDedisperseChannels", true); + + settings.writeToDisk = getBool("Cobalt.writeToDisk", true); settings.delayCompensation.enabled = getBool("Cobalt.delayCompensation", true); settings.delayCompensation.referencePhaseCenter = getDoubleVector("Observation.referencePhaseCenter", vector<double>(3,0), true); diff --git a/RTCP/Cobalt/CoInterface/src/Parset.h b/RTCP/Cobalt/CoInterface/src/Parset.h index e203bb7f4a4a01074caaa9b33100731e3a00b669..507b9c51d5bbcb9a49eb0bd881651ea388c3de15 100644 --- a/RTCP/Cobalt/CoInterface/src/Parset.h +++ b/RTCP/Cobalt/CoInterface/src/Parset.h @@ -175,6 +175,9 @@ namespace LOFAR // key: Observation.bandFilter std::string bandFilter; + // If false, do NOT write output to disk + bool writeToDisk; + struct AntennaFieldName { std::string station; std::string antennaField; diff --git a/RTCP/Cobalt/CoInterface/src/send_state.cc b/RTCP/Cobalt/CoInterface/src/send_state.cc index ebf6e77f34cf8489db7280eee1ffec443b0afefd..2f352e9bd6b568aa4bdf8854d7c652e2cc1712fc 100644 --- a/RTCP/Cobalt/CoInterface/src/send_state.cc +++ b/RTCP/Cobalt/CoInterface/src/send_state.cc @@ -24,6 +24,7 @@ #include <CoInterface/Parset.h> #include <MessageBus/MessageBus.h> #include <MessageBus/ToBus.h> +#include <MessageBus/Util.h> #include <MessageBus/Protocols/TaskFeedbackState.h> #include <boost/format.hpp> @@ -80,12 +81,11 @@ int main(int argc, char **argv) // Parse parset RAW to prevent exceptions caused by broken parsets. // We just need the sasID and momID. - ParameterSet parset; - parset.adoptFile(argv[optind]); + ParameterSet parset(argv[optind]); int success = atoi(argv[optind+1]); // send status feedback - ToBus bus("lofar.task.feedback.state"); + ToBus bus("mac.task.feedback.state", broker_state()); Protocols::TaskFeedbackState msg( "Cobalt/GPUProc/send_state", diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc index 78147b1e23e04d47311b790fde12c4533c7e3549..e2c0fa35b764715f142dcf3541544f0ad8672892 100644 --- a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc +++ b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc @@ -101,7 +101,7 @@ namespace LOFAR { */ // Each element represents 1 block of buffer. - for (size_t i = 0; i < 5; ++i) + for (size_t i = 0; i < 10; ++i) metaDataPool.free.append(new MPIData<SampleT>(startTime, ps.settings.subbands.size(), nrSamples), false); /* @@ -226,8 +226,8 @@ namespace LOFAR { const unsigned board = ps.settings.antennaFields[stationIdx].rspBoardMap[sb]; const unsigned slot = ps.settings.antennaFields[stationIdx].rspSlotMap[sb]; - ASSERT(board < nrBoards); - ASSERT(slot < mode.nrBeamletsPerBoard()); + ASSERTSTR(board < nrBoards, logPrefix << "RSP Board Map refers to board " << board << " but we have only " << nrBoards << " inputs" ); + ASSERTSTR(slot < mode.nrBeamletsPerBoard(), logPrefix << "RSP Slot Map refers to slot " << slot << " but each board only has " << mode.nrBeamletsPerBoard() ); // The specified (board,slot) is stored at position i ASSERTSTR(result[board][slot] == -1, "Station " << stationID.name() << ": board " << board << " slot " << slot << " is used multiple times!"); @@ -743,7 +743,7 @@ namespace LOFAR { Queue< SmartPtr< MPIData<SampleT> > > mpiQueue(str(format( "sendInputToPipeline::mpiQueue [station %s]") % stationID.name())); - MPISender sender(logPrefix, stationIdx, subbandDistribution); + MPISender sender(logPrefix, stationIdx, subbandDistribution, ps.settings.blockDuration()); /* * Stream the data. diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc index c493e80173ebaf17f13275cbf62b3b505215d289..e055c88443dab1ab2c14a4c7e807730904e211af 100644 --- a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc +++ b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc @@ -46,6 +46,7 @@ #include <Common/LofarLogger.h> #include <Common/Timer.h> #include <CoInterface/OMPThread.h> +#include <CoInterface/BudgetTimer.h> #include <InputProc/SampleType.h> using namespace LOFAR; @@ -197,14 +198,15 @@ namespace LOFAR { template struct MPIData< SampleType<i4complex> >; MPISender::MPISender( const std::string &logPrefix, size_t stationIdx, - const SubbandDistribution &subbandDistribution ) + const SubbandDistribution &subbandDistribution, float blockDuration ) : logPrefix(logPrefix), stationIdx(stationIdx), subbandDistribution(subbandDistribution), targetRanks(keys(subbandDistribution)), subbandOffsets(targetRanks.size(), 0), - nrSubbands(values(subbandDistribution).size()) + nrSubbands(values(subbandDistribution).size()), + blockDuration(blockDuration) { // Determine the offset of the set of subbands for each rank within // the members in MPIData<SampleT>. @@ -251,6 +253,10 @@ namespace LOFAR { SmartPtr< MPIData<SampleT> > mpiData; NSTimer mpiSendTimer(str(format("%s MPI send data") % logPrefix), true, true); + BudgetTimer copyTimer( + str(format("%s MPI send data") % logPrefix), + blockDuration, + true, true); size_t nrProcessedSamples = 0; size_t nrFlaggedSamples = 0; diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h index 36156bbe57c9a66fd2a318d1842d362f2feae819..413f91ad6d8e114ab4a0cbfa6883428e8cddb67c 100644 --- a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h +++ b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h @@ -129,7 +129,8 @@ namespace LOFAR { class MPISender { public: MPISender( const std::string &logPrefix, size_t stationIdx, - const SubbandDistribution &subbandDistribution ); + const SubbandDistribution &subbandDistribution, + float blockDuration ); /* * Sends blocks over MPI until a NULL is read from the inputQueue. @@ -153,6 +154,7 @@ namespace LOFAR { const std::vector<int> targetRanks; std::vector<size_t> subbandOffsets; const size_t nrSubbands; + const float blockDuration; /* diff --git a/RTCP/Cobalt/GPUProc/src/gpu_load.cc b/RTCP/Cobalt/GPUProc/src/gpu_load.cc index 10f7b5a9046ed6322a71ab70754a5b190a83bc9e..f3df161f40c566d5412a59b33e2be1d85fde78a2 100644 --- a/RTCP/Cobalt/GPUProc/src/gpu_load.cc +++ b/RTCP/Cobalt/GPUProc/src/gpu_load.cc @@ -60,7 +60,6 @@ int main(int argc, char **argv) { const size_t nrStations = ps.settings.antennaFields.size(); const size_t nrPolarisations = ps.settings.nrPolarisations; const size_t maxNrTABsPerSAP = ps.settings.beamFormer.maxNrCoherentTABsPerSAP(); - const size_t nrSamplesPerChannel = ps.settings.correlator.nrSamplesPerIntegration(); const size_t nrSamplesPerSubband = ps.settings.blockSize; const size_t nrBitsPerSample = ps.settings.nrBitsPerSample; const size_t nrBytesPerComplexSample = ps.nrBytesPerComplexSample(); @@ -91,7 +90,6 @@ int main(int argc, char **argv) { "\n nrStations = " << nrStations << "\n nrPolarisations = " << nrPolarisations << "\n maxNrTABsPerSAP = " << maxNrTABsPerSAP << - "\n nrSamplesPerChannel = " << nrSamplesPerChannel << "\n nrSamplesPerSubband = " << nrSamplesPerSubband << "\n nrBitsPerSample = " << nrBitsPerSample << "\n ----------------------------" << diff --git a/RTCP/Cobalt/GPUProc/src/rtcp.cc b/RTCP/Cobalt/GPUProc/src/rtcp.cc index ea73929a3b2cd6fc3b79a2e004bdbe13bfaf5be8..507f134ad52302c42bb2abf48e33df9bf6cfff0c 100644 --- a/RTCP/Cobalt/GPUProc/src/rtcp.cc +++ b/RTCP/Cobalt/GPUProc/src/rtcp.cc @@ -61,6 +61,7 @@ #include <CoInterface/Stream.h> #include <CoInterface/SelfDestructTimer.h> #include <InputProc/SampleType.h> +#include <InputProc/Delays/Delays.h> #include <InputProc/WallClockTime.h> #include <InputProc/Buffer/StationID.h> @@ -209,7 +210,7 @@ int main(int argc, char **argv) // // Threads get priority 5 by default, and thus cannot interrupt each other. // The vital threads (that deal with station input and polling MPI to make it progress) will get priority 10. - Thread::ScopedPriority sp(ps.settings.realTime ? SCHED_FIFO : SCHED_OTHER, 5); + //Thread::ScopedPriority sp(ps.settings.realTime ? SCHED_FIFO : SCHED_OTHER, 5); LOG_INFO("----- Determining NUMA bindings"); @@ -418,6 +419,13 @@ int main(int argc, char **argv) LOG_WARN_STR("Cannot lock all memory: " << ex.what()); } + // IERS table information + struct Delays::IERS_tablestats stats = Delays::get_IERS_tablestats(); + + LOG_INFO_STR("Using IERS table " << stats.realpath + << ", last entry is " << TimeDouble::toString(stats.last_entry_timestamp, false) + << ", table written on " << TimeDouble::toString(stats.last_fs_modification, false)); + /* * RUN stage */ diff --git a/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh b/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh index 221e0acf262c543e701fed321dc7d3f55a1f9899..77650bebd398e29984b82380441feb7a81dbb2d9 100755 --- a/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh +++ b/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh @@ -41,6 +41,10 @@ ln -sfT ~lofarsys/lofar-userdata/var var # Sym link installed etc/parset-additions.d/override to common location. ln -sfT ~lofarsys/lofar-userdata/parset-overrides etc/parset-additions.d/override +# Sym link installed var/ to NFS location. +ln -sfT /opt/shared/lofar-userdata nfs + + # Set capabilities so our soft real-time programs can elevate prios. # # cap_sys_nice: allow real-time priority for threads diff --git a/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh b/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh index 8745d4189b7e32bb7354e8f55e916098265015c4..a6a3e26e69895c67e6388583de732bb01ddc6373 100755 --- a/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh +++ b/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh @@ -183,7 +183,7 @@ rm -f $FEEDBACK_FILE if [ "$AUGMENT_PARSET" -eq "1" ] then - AUGMENTED_PARSET=$LOFARROOT/var/run/rtcp-$OBSID.parset + AUGMENTED_PARSET=$LOFARROOT/nfs/parset/rtcp-$OBSID.parset # Add static keys # Ignore sneaky .cobalt/ parset overrides in production (lofarsys). @@ -444,25 +444,6 @@ fi echo "[cobalt] Hosts = $HOSTS" -# Copy parset to all hosts -cksumline=`md5sum $PARSET` -for h in `echo $HOSTS | tr ',' ' '` -do - # Ignore empty hostnames - [ -z "$h" ] && continue; - - # Ignore hostnames that point to us - [ "$h" == "localhost" ] && continue; - [ "$h" == "`hostname`" ] && continue; - - # Ignore hosts that already have the same parset (for example, through NFS). - timeout $KILLOPT 5s ssh -o StrictHostKeyChecking=no -qn $h "[ -f $PARSET ] && echo \"$cksumline\" | md5sum -c --status" && continue - - # Copy parset to remote node - echo "Copying parset to $h:$PARSET" - timeout $KILLOPT 30s scp -o StrictHostKeyChecking=no -Bq $PARSET $h:$PARSET || error "[parset] Could not scp parset to $h" -done - # ************************************ # Start rtcp # *********************************** diff --git a/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh b/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh index 482f53a41e5eaa451346f778f21498804f7c036f..ed5dfb79fe1c59fdcdc59f8c16b777b674982958 100755 --- a/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh +++ b/RTCP/Cobalt/GPUProc/src/scripts/stopBGL.sh @@ -36,7 +36,7 @@ function writecommand { # Send the stop command for a graceful shutdown COMMAND_HOST=`getCobaltHosts -C $PARSET` - timeout 60s ssh $COMMAND_HOST "echo $CMD > $PIPE" + timeout 60s ssh $COMMAND_HOST "timeout 60s echo $CMD > $PIPE" } ( diff --git a/RTCP/Cobalt/InputProc/src/Delays/Delays.cc b/RTCP/Cobalt/InputProc/src/Delays/Delays.cc index e2b600d35ab32f51ce623241f43b96dcfcad44db..41bac56e8c06e6b57c869a30f4baebef169cf8bb 100644 --- a/RTCP/Cobalt/InputProc/src/Delays/Delays.cc +++ b/RTCP/Cobalt/InputProc/src/Delays/Delays.cc @@ -27,11 +27,24 @@ #include <Common/Thread/Mutex.h> #include <Common/Thread/Cancellation.h> #include <CoInterface/Exceptions.h> +#include <CoInterface/TimeFuncs.h> #ifdef HAVE_CASACORE #include <casacore/measures/Measures/MEpoch.h> #include <casacore/measures/Measures/MCDirection.h> #include <casacore/casa/Exceptions/Error.h> +#include <casacore/measures/Measures/MeasIERS.h> +#include <casacore/tables/Tables/Table.h> +#include <casacore/tables/Tables/TableRecord.h> +#include <casacore/tables/Tables/ScalarColumn.h> +#include <casacore/casa/Containers/RecordField.h> +#include <casacore/casa/OS/Time.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <cstdlib> +#include <limits.h> #endif @@ -39,7 +52,6 @@ namespace LOFAR { namespace Cobalt { - //##---------------- Public methods ----------------##// Delays::Delays(const Parset &parset, size_t stationIdx, const TimeStamp &from, size_t increment) @@ -91,9 +103,58 @@ namespace LOFAR return MVEpoch(day + 40587., frac); } + struct Delays::IERS_tablestats Delays::get_IERS_tablestats() + { + struct IERS_tablestats result; + + Table table; + TableRecord kws; + ROTableRow row; + RORecordFieldPtr<Double> rfp[1]; + String vs; + Double dt; + Int N = 0; + String rfn[1]; + + // locate TAI_UTC table + if (!MeasIERS::getTable(table, kws, row, rfp, vs, dt, N, rfn, "IERSeop97", "measures.ierseop97.directory", "")) + THROW(Exception, "Cannot open IERSeop97 table"); + + result.path = table.tableName(); + + // resolve any symlinks, which typically includes f.e. "current" + char resolved_path[PATH_MAX]; + if (!realpath(table.tableName().c_str(), resolved_path)) + THROW_SYSCALL("realpath()"); + + result.realpath = resolved_path; + + // obtain table last modification on file system + struct stat filestats; + + string tablef0 = string(resolved_path) + "/table.f0"; + + if (stat(tablef0.c_str(), &filestats) != 0) + THROW_SYSCALL("stat(): Cannot access IERSeop97 table"); + + result.last_fs_modification = filestats.st_mtime; + + // read which timestamps the table contains + casacore::Vector<Double> mjds; + ScalarColumn<Double>(table, "MJD").getColumn(mjds); + casacore::Double last_mjd = mjds[mjds.size()-1]; + + Time last_time(last_mjd + 2400000); + + // MJD -> seconds since 1970 + result.last_entry_timestamp = (last_mjd - 40587) * 86400; + + return result; + } bool Delays::test() { + // Test whether delays can be calculated try { ScopedLock lock(casacoreMutex); ScopedDelayCancellation dc; @@ -122,6 +183,8 @@ namespace LOFAR return false; } + // Print information about age of TAI_UTC table + return true; } @@ -234,6 +297,19 @@ namespace LOFAR void Delays::init() { } + struct Delays::IERS_tablestats Delays::get_IERS_tablestats() { + struct IERS_tablestats result; + result.path = "/dev/null"; + result.realpath = "/dev/null"; + result.last_fs_modification = 0; + result.last_entry_timestamp = 0; + return result; + } + + std::time_t Delays::TAI_UTC_tableage() { + return 0; + } + void Delays::calcDelays( const TimeStamp ×tamp, AllDelays &result ) { (void)timestamp; diff --git a/RTCP/Cobalt/InputProc/src/Delays/Delays.h b/RTCP/Cobalt/InputProc/src/Delays/Delays.h index f81abb6241f45a6ee2428a755606b38ba3395b0b..cd85ef8b4e962b5e580b7f20b36796ac887f6a4e 100644 --- a/RTCP/Cobalt/InputProc/src/Delays/Delays.h +++ b/RTCP/Cobalt/InputProc/src/Delays/Delays.h @@ -45,6 +45,8 @@ #include <casacore/casa/Quanta/MVDirection.h> #include <casacore/casa/Quanta/MVPosition.h> #include <casacore/casa/Quanta/MVEpoch.h> + +#include <ctime> #endif namespace LOFAR @@ -149,6 +151,15 @@ namespace LOFAR */ void generateMetaData( const AllDelays &delaysAtBegin, const AllDelays &delaysAfterEnd, const std::vector<size_t> &subbands, std::vector<SubbandMetaData> &metaDatas, std::vector<ssize_t> &read_offsets ); + struct IERS_tablestats { + std::string path; // path to the IERS table, as provided by casacore + std::string realpath; // path to the IERS table, with all symlinks resolved + time_t last_fs_modification; // last modification to the IERS table, according to the file system (f.e. when table was written) + time_t last_entry_timestamp; // timestamp of last entry in the table + }; + + static struct IERS_tablestats get_IERS_tablestats(); + private: const Parset &parset; const size_t stationIdx; diff --git a/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc b/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc index 143732f59a2d5bf55bc3acc70e2eeec4e28a17b2..64206f7b0b11743da55b630b2054bc3a8d3d025b 100644 --- a/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc +++ b/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc @@ -465,7 +465,7 @@ namespace LOFAR { // the lock and just wait with a timeout if (!requests.empty()) { struct timespec deadline = TimeSpec::now(); - TimeSpec::inc(deadline, 0.0005); + TimeSpec::inc(deadline, 0.00025); newRequest.wait(mutex, deadline); } diff --git a/RTCP/Cobalt/InputProc/test/tDelays.cc b/RTCP/Cobalt/InputProc/test/tDelays.cc index 5bb2cca2454b5da5fdecd80c81d78d13ac6037d3..e1414cb96353cb9d7aa5af9338bb6476be3aeb5c 100644 --- a/RTCP/Cobalt/InputProc/test/tDelays.cc +++ b/RTCP/Cobalt/InputProc/test/tDelays.cc @@ -26,6 +26,7 @@ #include <Common/LofarLogger.h> #include <Stream/FixedBufferStream.h> +#include <CoInterface/TimeFuncs.h> #include <InputProc/Delays/Delays.h> @@ -131,6 +132,13 @@ int main() { INIT_LOGGER( "tDelays" ); + // IERS table information + struct Delays::IERS_tablestats stats = Delays::get_IERS_tablestats(); + + LOG_INFO_STR("Using IERS table " << stats.realpath + << ", last entry is " << TimeDouble::toString(stats.last_entry_timestamp, false) + << ", table written on " << TimeDouble::toString(stats.last_fs_modification, false)); + return UnitTest::RunAllTests() > 0; } diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc index 6bf21ba685d6aa00dc89ed3d6395045429b19c5e..73b72f7d8d8cb13e4cd69be54c542eaba055bf66 100644 --- a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc +++ b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc @@ -190,7 +190,7 @@ namespace LOFAR // BF_File specific root group parameters file.createOfflineOnline().value = itsParset.settings.realTime ? "Online" : "Offline"; file.BFFormat().value = "TAB"; - file.BFVersion().value = str(format("Cobalt/OutputProc %s r%s using DAL %s and HDF5 %s") % OutputProcVersion::getVersion() % OutputProcVersion::getRevision() % dal::version().to_string() % dal::version_hdf5().to_string()); + file.BFVersion().value = str(format("Cobalt2/OutputProc %s r%s using DAL %s and HDF5 %s") % OutputProcVersion::getVersion() % OutputProcVersion::getRevision() % dal::version().to_string() % dal::version_hdf5().to_string()); file.totalIntegrationTime().value = itsNrExpectedBlocks * itsParset.settings.blockDuration(); file.totalIntegrationTimeUnit().value = "s"; diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc b/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc index f1043edc4e5546cf536b0bc38a25173f42130c06..0295fe9a45cfdaada431ad54fda64bb0b80decf0 100644 --- a/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc +++ b/RTCP/Cobalt/OutputProc/src/MSWriterNull.cc @@ -51,13 +51,7 @@ namespace LOFAR void MSWriterNull::write(StreamableData *data) { - // We do not know why the creation of the proper writer failed. - // Assume nothing and only report that we did not write anything - itsConfiguration.replace("percentageWritten", str(format("%u") % 0)); - itsConfiguration.replace("size", str(format("%u") % getDataSize())); - itsConfiguration.replace("duration", - str(format("%f") % ((data->sequenceNumber() + 1) * - itsParset.settings.correlator.integrationTime()))); + (void)data; } } // namespace Cobalt diff --git a/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc b/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc index 3bdf9534242767d126ffdd1c6eb008ab8c85359d..8b226b2e9c348ef3b61d9db157751fd49fbb81f6 100644 --- a/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc +++ b/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc @@ -572,7 +572,7 @@ namespace LOFAR time.put (rownr, Time().modifiedJulianDay() * 24. * 3600.); obsId.put (rownr, 0); message.put (rownr, "parameters"); - application.put (rownr, "OLAP"); + application.put (rownr, "COBALT2"); priority.put (rownr, "NORMAL"); origin.put (rownr, Version::getInfo<OutputProcVersion>("OutputProc", "full")); parms.put (rownr, appvec); @@ -586,7 +586,7 @@ namespace LOFAR // Fill the columns msproc.addRow(); msprocCol.type().put (0, "CORRELATOR"); - msprocCol.subType().put (0, "LOFAR-COBALT"); + msprocCol.subType().put (0, "LOFAR-COBALT2"); msprocCol.typeId().put (0, -1); msprocCol.modeId().put (0, -1); msprocCol.flagRow().put (0, False); diff --git a/RTCP/Cobalt/OutputProc/src/OutputThread.cc b/RTCP/Cobalt/OutputProc/src/OutputThread.cc index 3098403475ba2981be4855e436ecd677bad7d8a3..6f9438c3763dda8e5857a75473cee871d4d301f2 100644 --- a/RTCP/Cobalt/OutputProc/src/OutputThread.cc +++ b/RTCP/Cobalt/OutputProc/src/OutputThread.cc @@ -133,7 +133,8 @@ namespace LOFAR try { BudgetTimer::StartStop ss(writeTimer); - itsWriter->write(data); + if (itsParset.settings.writeToDisk) + itsWriter->write(data); } catch (SystemCallException &ex) { LOG_WARN_STR(itsLogPrefix << "OutputThread caught non-fatal exception: " << ex.what()); itsBlocksDroppedByMe++; diff --git a/SubSystems/Online_Cobalt/test/testFuncs.sh.in b/SubSystems/Online_Cobalt/test/testFuncs.sh.in index 0a8bf55b4a3f749e67eb3b4cb07fd22822cf1a9a..5c5f07b39250c9562d29bfea36e61dc734219d69 100755 --- a/SubSystems/Online_Cobalt/test/testFuncs.sh.in +++ b/SubSystems/Online_Cobalt/test/testFuncs.sh.in @@ -17,6 +17,7 @@ error() # Create runtime output directories if not exists. # Not done at build, because it is a post-install setting. Different in production. mkdir -p "$LOFARROOT/var/log" "$LOFARROOT/var/run" || error "Failed to create runtime output directories" +mkdir -p "$LOFARROOT/nfs/parset/" || error "Failed to create shared directories" mkdir -p "$LOFARROOT/etc/parset-additions.d/override" || error "Failed to create runtime input directories" # Set all locales to "C" to avoid problems with, e.g., perl.