From 1c36abe9e8e12a69a93b2fe9f98678f762ec9be4 Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Mon, 3 Aug 2015 10:07:57 +0000
Subject: [PATCH] Task #8092: Merge from release branch 2.11 --- merge -r
31863:HEAD --accept=working ^/branches/LOFAR-Release-2_11
---
.gitattributes | 7 +
CEP/MS/src/CMakeLists.txt | 1 -
CEP/Pipeline/recipes/sip/tasks.cfg.in | 6 +-
CMake/LofarPackageList.cmake | 3 +-
CMake/variants/variants.kis001 | 4 +-
CMake/variants/variants.lhn001 | 18 +-
CMake/variants/variants.locus102 | 13 +-
LTA/LTAIngest/CMakeLists.txt | 28 ++-
LTA/LTAIngest/ingest_config.py.in | 4 +-
LTA/LTAIngest/ingest_config_test.py.in | 4 +-
.../ingest_install_logrotate_crontab.sh.in | 8 +-
LTA/LTAIngest/ingestpipeline.py | 76 +++++-
LTA/LTAIngest/job_group.py.in | 9 +-
LTA/LTAIngest/job_parser.py | 15 ++
LTA/LTAIngest/ltaingest_build.sh | 41 ++++
LTA/LTAIngest/master.py | 36 +--
LTA/LTAIngest/mom_http.py | 3 +-
LTA/LTAIngest/slave.py | 16 +-
LTA/LTAIngest/startup.csh | 23 +-
LTA/LTAIngest/startup_slave.csh | 22 +-
LTA/LTAIngest/startup_target.csh | 21 +-
LTA/LTAIngest/startup_target_test.csh | 21 +-
LTA/LTAIngest/startup_test.csh | 24 +-
LTA/LTAIngest/test/CMakeLists.txt | 1 +
LTA/LTAIngest/test/test_ingest_sip.py | 93 +++++++
LTA/LTAIngest/test/test_ingest_sip.run | 3 +
LTA/LTAIngest/test/test_ingest_sip.sh | 3 +
.../AntennaFields/PL610-AntennaField.conf | 227 ++++++++++++++++++
.../AntennaFields/PL611-AntennaField.conf | 227 ++++++++++++++++++
.../AntennaFields/PL612-AntennaField.conf | 227 ++++++++++++++++++
.../data/StaticMetaData/RSPImage.conf | 3 +
.../GPUProc/src/scripts/runObservation.sh | 20 +-
SAS/Scheduler/src/Controller.cpp | 4 -
SAS/Scheduler/src/taskstorage.cpp | 48 ++++
SubSystems/CMakeLists.txt | 1 +
SubSystems/SAS_Tools/CMakeLists.txt | 4 +
36 files changed, 1163 insertions(+), 101 deletions(-)
create mode 100755 LTA/LTAIngest/ltaingest_build.sh
create mode 100755 LTA/LTAIngest/test/test_ingest_sip.py
create mode 100755 LTA/LTAIngest/test/test_ingest_sip.run
create mode 100755 LTA/LTAIngest/test/test_ingest_sip.sh
create mode 100644 MAC/Deployment/data/StaticMetaData/AntennaFields/PL610-AntennaField.conf
create mode 100644 MAC/Deployment/data/StaticMetaData/AntennaFields/PL611-AntennaField.conf
create mode 100644 MAC/Deployment/data/StaticMetaData/AntennaFields/PL612-AntennaField.conf
create mode 100644 SubSystems/SAS_Tools/CMakeLists.txt
diff --git a/.gitattributes b/.gitattributes
index ac5bc35aa75..cf6ccb14b38 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2788,6 +2788,7 @@ LTA/LTAIngest/doc/LTA-SIP.xsd -text
LTA/LTAIngest/example.job -text
LTA/LTAIngest/fpconst-0.7.0/PKG-INFO -text
LTA/LTAIngest/fpconst-0.7.0/fpconst-0.7.0/PKG-INFO -text
+LTA/LTAIngest/ltaingest_build.sh eol=lf
LTA/LTAIngest/md5adler/a32 -text
LTA/LTAIngest/md5adler/foo -text
LTA/LTAIngest/md5adler/md5a32 -text
@@ -2813,6 +2814,8 @@ LTA/LTAIngest/ssh_check.sh eol=lf
LTA/LTAIngest/test.xmlrpc -text
LTA/LTAIngest/test/test_ingest_logging.run eol=lf
LTA/LTAIngest/test/test_ingest_logging.sh eol=lf
+LTA/LTAIngest/test/test_ingest_sip.run eol=lf
+LTA/LTAIngest/test/test_ingest_sip.sh eol=lf
LTA/LTAIngest/user_ingest_example -text
MAC/APL/APLCommon/include/APL/APLCommon/AntennaField.h -text
MAC/APL/APLCommon/src/AntennaField.cc -text
@@ -3440,6 +3443,9 @@ MAC/Deployment/data/StaticMetaData/AntennaFields/CS501-AntennaField.conf -text
MAC/Deployment/data/StaticMetaData/AntennaFields/DE604-AntennaField.conf -text
MAC/Deployment/data/StaticMetaData/AntennaFields/DE609-AntennaField.conf -text
MAC/Deployment/data/StaticMetaData/AntennaFields/FR606-AntennaField.conf -text
+MAC/Deployment/data/StaticMetaData/AntennaFields/PL610-AntennaField.conf -text
+MAC/Deployment/data/StaticMetaData/AntennaFields/PL611-AntennaField.conf -text
+MAC/Deployment/data/StaticMetaData/AntennaFields/PL612-AntennaField.conf -text
MAC/Deployment/data/StaticMetaData/AntennaFields/RS106-AntennaField.conf -text
MAC/Deployment/data/StaticMetaData/AntennaFields/RS205-AntennaField.conf -text
MAC/Deployment/data/StaticMetaData/AntennaFields/RS208-AntennaField.conf -text
@@ -4920,6 +4926,7 @@ SubSystems/Online_Cobalt/validation/system/users/localhome.test eol=lf
SubSystems/Online_Cobalt/validation/system/users/login-shell.test -text
SubSystems/Online_Cobalt/validation/system/users/sudo.test eol=lf
SubSystems/Online_Cobalt/validation/validate eol=lf
+SubSystems/SAS_Tools/CMakeLists.txt -text
/jenkins_make -text
/lofar_config.h.cmake -text
support/tools/BaseSimGUI/images/4.gif -text svneol=unset#unset
diff --git a/CEP/MS/src/CMakeLists.txt b/CEP/MS/src/CMakeLists.txt
index 445add81d02..b3cfa472d4d 100644
--- a/CEP/MS/src/CMakeLists.txt
+++ b/CEP/MS/src/CMakeLists.txt
@@ -16,7 +16,6 @@ set(ms_PROGRAMS
combinevds
makems
msplay
- msselect
msoverview
)
diff --git a/CEP/Pipeline/recipes/sip/tasks.cfg.in b/CEP/Pipeline/recipes/sip/tasks.cfg.in
index 1282aa18d15..c1545c94024 100644
--- a/CEP/Pipeline/recipes/sip/tasks.cfg.in
+++ b/CEP/Pipeline/recipes/sip/tasks.cfg.in
@@ -60,7 +60,7 @@ recipe = imager_prepare
ndppp_exec = %(lofarroot)s/bin/NDPPP
asciistat_executable = %(lofarroot)s/bin/asciistats.py
statplot_executable = %(lofarroot)s/bin/statsplot.py
-msselect_executable = %(lofarroot)s/bin/msselect
+msselect_executable = %(casaroot)s/bin/msselect
rficonsole_executable = %(lofarroot)s/bin/rficonsole
[long_baseline]
@@ -68,7 +68,7 @@ recipe = long_baseline
ndppp_exec = %(lofarroot)s/bin/NDPPP
asciistat_executable = %(lofarroot)s/bin/asciistats.py
statplot_executable = %(lofarroot)s/bin/statsplot.py
-msselect_executable = %(lofarroot)s/bin/msselect
+msselect_executable = %(casaroot)s/bin/msselect
rficonsole_executable = %(lofarroot)s/bin/rficonsole
nproc = 1
@@ -116,4 +116,4 @@ bbs_executable = %(lofarroot)s/bin/bbs-reducer
[selfcal_finalize]
recipe = selfcal_finalize
fillrootimagegroup_exec = %(lofarroot)s/bin/fillRootImageGroup
-msselect_executable = %(lofarroot)s/bin/msselect
+msselect_executable = %(casaroot)s/bin/msselect
diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index 9d5f1630d0a..169e826ad19 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -1,7 +1,7 @@
# - Create for each LOFAR package a variable containing the absolute path to
# its source directory.
#
-# Generated by gen_LofarPackageList_cmake.sh at Thu Jun 18 07:25:53 UTC 2015
+# Generated by gen_LofarPackageList_cmake.sh at Tue Jun 23 21:08:57 UTC 2015
#
# ---- DO NOT EDIT ----
#
@@ -146,6 +146,7 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(ST_MAC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/ST_MAC)
set(Offline_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/Offline)
set(SAS_OTDB_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/SAS_OTDB)
+ set(SAS_Tools_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/SAS_Tools)
set(PVSS_DB_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/PVSS_DB)
set(LAPS_CEP_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/LAPS_CEP)
endif(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
diff --git a/CMake/variants/variants.kis001 b/CMake/variants/variants.kis001
index cb4cb0cca67..b12c1988086 100644
--- a/CMake/variants/variants.kis001
+++ b/CMake/variants/variants.kis001
@@ -5,8 +5,8 @@ set(ENV{JAVA_HOME} /usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0)
#set(PVSS_ROOT_DIR /opt/pvss/pvss2_v3.7)
set(PVSS_ROOT_DIR /opt/WinCC_OA/3.10)
set(LOG4CPLUS_ROOT_DIR "/usr/local/log4cplus")
-#set(CASACORE_ROOT_DIR "/opt/casacore")
-#set(BOOST_ROOT_DIR "/opt/boost")
+set(CASACORE_ROOT_DIR "/opt/casacore")
+set(BOOST_ROOT_DIR "/opt/boost")
set(QPID_ROOT_DIR /opt/qpid)
set(CTEST_CUSTOM_WARNING_EXCEPTION
diff --git a/CMake/variants/variants.lhn001 b/CMake/variants/variants.lhn001
index 98e03981464..845364d5517 100644
--- a/CMake/variants/variants.lhn001
+++ b/CMake/variants/variants.lhn001
@@ -1,14 +1,14 @@
-set(CASACORE_ROOT_DIR /opt/cep/casacore)
-set(CASAREST_ROOT_DIR /opt/cep/casarest)
-set(DAL_ROOT_DIR /opt/cep/dal/current)
+option(BUILD_SHARED_LIBS "Build shared libraries" ON)
-set(PYRAP_ROOT_DIR /opt/cep/pyrap)
+set(CASACORE_ROOT_DIR /opt/cep/casacore/builds/casacore-2.0.1/build/gnu_opt)
+set(CASAREST_ROOT_DIR /opt/cep/casarest/builds/casarest-svn8765)
+set(PYRAP_ROOT_DIR /opt/cep/pyrap/builds/python-casacore-2.0.0)
+set(WCSLIB_ROOT_DIR /opt/cep/wcslib/builds/wcslib-4.23)
+set(DAL_ROOT_DIR /opt/cep/dal/current)
set(QPID_ROOT_DIR /opt/qpid)
-set(WCSLIB_ROOT_DIR /opt/cep/wcslib)
-
-set(LOG4CXX_ROOT_DIR /opt/cep/LofIm/external/log4cxx)
+set(LOG4CXX_ROOT_DIR /opt/cep/lofar/external/log4cxx)
set(LOG4CPLUS_ROOT_DIR /opt/cep/lofar/external/log4cplus)
#set(ENV{JAVA_HOME} /usr/lib/jvm/java-1.5.0-sun)
@@ -17,7 +17,3 @@ set(CTEST_CUSTOM_WARNING_EXCEPTION
"/log4cxx/helpers/objectptr.h:[0-9]+: warning: base class"
"/log4cxx/helpers/objectptr.h:[0-9]+: warning: dereferencing type-punned pointer"
)
-
-# Drop the now default C++ debug compile flag -D_GLIBCXX_DEBUG, because Python
-# (or Boost.Python) barfs on it (glibc detected: python free(): invalid pointer)
-set(GNU_CXX_FLAGS_DEBUG "-g")
diff --git a/CMake/variants/variants.locus102 b/CMake/variants/variants.locus102
index 0e24f3eca50..845364d5517 100644
--- a/CMake/variants/variants.locus102
+++ b/CMake/variants/variants.locus102
@@ -1,15 +1,14 @@
option(BUILD_SHARED_LIBS "Build shared libraries" ON)
-set(CASACORE_ROOT_DIR /opt/cep/casacore)
-set(CASAREST_ROOT_DIR /opt/cep/casarest)
+set(CASACORE_ROOT_DIR /opt/cep/casacore/builds/casacore-2.0.1/build/gnu_opt)
+set(CASAREST_ROOT_DIR /opt/cep/casarest/builds/casarest-svn8765)
+set(PYRAP_ROOT_DIR /opt/cep/pyrap/builds/python-casacore-2.0.0)
+set(WCSLIB_ROOT_DIR /opt/cep/wcslib/builds/wcslib-4.23)
+
set(DAL_ROOT_DIR /opt/cep/dal/current)
set(QPID_ROOT_DIR /opt/qpid)
-set(PYRAP_ROOT_DIR /opt/cep/pyrap)
-
-set(WCSLIB_ROOT_DIR /opt/cep/wcslib)
-
-set(LOG4CXX_ROOT_DIR /opt/cep/LofIm/external/log4cxx)
+set(LOG4CXX_ROOT_DIR /opt/cep/lofar/external/log4cxx)
set(LOG4CPLUS_ROOT_DIR /opt/cep/lofar/external/log4cplus)
#set(ENV{JAVA_HOME} /usr/lib/jvm/java-1.5.0-sun)
diff --git a/LTA/LTAIngest/CMakeLists.txt b/LTA/LTAIngest/CMakeLists.txt
index 3838ae96940..70767005e9b 100644
--- a/LTA/LTAIngest/CMakeLists.txt
+++ b/LTA/LTAIngest/CMakeLists.txt
@@ -7,13 +7,21 @@ set(LTAINGEST_BUILD_TARGET "test" CACHE STRING "Specify for which specific desti
message(STATUS "Using LTAINGEST_BUILD_TARGET \"${LTAINGEST_BUILD_TARGET}\".")
+set(MOM3SCHEMA "lofar_mom3")
+set(EXPORTSCHEMA "lofar_export")
+
#switch with config variables for the various build targets.
if(LTAINGEST_BUILD_TARGET STREQUAL "lexar")
set(LTAINGEST_LOG_ROOT_DIR "/log/ingest/")
+ set(LTAINGEST_DATA_DIR "/data/ingest/")
elseif(LTAINGEST_BUILD_TARGET STREQUAL "lotar")
set(LTAINGEST_LOG_ROOT_DIR "/home/lofarlocal/log/ingest/")
+ set(LTAINGEST_DATA_DIR "/home/lofarlocal/.ingest/")
elseif(LTAINGEST_BUILD_TARGET STREQUAL "test")
set(LTAINGEST_LOG_ROOT_DIR "/tmp/ingest/")
+ set(LTAINGEST_DATA_DIR "/tmp/ingest/")
+ set(MOM3SCHEMA "lofartest_sas099_mom3_two")
+ set(EXPORTSCHEMA "lofartest_sas099_export")
else()
message(SEND_ERROR "Unknown LTAINGEST_BUILD_TARGET given: '${LTAINGEST_BUILD_TARGET}'.")
endif()
@@ -252,22 +260,32 @@ python_install(
do_ltacp.py
simple_server.py
__init__.py
+ doc/LTA-SIP.xsd #not a py file, but needed by ingest in python ingest lib dir
DESTINATION LTAIngest)
+#deploy startup scripts in same python ingest lib dir
+#TODO: should be moved to LOFARROOT/bin. Currently probably not possible yet due to unknown dependencies.
+set(STARTUP_SCRIPTS
+ startup.csh
+ startup_slave.csh
+ startup_target.csh
+ startup_target_test.csh
+ startup_test.csh)
+install(PROGRAMS ${STARTUP_SCRIPTS} DESTINATION ${PYTHON_INSTALL_DIR}/LTAIngest)
+
#determine LTAINGEST_LOG_ROOT_DIR for various build hosts (used in ingest_config.py.in)
if(NOT DEFINED LTAINGEST_LOG_ROOT_DIR)
set(LTAINGEST_LOG_ROOT_DIR "/tmp/ingest/")
endif()
-#get momreadonly account settings from environment
-set(MOMREADONLY_USER $ENV{MOMREADONLY_USER})
-set(MOMREADONLY_PASSWORD $ENV{MOMREADONLY_PASSWORD})
-
#settings dir for ingest
-set(LTAINGEST_CFG_DIR "${CMAKE_INSTALL_PREFIX}/.ltaingest")
+set(LTAINGEST_CFG_DIR "${CMAKE_INSTALL_PREFIX}/etc/ltaingest")
#set various user/passwords from environment
#mail schaap@astron.nl for a shell script which sets up your build environment with these secret variables
+if ("$ENV{MOMREADONLY_USER}" STREQUAL "")
+ message(WARNING "MOMREADONLY_USER not found in environment. Did you source your build environment file?")
+endif()
set(MOMREADONLY_USER $ENV{MOMREADONLY_USER})
set(MOMREADONLY_PASSWORD $ENV{MOMREADONLY_PASSWORD})
set(EXPORTCLIENT_USERNAME $ENV{EXPORTCLIENT_USERNAME})
diff --git a/LTA/LTAIngest/ingest_config.py.in b/LTA/LTAIngest/ingest_config.py.in
index 2a7ef8936ee..548b03f938d 100644
--- a/LTA/LTAIngest/ingest_config.py.in
+++ b/LTA/LTAIngest/ingest_config.py.in
@@ -153,8 +153,8 @@ if master:
raise
# Used by Master to send error and (re) start messages
-mailCommand = '-c renting@astron.nl,holties@astron.nl,wjvriend@astro.rug.nl,observer@astron.nl sciencesupport@astron.nl'
+mailCommand = '-c schaap@astron.nl,holties@astron.nl,wjvriend@astro.rug.nl,observer@astron.nl sciencesupport@astron.nl'
# Used by Slave to send SendStatus warnings
-mailSlCommand = '-c holties@astron.nl,wjvriend@astro.rug.nl renting@astron.nl'
+mailSlCommand = '-c holties@astron.nl,wjvriend@astro.rug.nl schaap@astron.nl'
logger.info("Configuration complete")
diff --git a/LTA/LTAIngest/ingest_config_test.py.in b/LTA/LTAIngest/ingest_config_test.py.in
index 2726cd6e2f0..6f8cdfea50c 100644
--- a/LTA/LTAIngest/ingest_config_test.py.in
+++ b/LTA/LTAIngest/ingest_config_test.py.in
@@ -145,8 +145,8 @@ if master:
raise
# Used by Master to send error and (re) start messages
-mailCommand = ' renting@astron.nl'
+mailCommand = ' schaap@astron.nl'
# Used by Slave to send SendStatus warnings
-mailSlCommand = ' renting@astron.nl'
+mailSlCommand = ' schaap@astron.nl'
logger.info("Configuration complete")
diff --git a/LTA/LTAIngest/ingest_install_logrotate_crontab.sh.in b/LTA/LTAIngest/ingest_install_logrotate_crontab.sh.in
index 91e7b1c23d0..8834f094c88 100644
--- a/LTA/LTAIngest/ingest_install_logrotate_crontab.sh.in
+++ b/LTA/LTAIngest/ingest_install_logrotate_crontab.sh.in
@@ -2,12 +2,16 @@
#directory for various ltaingest config and state files
LTAINGEST_CFG_DIR=@LTAINGEST_CFG_DIR@
+LTAINGEST_DATA_DIR=@LTAINGEST_DATA_DIR@
+
+mkdir -p $LTAINGEST_CFG_DIR
+mkdir -p $LTAINGEST_DATA_DIR
#logrotate command to be executed for ingest
-command="logrotate $LTAINGEST_CFG_DIR/ingest_logrotate.conf -s $LTAINGEST_CFG_DIR/ingest_logrotate.state >> /dev/null 2>&1"
+command="/usr/sbin/logrotate $LTAINGEST_CFG_DIR/ingest_logrotate.conf -s $LTAINGEST_DATA_DIR/ingest_logrotate.state 2>&1 /dev/null"
#ingest logrotate line for crontab
-job="@daily $command"
+job="0 0 * * * $command"
#check if job is not installed in crontab yet, if so install it.
cat <(fgrep -i -v "$command" <(crontab -l)) <(echo "$job") | crontab -
diff --git a/LTA/LTAIngest/ingestpipeline.py b/LTA/LTAIngest/ingestpipeline.py
index 298fe115f3f..7fb095c3e3f 100755
--- a/LTA/LTAIngest/ingestpipeline.py
+++ b/LTA/LTAIngest/ingestpipeline.py
@@ -5,6 +5,18 @@ from lxml import etree
from cStringIO import StringIO
from job_group import corr_type, bf_type, img_type, unspec_type, pulp_type
+def humanreadablesize(num, suffix='B'):
+ """ converts the given size (number) to a human readable string in powers of 1024
+ """
+ try:
+ for unit in ['','K','M','G','T','P','E','Z']:
+ if abs(num) < 1024.0:
+ return "%3.1f%s%s" % (num, unit, suffix)
+ num /= 1024.0
+ return "%.1f%s%s" % (num, 'Y', suffix)
+ except TypeError:
+ return str(num)
+
IngestStarted = 10
## 20 not used
IngestSIPComplete = 30
@@ -184,7 +196,8 @@ class IngestPipeline():
start = time.time()
p = subprocess.Popen(cmd, stdin=open('/dev/null'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logs = p.communicate()
- self.logger.debug("File transfer for %s took %ds" % (self.JobId, time.time() - start))
+ elapsed = time.time() - start
+ self.logger.debug("File transfer for %s took %d sec" % (self.JobId, elapsed))
## time.sleep(10)
## logs = ("hoeba","bla")
log = logs[0].split('\n')
@@ -197,6 +210,12 @@ class IngestPipeline():
self.logger.error("Parsing ltacp result failed for %s" % self.JobId)
raise Exception('File transfer failed of %s' % self.JobId)
else:
+ try:
+ if int(self.FileSize) > 0:
+ avgSpeed = float(self.FileSize) / elapsed
+ self.logger.debug("File transfer for %s took %d sec with an average speed of %s for %s including ltacp overhead" % (self.JobId, elapsed, humanreadablesize(avgSpeed, 'Bps'), humanreadablesize(float(self.FileSize), 'B')))
+ except Exception:
+ pass
self.CheckChecksums()
else: # need to communicate that LTA transaction is to be rolled back but ingest not to be set to "hold"
#os.system('echo "Dataproduct for %s not found on %s.\nConsidering dataproduct to be non existent"|mailx -s "Warning: Dataproduct not found on CEP host" ' % (self.JobId, self.HostLocation) + self.mailCommand)
@@ -207,7 +226,7 @@ class IngestPipeline():
def CheckChecksums(self):
if self.MD5Checksum and self.Adler32Checksum and self.FileSize:
try:
- self.logger.debug('Valid checksums found for %s with filesize %s' % (self.JobId, self.FileSize))
+ self.logger.debug('Valid checksums found for %s with filesize %sB (%s)' % (self.JobId, self.FileSize, humanreadablesize(float(self.FileSize), 'B')))
except:
self.logger.debug('Valid checksums found for %s' % (self.JobId))
else:
@@ -277,17 +296,55 @@ class IngestPipeline():
self.logger.error('CheckSIP failed: ' + str(e))
return False
+ def CheckSIPContent(self):
+ try:
+ start = time.time()
+ sip = StringIO(self.SIP)
+ tree = etree.parse(sip)
+ root = tree.getroot()
+ dataProducts = root.xpath('dataProduct')
+ if len(dataProducts) != 1:
+ self.logger.error("CheckSIPContent for %s could not find single dataProduct in SIP" % (self.JobId))
+ return False
+ dataProductIdentifierIDs = dataProducts[0].xpath('dataProductIdentifier/identifier')
+ if len(dataProductIdentifierIDs) != 1:
+ self.logger.error("CheckSIPContent for %s could not find single dataProductIdentifier/identifier in SIP dataProduct" % (self.JobId))
+ return False
+ if dataProductIdentifierIDs[0].text != str(self.MomId):
+ self.logger.error("CheckSIPContent for %s dataProductIdentifier/identifier %s does not match expected %s" % (self.JobId, dataProductIdentifierIDs[0].text, self.MomId))
+ return False
+ dataProductIdentifierNames = dataProducts[0].xpath('dataProductIdentifier/name')
+ if len(dataProductIdentifierNames) != 1:
+ self.logger.error("CheckSIPContent for %s could not find single dataProductIdentifier/name in SIP dataProduct" % (self.JobId))
+ return False
+ if not dataProductIdentifierNames[0].text in self.FileName:
+ self.logger.error("CheckSIPContent for %s dataProductIdentifier/name %s does not match expected %s" % (self.JobId, dataProductIdentifierNames[0].text, self.FileName))
+ return False
+ storageTickets = dataProducts[0].xpath('storageTicket')
+ if len(storageTickets) != 1:
+ self.logger.error("CheckSIPContent for %s could not find single storageTickets in SIP dataProduct" % (self.JobId))
+ return False
+ if storageTickets[0].text != str(self.ticket):
+ self.logger.error("CheckSIPContent for %s storageTicket %s does not match expected %s" % (self.JobId, storageTickets[0].text, self.ticket))
+ return False
+
+ return True
+ except Exception as e:
+ self.logger.error('CheckSIPContent failed: ' + str(e))
+ return False
+
def GetSIP(self):
if self.Type == "MoM":
try:
start = time.time()
+ self.logger.debug("GetSIP for %s with mom2DPId %s - StorageTicket %s - FileName %s - Uri %s" % (self.JobId, self.MomId, self.ticket, self.FileName, self.PrimaryUri))
sip = self.momClient.getSIP(self.MomId, self.ticket, self.FileName, self.PrimaryUri, self.FileSize, self.MD5Checksum, self.Adler32Checksum)
self.SIP = sip.replace('<stationType>Europe</stationType>','<stationType>International</stationType>')
self.logger.debug("GetSIP for %s took %ds" % (self.JobId, time.time() - start))
except:
self.logger.exception('Getting SIP from MoM failed')
raise
- self.logger.debug('SIP received for %s from MoM with size %d: %s' % (self.JobId, len(self.SIP), self.SIP[0:400]))
+ self.logger.debug('SIP received for %s from MoM with size %d (%s): %s' % (self.JobId, len(self.SIP), humanreadablesize(len(self.SIP)), self.SIP[0:1024]))
else:
self.SIP = unspecifiedSIP.makeSIP(self.Project, self.ObsId, self.MomId, self.ticket, self.FileName, self.FileSize, self.MD5Checksum, self.Adler32Checksum, self.Type)
self.FileType = unspec_type
@@ -301,6 +358,8 @@ class IngestPipeline():
raise
self.logger.debug('Unspecified SIP created for %s: %s' % (self.JobId, self.SIP[0:400]))
###raise Exception('Got a malformed SIP from MoM: %s' % self.SIP[0:50])
+ if not self.CheckSIPContent():
+ raise PipelineError('Got a SIP with wrong contents from MoM for %s : %s' % (self.JobId, self.SIP), func.__name__)
def SendSIP(self):
try:
@@ -372,9 +431,16 @@ class IngestPipeline():
self.RetryRun(self.GetSIP, self.momRetry, 'Get SIP from MoM')
self.RetryRun(self.SendSIP, self.ltaRetry, 'Sending SIP')
self.RetryRun(self.SendStatus, self.ltaRetry, 'Setting LTA status', IngestSuccessful)
- self.logger.debug("Ingest Pipeline finished for %s in %d" % (self.JobId, time.time() - start))
+ elapsed = time.time() - start
+ try:
+ if int(self.FileSize) > 0:
+ avgSpeed = float(self.FileSize) / elapsed
+ self.logger.debug("Ingest Pipeline finished for %s in %d sec with average speed of %s for %s including all overhead" % (self.JobId, elapsed, humanreadablesize(avgSpeed, 'Bps'), humanreadablesize(float(self.FileSize), 'B')))
+ except Exception:
+ self.logger.debug("Ingest Pipeline finished for %s in %d sec" % (self.JobId, elapsed))
+
except PipelineError as pe:
- self.logger.debug('Encountered PipelineError for %s' % (self.JobId))
+ self.logger.debug('Encountered PipelineError for %s : %s' % (self.JobId, str(pe)))
## roll back transfer if necessary
if self.PrimaryUri or self.tempPrimary:
if not (pe.type == PipelineNoSourceError):
diff --git a/LTA/LTAIngest/job_group.py.in b/LTA/LTAIngest/job_group.py.in
index f2c19afe174..c74fb343a49 100755
--- a/LTA/LTAIngest/job_group.py.in
+++ b/LTA/LTAIngest/job_group.py.in
@@ -131,7 +131,7 @@ class job_group():
def check_finished(self):
total = len(self.jobs)
finished = (self.scheduled['Total'] == 0) and (self.active['Total'] == 0) and (self.inactive['Total'] == 0) and (self.retry['Total'] == 0)
- self.logger.debug('job_group %i progress: #done=%i #failed=%i #scheduled=%i #active=%i #inactive=%i #retry=%i' % (self.Id, self.done['Total'], self.done['Total'], self.failed['Total'], self.scheduled['Total'], self.active['Total'], self.inactive['Total'], self.retry['Total']))
+ self.logger.debug('job_group %i progress: #done=%i #failed=%i #scheduled=%i #active=%i #inactive=%i #retry=%i' % (self.Id, self.done['Total'], self.failed['Total'], self.scheduled['Total'], self.active['Total'], self.inactive['Total'], self.retry['Total']))
if finished and (total == self.failed['Total'] + self.done['Total']): ## sanity check, somewhat redundant
self.logger.info('job_group %i is finished, total of %i files' % (self.Id, total))
return True
@@ -149,11 +149,8 @@ class job_group():
'project': 'unknown', 'ltalocation': "unknown", 'eid': "unknown"}
if self.Type == 'MoM':
try:
- ## should be read from config file
- #m = "lofartest_sas099_mom3_two"
- m = "lofar_mom3"
- #e = "lofartest_sas099_export"
- e = "lofar_export"
+ m = "@MOM3SCHEMA@"
+ e = "@EXPORTSCHEMA@"
db = MySQLdb.connect(host="mysql1.control.lofar", user="@MOMREADONLY_USER@", passwd="@MOMREADONLY_PASSWORD@", db=m)
c = db.cursor()
c.execute("SELECT a.id,toexportdate as started,a.status_date as last_update,a.exportername,b.name as state,d.name,e.name as projectname,g.name as location,a.data_location,d.mom2id FROM %s.exports AS a, %s.status AS b, %s.mom_references AS c, %s.mom2object AS d, %s.mom2object AS e, %s.resource AS f, %s.resourcetype AS g WHERE status_id=b.id AND a.mom_reference_id = c.id AND c.mom_id=d.mom2id AND d.ownerprojectid = e.id AND e.id=f.projectid AND g.type = \"LTA_STORAGE\" AND f.resourcetypeid = g.id and a.id = %i;" % (e,e,e,m,m,m,m,self.Id))
diff --git a/LTA/LTAIngest/job_parser.py b/LTA/LTAIngest/job_parser.py
index 016f278aace..4f1bc880ea9 100755
--- a/LTA/LTAIngest/job_parser.py
+++ b/LTA/LTAIngest/job_parser.py
@@ -9,6 +9,21 @@ JobScheduled = 1
JobProducing = 2
JobProduced = 3
+def jobState2String(jobstate):
+ if jobstate == JobRetry:
+ return "%d (JobRetry)" % jobstate
+ elif jobstate == JobError:
+ return "%d (JobError)" % jobstate
+ elif jobstate == JobHold:
+ return "%d (JobHold)" % jobstate
+ elif jobstate == JobScheduled:
+ return "%d (JobScheduled)" % jobstate
+ elif jobstate == JobProducing:
+ return "%d (JobProducing)" % jobstate
+ elif jobstate == JobProduced:
+ return "%d (JobProduced)" % jobstate
+ return str(jobstate)
+
## Job should probably be refactored into a class at some point.
## Right now it's just a big dict.
diff --git a/LTA/LTAIngest/ltaingest_build.sh b/LTA/LTAIngest/ltaingest_build.sh
new file mode 100755
index 00000000000..17d60e6d892
--- /dev/null
+++ b/LTA/LTAIngest/ltaingest_build.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# $Id$
+
+if [ "$#" -ne 1 ]; then
+ echo "Usage: ./ltaingest_build.sh <tag>"
+ echo "where tag is a name or version number which is added to the tarballs."
+ echo "This script creates three build flavours (test/lexar/lotar) in source_root_dir/build and builds LTAIngest"
+ echo "Then it performs a local install (in the each specific build dir) and creates a deployable tarball"
+ echo "Final result is three tarballs in source_root_dir/build which can be copied to the ingest servers"
+ exit 1
+fi
+
+#get path of this build script and determine source root from there
+REL_PATH="`dirname \"$0\"`"
+ABS_PATH="`( cd \"$REL_PATH\" && pwd )`"
+SOURCE_ROOT="$ABS_PATH/../.."
+
+echo "Using '$SOURCE_ROOT' as source route"
+
+BUILD_TAG="$1"
+echo "Using Build tag $BUILD_TAG"
+
+TEST_BUILD_DIR=$SOURCE_ROOT/build/test/gnu_debug
+LEXAR_BUILD_DIR=$SOURCE_ROOT/build/lexar/gnu_debug
+LOTAR_BUILD_DIR=$SOURCE_ROOT/build/lotar/gnu_debug
+
+mkdir -p $TEST_BUILD_DIR
+mkdir -p $LEXAR_BUILD_DIR
+mkdir -p $LOTAR_BUILD_DIR
+
+cd $TEST_BUILD_DIR && cmake -DBUILD_PACKAGES=LTAIngest -DLTAINGEST_BUILD_TARGET=test -DCMAKE_INSTALL_PREFIX=/globalhome/ingesttest/ltaingest_$BUILD_TAG $SOURCE_ROOT
+cd $LEXAR_BUILD_DIR && cmake -DBUILD_PACKAGES=LTAIngest -DLTAINGEST_BUILD_TARGET=lexar -DCMAKE_INSTALL_PREFIX=/globalhome/ingest/ltaingest_$BUILD_TAG $SOURCE_ROOT
+cd $LOTAR_BUILD_DIR && cmake -DBUILD_PACKAGES=LTAIngest -DLTAINGEST_BUILD_TARGET=lotar -DCMAKE_INSTALL_PREFIX=/home/lofarlocal/ltaingest_$BUILD_TAG $SOURCE_ROOT
+
+cd $TEST_BUILD_DIR && make && make test && rm -rf ./local_install && make DESTDIR=./local_install install
+cd $LEXAR_BUILD_DIR && make && rm -rf ./local_install && make DESTDIR=./local_install install
+cd $LOTAR_BUILD_DIR && make && rm -rf ./local_install && make DESTDIR=./local_install install
+
+cd $TEST_BUILD_DIR/local_install/globalhome/ingesttest && tar cvzf $SOURCE_ROOT/build/ltaingest_"$BUILD_TAG"_test.tgz ltaingest_$BUILD_TAG
+cd $LEXAR_BUILD_DIR/local_install/globalhome/ingest && tar cvzf $SOURCE_ROOT/build/ltaingest_"$BUILD_TAG"_lexar.tgz ltaingest_$BUILD_TAG
+cd $LOTAR_BUILD_DIR/local_install/home/lofarlocal && tar cvzf $SOURCE_ROOT/build/ltaingest_"$BUILD_TAG"_lotar.tgz ltaingest_$BUILD_TAG
diff --git a/LTA/LTAIngest/master.py b/LTA/LTAIngest/master.py
index a88048e0d4d..8682deea9aa 100755
--- a/LTA/LTAIngest/master.py
+++ b/LTA/LTAIngest/master.py
@@ -5,6 +5,7 @@ from Queue import Empty
from job_group import job_group
import job_parser as parser
from job_parser import JobRetry, JobError, JobHold, JobScheduled, JobProducing, JobProduced
+from job_parser import jobState2String
import os, time, sys, shutil
@@ -90,7 +91,7 @@ class momTalker(Process):
self.logger.warning(message)
self.logger.info(message)
except:
- self.logger.exception('Could not update job %s status to %s.' % (str(job['ExportID']), str(job['Status'])))
+ self.logger.exception('Could not update job %s status to %s.' % (str(job['ExportID']), jobState2String(job['Status'])))
def run(self):
self.logger.info('momTalker started')
@@ -163,7 +164,7 @@ class jobHandler(Process):
self.update_file(job)
if job['Status'] == JobRetry:
self.scheduled.put(job) ## We need to try it again
- self.logger.debug("Job %s no longer active because of state %s" % (job['ExportID'], job['Status']))
+ self.logger.debug("Job %s no longer active because of state %s" % (job['ExportID'], jobState2String(job['Status'])))
def run(self):
## ====== Waiting for slaves ======
@@ -264,19 +265,24 @@ class queueHandler(Process):
def run(self):
while True:
- sleep = True
- try:
- msg = self.update_job_msg.get_nowait()
- sleep = False
- self.update_job(msg[0], msg[1], msg[2], msg[3])
- except Empty: pass
try:
- fileName = self.incomming.get_nowait()
- sleep = False
- self.newJob(fileName)
- except Empty: pass
- if sleep: ## nothing to do, time for a nap.
- time.sleep(10)
+ sleep = True
+ try:
+ msg = self.update_job_msg.get_nowait()
+ sleep = False
+ self.update_job(msg[0], msg[1], msg[2], msg[3])
+ except Empty: pass
+ try:
+ fileName = self.incomming.get_nowait()
+ sleep = False
+ self.newJob(fileName)
+ except Empty: pass
+ if sleep: ## nothing to do, time for a nap.
+ time.sleep(10)
+ except Exception as e:
+ self.logger.error("Exception in queueHandler main loop: %s" % e)
+ time.sleep(250)
+
## Startup ----------------------------------------------------------
@@ -376,7 +382,7 @@ class ltaMaster():
job['errors'].append(result)
self.update_job_msg.put((job, JobProducing, job['Status'], fileType))
self.job_done_msg.put(job)
- self.logger.debug('Slave reported done with %s, status %s' % (job['ExportID'], job['Status']))
+ self.logger.debug('Slave reported done with %s, status %s' % (job['ExportID'], jobState2String(job['Status'])))
def serve(self):
class manager(SyncManager): pass
diff --git a/LTA/LTAIngest/mom_http.py b/LTA/LTAIngest/mom_http.py
index d65730eb0dd..f6f330e61a3 100755
--- a/LTA/LTAIngest/mom_http.py
+++ b/LTA/LTAIngest/mom_http.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
import urllib, urllib2, cookielib, os.path, ClientForm, socket
+from job_parser import jobState2String
class client:
"""This is an HTTP client that knows how to use the Single Sign On of Mom2.
@@ -47,7 +48,7 @@ class client:
response = self.opener.open(request) ## We tell what we want
reply = response.readlines()
if reply == ['ok']:
- result = (0, 'http_login updated ' + str(exportID) + ' to ' + str(status))
+ result = (0, 'http_login updated ' + str(exportID) + ' to ' + jobState2String(int(status)))
else:
result = (1, 'http_login for ' + str(exportID) + ' failed on: ' + str(reply))
except Exception, e:
diff --git a/LTA/LTAIngest/slave.py b/LTA/LTAIngest/slave.py
index f72eaccabaa..91e195f404d 100755
--- a/LTA/LTAIngest/slave.py
+++ b/LTA/LTAIngest/slave.py
@@ -3,6 +3,7 @@ from multiprocessing import Process, Queue, Value
from Queue import Empty as QueueEmpty
from multiprocessing.managers import SyncManager
from job_parser import JobRetry, JobError, JobHold, JobScheduled, JobProducing, JobProduced
+from job_parser import jobState2String
import os, time, sys
from ingestpipeline import IngestPipeline, PipelineError, PipelineJobFailedError
from ingestpipeline import PipelineNoSourceError, PipelineAlreadyInLTAError, PipelineAlreadyInLTAError, PipelineNoProjectInLTAError
@@ -48,7 +49,7 @@ class momTalker(Process):
self.logger.warning(message)
self.logger.info(message)
except:
- self.logger.exception('Could not update job %s status to %s.' % (str(job['ExportID']), str(job['Status'])))
+ self.logger.exception('Could not update job %s status to %s.' % (str(job['ExportID']), jobState2String(job['Status'])))
def run(self):
self.logger.info('momTalker started')
@@ -85,8 +86,14 @@ class executer(Process):
logger.info('Executer initialzed for %s (pid: %i)' % (job['ExportID'], os.getpid()))
def run(self):
+ start = time.time()
+ self.logger.debug("Slave Pipeline executer starting for %s" % (self.job['ExportID']))
self.job['Status'] = JobProducing
- self.talker.put(self.job)
+ if not self.talker.full():
+ self.talker.put(self.job)
+ else:
+ self.logger.debug("MoM queue full, skipping JobProducing status update for %s" % (self.job['ExportID']))
+
pipeline = IngestPipeline(self.logdir, self.job, self.momClient, self.ltaClient, self.host, self.ltacpport, self.mailCommand, self.momRetry, self.ltaRetry, self.srmRetry, self.srmInit)
try:
pipeline.run()
@@ -125,11 +132,14 @@ class executer(Process):
self.job['retry'] += 1
if self.job['retry'] < self.pipelineRetry:
self.job['Status'] = JobRetry
- if (self.job['Status'] == JobProduced) or (self.job['Status'] == JobError):
+ if (self.job['Status'] == JobProduced):
self.talker.put(self.job)
+ elif (self.job['Status'] == JobError):
+ self.logger.warning('Skipping JobError status update to MoM to prevent the hold/running slow flipping bug for %s' % self.job['ExportID'])
self.manager.slave_done(self.job, self.result, pipeline.FileType)
with self.jobs.get_lock():
self.jobs.value -= 1
+ self.logger.debug("Slave Pipeline executer finished for %s in %d sec" % (self.job['ExportID'], time.time() - start))
## ---------------- LTA Slave --------------------------------------------
class ltaSlave():
diff --git a/LTA/LTAIngest/startup.csh b/LTA/LTAIngest/startup.csh
index 461119c4d3b..9f5b8543f85 100755
--- a/LTA/LTAIngest/startup.csh
+++ b/LTA/LTAIngest/startup.csh
@@ -1,17 +1,28 @@
#!/bin/csh -f
#starting ingest pipeline
-#ar: 15 may 2013
-cd /globalhome/ingest/LTAIngest
-setenv PYTHONPATH /globalhome/ingest/LTAIngest
+# $Id$
+
+set LTAINGEST_PATH = /globalhome/ingest/LTAIngest
+echo "LTAINGEST_PATH=$LTAINGEST_PATH"
+cd $LTAINGEST_PATH
+if ( ! ($?PYTHONPATH) ) then
+ setenv PYTHONPATH $LTAINGEST_PATH
+else if ( $LTAINGEST_PATH !~ $PYTHONPATH ) then
+ setenv PYTHONPATH $LTAINGEST_PATH\:$PYTHONPATH
+endif
+
+echo "PYTHONPATH=$PYTHONPATH"
+
if (! `ps uxf | grep -v grep | grep -c master.py` ) then
- nohup master.py ingest_config >& nohup.out &
+ echo "Starting Master"
+ nohup python ./master.py ingest_config >& nohup.out &
else
echo "Master running already"
endif
sleep 1
if (! `ps uxf | grep -v grep | grep -c slave.py` ) then
- nohup slave.py ingest_config >& nohup_slave.out &
+ echo "Starting Slave"
+ nohup python ./slave.py ingest_config >& nohup_slave.out &
else
echo "Slave is running already"
endif
-cd /globalhome/ingest
diff --git a/LTA/LTAIngest/startup_slave.csh b/LTA/LTAIngest/startup_slave.csh
index 01ece143f76..54cbb9d970e 100755
--- a/LTA/LTAIngest/startup_slave.csh
+++ b/LTA/LTAIngest/startup_slave.csh
@@ -1,11 +1,21 @@
#!/bin/csh -f
-#starting ingest pipeline
-#ar: 15 may 2013
-cd /globalhome/ingest/LTAIngest
-setenv PYTHONPATH /globalhome/ingest/LTAIngest
+#starting ingest pipeline, slave only
+# $Id$
+
+set LTAINGEST_PATH = /globalhome/ingest/LTAIngest
+echo "LTAINGEST_PATH=$LTAINGEST_PATH"
+cd $LTAINGEST_PATH
+if ( ! ($?PYTHONPATH) ) then
+ setenv PYTHONPATH $LTAINGEST_PATH
+else if ( $LTAINGEST_PATH !~ $PYTHONPATH ) then
+ setenv PYTHONPATH $LTAINGEST_PATH\:$PYTHONPATH
+endif
+
+echo "PYTHONPATH=$PYTHONPATH"
+
if (! `ps uxf | grep -v grep | grep -c slave.py` ) then
- nohup slave.py ingest_config >& nohup_slave.out &
+ echo "Starting Slave"
+ nohup python ./slave.py ingest_config >& nohup_slave.out &
else
echo "Slave is running already"
endif
-cd /globalhome/ingest
diff --git a/LTA/LTAIngest/startup_target.csh b/LTA/LTAIngest/startup_target.csh
index 3985ee19271..0b4fbcec8af 100755
--- a/LTA/LTAIngest/startup_target.csh
+++ b/LTA/LTAIngest/startup_target.csh
@@ -1,10 +1,21 @@
#!/bin/csh -f
-#starting ingest pipeline
-#ar: 28 august 2014
-cd /home/lofarlocal/LTAIngest
-setenv PYTHONPATH /home/lofarlocal/LTAIngest
+#starting ingest pipeline, slave only on target
+# $Id$
+
+set LTAINGEST_PATH = /home/lofarlocal/LTAIngest
+echo "LTAINGEST_PATH=$LTAINGEST_PATH"
+cd $LTAINGEST_PATH
+if ( ! ($?PYTHONPATH) ) then
+ setenv PYTHONPATH $LTAINGEST_PATH
+else if ( $LTAINGEST_PATH !~ $PYTHONPATH ) then
+ setenv PYTHONPATH $LTAINGEST_PATH\:$PYTHONPATH
+endif
+
+echo "PYTHONPATH=$PYTHONPATH"
+
if (! `ps uxf | grep -v grep | grep -c slave.py` ) then
- nohup ./slave.py ingest_config >& nohup_slave.out &
+ echo "Starting Slave"
+ nohup python ./slave.py ingest_config >& nohup_slave.out &
else
echo "Slave is running already"
endif
diff --git a/LTA/LTAIngest/startup_target_test.csh b/LTA/LTAIngest/startup_target_test.csh
index 665e3412b8b..5634d714745 100755
--- a/LTA/LTAIngest/startup_target_test.csh
+++ b/LTA/LTAIngest/startup_target_test.csh
@@ -1,10 +1,21 @@
#!/bin/csh -f
-#starting ingest pipeline
-#ar: 28 august 2014
-cd /home/lofarlocal/LTAIngest
-setenv PYTHONPATH /home/lofarlocal/LTAIngest
+#starting ingest test pipeline, slave only on target
+# $Id$
+
+set LTAINGEST_PATH = /home/lofarlocal/LTAIngest
+echo "LTAINGEST_PATH=$LTAINGEST_PATH"
+cd $LTAINGEST_PATH
+if ( ! ($?PYTHONPATH) ) then
+ setenv PYTHONPATH $LTAINGEST_PATH
+else if ( $LTAINGEST_PATH !~ $PYTHONPATH ) then
+ setenv PYTHONPATH $LTAINGEST_PATH\:$PYTHONPATH
+endif
+
+echo "PYTHONPATH=$PYTHONPATH"
+
if (! `ps uxf | grep -v grep | grep -c slave.py` ) then
- nohup ./slave.py ingest_config_test >& nohup_slave.out &
+ echo "Starting Slave"
+ nohup python ./slave.py ingest_config_test >& nohup_slave.out &
else
echo "Slave is running already"
endif
diff --git a/LTA/LTAIngest/startup_test.csh b/LTA/LTAIngest/startup_test.csh
index 3102ceb8ed2..1679a86bb84 100755
--- a/LTA/LTAIngest/startup_test.csh
+++ b/LTA/LTAIngest/startup_test.csh
@@ -1,16 +1,28 @@
#!/bin/csh -f
-#starting ingest pipeline
-#ar: 15 may 2013
-cd /globalhome/ingesttest/LTAIngest
-setenv PYTHONPATH /globalhome/ingesttest/LTAIngest
+#starting ingest test pipeline
+# $Id$
+
+set LTAINGEST_PATH = /globalhome/ingesttest/LTAIngest
+echo "LTAINGEST_PATH=$LTAINGEST_PATH"
+cd $LTAINGEST_PATH
+if ( ! ( $?PYTHONPATH ) ) then
+ setenv PYTHONPATH $LTAINGEST_PATH
+else if ( $LTAINGEST_PATH !~ $PYTHONPATH ) then
+ setenv PYTHONPATH $LTAINGEST_PATH\:$PYTHONPATH
+endif
+
+echo "PYTHONPATH=$PYTHONPATH"
+
if (! `ps uxf | grep -v grep | grep -c master.py` ) then
- nohup ./master.py ingest_config_test >& nohup.out &
+ echo "Starting Master"
+ nohup python ./master.py ingest_config_test >& nohup.out &
else
echo "Master running already"
endif
sleep 1
if (! `ps uxf | grep -v grep | grep -c slave.py` ) then
- nohup ./slave.py ingest_config_test >& nohup_slave.out &
+ echo "Starting Slave"
+ nohup python ./slave.py ingest_config_test >& nohup_slave.out &
else
echo "Slave is running already"
endif
diff --git a/LTA/LTAIngest/test/CMakeLists.txt b/LTA/LTAIngest/test/CMakeLists.txt
index 2541d0e5778..04f2987e5f3 100644
--- a/LTA/LTAIngest/test/CMakeLists.txt
+++ b/LTA/LTAIngest/test/CMakeLists.txt
@@ -2,5 +2,6 @@
include(LofarCTest)
lofar_add_test(test_ingest_logging)
+lofar_add_test(test_ingest_sip)
diff --git a/LTA/LTAIngest/test/test_ingest_sip.py b/LTA/LTAIngest/test/test_ingest_sip.py
new file mode 100755
index 00000000000..36b01939465
--- /dev/null
+++ b/LTA/LTAIngest/test/test_ingest_sip.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+import unittest
+import logging
+import LTAIngest.ingestpipeline as ingestpipeline
+from LTAIngest.job_parser import JobScheduled
+from LTAIngest.unspecifiedSIP import makeSIP
+
+class TestIngestSIP(unittest.TestCase):
+ """Tests for getting and processing SIP in Ingest"""
+
+ def testSIPContents(self):
+ """test CheckSIPContent method with valid and invalid SIP"""
+
+ #logging
+ logger = logging.getLogger('Slave')
+ logger.addHandler(logging.StreamHandler())
+ logger.setLevel(logging.DEBUG)
+
+ #create mock job
+ job = {}
+ job['ExportID'] = '0000'
+ job['JobId'] = 'X_0000_0000_1234567_12345678_L123456_SB000_uv.MS'
+ job['Status'] = JobScheduled
+ job['DataProduct'] = 'L123456_SB000_uv.MS'
+ job['FileName'] = job['DataProduct'] + '.tar'
+ job['MomId'] = 12345678
+ job['ObservationId'] = 123456
+ job["Type"] = 'MoM'
+ job['Project'] = 'test'
+ job['Location'] = 'locus099:/data/L123456/L123456_SB000_uv.MS'
+
+ #create mock storage ticket
+ ticket = 'ABCDEF123456789'
+
+ #create mock momClient
+ class MockMoMClient():
+ pass
+ momClient = MockMoMClient()
+
+ #create mock ltaClient
+ class MockLTAClient():
+ pass
+ ltaClient = MockLTAClient()
+
+ #create misc mock args
+ ltacphost = 'mock-ltacphost'
+ ltacpport = -1
+ mailCommand = ''
+ srmInit = ''
+
+ #create pipeline instance with mock info
+ pipeline = ingestpipeline.IngestPipeline('/tmp', job, momClient, ltaClient, ltacphost, ltacpport, mailCommand, 1, 1, 1, srmInit)
+
+ #end of setup
+ #start testing
+
+ #create proper SIP with valid ticket, momid and filename
+ pipeline.SIP = makeSIP(job['Project'], job['ObservationId'], job['MomId'], ticket, job['FileName'], 0, 0, 0, job["Type"])
+ pipeline.ticket = ticket
+
+ #check SIP content
+ self.assertTrue(pipeline.CheckSIPContent(), pipeline.SIP)
+
+ #now invalidate the SIP in various ways
+
+ #with wrong momid
+ pipeline.SIP = makeSIP(job['Project'], job['ObservationId'], 'wrong_MomId', ticket, job['FileName'], 0, 0, 0, job["Type"])
+ self.assertFalse(pipeline.CheckSIPContent(), pipeline.SIP)
+
+ #without momid
+ pipeline.SIP = pipeline.SIP.replace('<identifier>wrong_MomId</identifier>', '')
+ self.assertFalse(pipeline.CheckSIPContent(), pipeline.SIP)
+
+ #with wrong filename
+ pipeline.SIP = makeSIP(job['Project'], job['ObservationId'], job['MomId'], ticket, 'wrong_filename', 0, 0, 0, job["Type"])
+ self.assertFalse(pipeline.CheckSIPContent(), pipeline.SIP)
+
+ #without filename
+ pipeline.SIP = pipeline.SIP.replace('<name>wrong_filename</name>', '')
+ self.assertFalse(pipeline.CheckSIPContent(), pipeline.SIP)
+
+ #with wrong ticket
+ pipeline.SIP = makeSIP(job['Project'], job['ObservationId'], job['MomId'], 'wrong_ticket', job['FileName'], 0, 0, 0, job["Type"])
+ self.assertFalse(pipeline.CheckSIPContent(), pipeline.SIP)
+
+ #without ticket
+ pipeline.SIP = pipeline.SIP.replace('<storageTicket>wrong_ticket</storageTicket>', '')
+ self.assertFalse(pipeline.CheckSIPContent(), pipeline.SIP)
+
+#run tests if main
+if __name__ == '__main__':
+ unittest.main()
diff --git a/LTA/LTAIngest/test/test_ingest_sip.run b/LTA/LTAIngest/test/test_ingest_sip.run
new file mode 100755
index 00000000000..3d37bf71446
--- /dev/null
+++ b/LTA/LTAIngest/test/test_ingest_sip.run
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python test_ingest_sip.py
diff --git a/LTA/LTAIngest/test/test_ingest_sip.sh b/LTA/LTAIngest/test/test_ingest_sip.sh
new file mode 100755
index 00000000000..ed63cb7d6f8
--- /dev/null
+++ b/LTA/LTAIngest/test/test_ingest_sip.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh test_ingest_sip
diff --git a/MAC/Deployment/data/StaticMetaData/AntennaFields/PL610-AntennaField.conf b/MAC/Deployment/data/StaticMetaData/AntennaFields/PL610-AntennaField.conf
new file mode 100644
index 00000000000..49f873d1bf1
--- /dev/null
+++ b/MAC/Deployment/data/StaticMetaData/AntennaFields/PL610-AntennaField.conf
@@ -0,0 +1,227 @@
+#
+# DUMMY AntennaPositions for PL610
+# ITRF2005 target_date = 2015.5
+# Created: 2015-06-23 19:24:00
+#
+
+NORMAL_VECTOR LBA
+3 [ 0.582686 0.102609 0.806194 ]
+
+ROTATION_MATRIX LBA
+3 x 3 [
+ -0.1384370000 -0.8008200000 0.5826860000
+ 0.9900310000 -0.0964870000 0.1026090000
+ -0.0259500000 0.5910820000 0.8061940000
+]
+
+LBA
+3 [ 3727207.315770000 655185.261474000 5117000.905 ]
+96 x 2 x 3 [
+ -2.385000 -13.491000 3.440990 -2.385000 -13.491000 3.440990
+ -7.901000 -8.415000 6.781990 -7.901000 -8.415000 6.781990
+ -5.396000 -14.324000 5.722990 -5.396000 -14.324000 5.722990
+-10.186000 -10.119001 8.649990 -10.186000 -10.119001 8.649990
+ -0.124000 20.778000 -2.555010 -0.124000 20.778000 -2.555010
+ -4.701000 24.130000 0.326990 -4.701000 24.130000 0.326990
+ 5.326000 25.166001 -7.052000 5.326000 25.166001 -7.052000
+ 2.267000 28.650001 -5.285010 2.267000 28.650001 -5.285010
+ -3.652000 -19.315000 5.097990 -3.652000 -19.315000 5.097990
+ 0.017010 -24.962000 3.165000 0.017010 -24.962000 3.165000
+ -1.878990 -28.049000 4.928000 -1.878990 -28.049000 4.928000
+ -5.235990 -27.314000 7.260990 -5.235990 -27.314000 7.260990
+ -0.077000 5.075000 -0.590010 -0.077000 5.075000 -0.590010
+ -3.595000 1.673000 2.385990 -3.595000 1.673000 2.385990
+ -5.802000 3.931000 3.692990 -5.802000 3.931000 3.692990
+ -4.688000 8.401000 2.318990 -4.688000 8.401000 2.318990
+ -8.598000 3.337999 5.788990 -8.598000 3.337999 5.788990
+ -9.953000 -1.251001 7.352990 -9.953000 -1.251001 7.352990
+-14.253000 5.953999 9.543990 -14.253000 5.953999 9.543990
+-16.063000 2.028999 11.351990 -16.063000 2.028999 11.351990
+ 17.152000 -3.302998 -11.976000 17.152000 -3.302998 -11.976000
+ 17.566000 -12.263998 -11.135000 17.566000 -12.263998 -11.135000
+ 24.669000 -4.977997 -17.197000 24.669000 -4.977997 -17.197000
+ 21.351010 -15.981998 -13.397000 21.351010 -15.981998 -13.397000
+ 3.774000 -12.505999 -1.136000 3.774000 -12.505999 -1.136000
+ 2.166000 -15.975999 0.468000 2.166000 -15.975999 0.468000
+ 6.991000 -14.399999 -3.220000 6.991000 -14.399999 -3.220000
+ 9.106010 -17.695999 -4.329000 9.106010 -17.695999 -4.329000
+ 12.973000 11.788002 -10.876000 12.973000 11.788002 -10.876000
+ 14.427000 15.115002 -12.351000 14.427000 15.115002 -12.351000
+ 19.444000 6.923002 -14.935000 19.444000 6.923002 -14.935000
+ 21.584000 12.082002 -17.138000 21.584000 12.082002 -17.138000
+-15.908000 9.413999 10.299990 -15.908000 9.413999 10.299990
+-15.203000 15.815999 8.974990 -15.203000 15.815999 8.974990
+-22.162000 21.404998 13.292990 -22.162000 21.404998 13.292990
+-23.439000 15.267998 14.997990 -23.439000 15.267998 14.997990
+ 3.902000 5.015001 -3.459000 3.902000 5.015001 -3.459000
+ 1.084000 9.810000 -2.032000 1.084000 9.810000 -2.032000
+ 6.434000 7.284001 -5.577000 6.434000 7.284001 -5.577000
+ 5.172000 10.742001 -5.105000 5.172000 10.742001 -5.105000
+-13.778000 -17.568001 12.193990 -13.778000 -17.568001 12.193990
+-17.938000 -8.584001 14.056990 -17.938000 -8.584001 14.056990
+-20.272000 -16.947002 16.808990 -20.272000 -16.947002 16.808990
+-23.586000 -7.012002 17.938990 -23.586000 -7.012002 17.938990
+ 5.987000 -1.138999 -4.182000 5.987000 -1.138999 -4.182000
+ 8.740000 0.756001 -6.413000 8.740000 0.756001 -6.413000
+ 7.442000 -5.689999 -4.655000 7.442000 -5.689999 -4.655000
+ 10.784000 -2.740999 -7.446000 10.784000 -2.740999 -7.446000
+ -4.781000 17.368000 1.244990 -4.781000 17.368000 1.244990
+ 16.096000 3.398002 -12.066000 16.096000 3.398002 -12.066000
+ -7.428000 28.024000 1.801990 -7.428000 28.024000 1.801990
+-12.607000 16.980999 6.950990 -12.607000 16.980999 6.950990
+ 13.586010 -19.721998 -7.309000 13.586010 -19.721998 -7.309000
+ 25.303000 2.547003 -18.612000 25.303000 2.547003 -18.612000
+-14.951000 24.151999 7.731990 -14.951000 24.151999 7.731990
+ -1.321000 8.211000 -0.090010 -1.321000 8.211000 -0.090010
+-11.883000 -24.202001 11.668990 -11.883000 -24.202001 11.668990
+ 13.136000 -12.848998 -7.859000 13.136000 -12.848998 -7.859000
+ 2.007000 -19.867999 1.078000 2.007000 -19.867999 1.078000
+ 17.313000 20.669002 -15.144000 17.313000 20.669002 -15.144000
+ 12.879000 4.744002 -9.912000 12.879000 4.744002 -9.912000
+ 12.378000 -9.072998 -7.792000 12.378000 -9.072998 -7.792000
+-14.753000 -7.090001 11.565990 -14.753000 -7.090001 11.565990
+ -5.135000 -2.138000 3.982990 -5.135000 -2.138000 3.982990
+ -0.165000 15.213000 -1.817010 -0.165000 15.213000 -1.817010
+ -1.108000 -7.354000 1.736990 -1.108000 -7.354000 1.736990
+ 10.425000 18.470001 -9.886000 10.425000 18.470001 -9.886000
+ 8.508000 11.428001 -7.603000 8.508000 11.428001 -7.603000
+ 7.179010 -22.427999 -2.334000 7.179010 -22.427999 -2.334000
+ 5.515010 -24.913999 -0.815000 5.515010 -24.913999 -0.815000
+ 2.140000 -4.021999 -1.035000 2.140000 -4.021999 -1.035000
+-10.858000 -14.764001 9.726990 -10.858000 -14.764001 9.726990
+ 9.646000 4.173001 -7.503000 9.646000 4.173001 -7.503000
+ -9.835000 12.923999 5.462990 -9.835000 12.923999 5.462990
+ 2.586000 15.972001 -3.902000 2.586000 15.972001 -3.902000
+-12.249000 25.292999 5.633990 -12.249000 25.292999 5.633990
+ 2.427000 -9.188999 -0.584000 2.427000 -9.188999 -0.584000
+ 13.400000 -0.374998 -9.637000 13.400000 -0.374998 -9.637000
+ -6.464000 -20.643000 7.298990 -6.464000 -20.643000 7.298990
+-20.268000 -0.557002 14.719990 -20.268000 -0.557002 14.719990
+-13.727000 -1.982001 10.172990 -13.727000 -1.982001 10.172990
+ -7.039000 20.879000 2.429990 -7.039000 20.879000 2.429990
+-11.211000 6.636999 7.257990 -11.211000 6.636999 7.257990
+ -1.969000 -3.827000 1.909990 -1.969000 -3.827000 1.909990
+ 21.307000 -1.589998 -15.198000 21.307000 -1.589998 -15.198000
+ 3.916000 1.518001 -3.024000 3.916000 1.518001 -3.024000
+ 10.477000 28.105001 -11.149000 10.477000 28.105001 -11.149000
+ 7.760000 19.672001 -8.112000 7.760000 19.672001 -8.112000
+ -7.629000 -3.386000 5.944990 -7.629000 -3.386000 5.944990
+-25.785000 3.657998 18.170990 -25.785000 3.657998 18.170990
+ -2.791000 -9.971000 3.286990 -2.791000 -9.971000 3.286990
+ 16.219010 -19.770998 -9.206000 16.219010 -19.770998 -9.206000
+-19.359000 7.202998 13.074990 -19.359000 7.202998 13.074990
+ 4.190000 -6.577999 -2.191000 4.190000 -6.577999 -2.191000
+ 8.585000 -8.974999 -5.063000 8.585000 -8.974999 -5.063000
+ -6.503000 11.141000 3.281990 -6.503000 11.141000 3.281990
+]
+
+NORMAL_VECTOR HBA
+3 [ 0.583126 0.102326 0.805912 ]
+
+ROTATION_MATRIX HBA
+3 x 3 [
+ -0.1382550000 -0.8005310000 0.5831260000
+ 0.9900640000 -0.0964520000 0.1023260000
+ -0.0256710000 0.5914790000 0.8059120000
+]
+
+HBA
+3 [ 3727217.665780000 655109.182475000 5117003.127 ]
+96 x 2 x 3 [
+-22.005000 7.896998 14.918990 -22.005000 7.896998 14.918990
+-19.653000 11.909998 12.707990 -19.653000 11.909998 12.707990
+-17.301000 15.922999 10.496990 -17.301000 15.922999 10.496990
+-14.950000 19.935999 8.286000 -14.950000 19.935999 8.286000
+-12.598000 23.948999 6.075000 -12.598000 23.948999 6.075000
+-20.896000 0.699998 15.030990 -20.896000 0.699998 15.030990
+-18.544000 4.712999 12.818990 -18.544000 4.712999 12.818990
+-16.193000 8.724999 10.609000 -16.193000 8.724999 10.609000
+-13.841000 12.738999 8.397000 -13.841000 12.738999 8.397000
+-11.489000 16.751999 6.186000 -11.489000 16.751999 6.186000
+ -9.137000 20.763999 3.975000 -9.137000 20.763999 3.975000
+ -6.786000 24.777000 1.764000 -6.786000 24.777000 1.764000
+-19.788000 -6.498002 15.142990 -19.788000 -6.498002 15.142990
+-17.436000 -2.484001 12.931000 -17.436000 -2.484001 12.931000
+-15.084000 1.527999 10.720000 -15.084000 1.527999 10.720000
+-12.732000 5.540999 8.509000 -12.732000 5.540999 8.509000
+-10.381000 9.552999 6.298000 -10.381000 9.552999 6.298000
+ -8.029000 13.567000 4.087000 -8.029000 13.567000 4.087000
+ -5.678000 17.580000 1.876000 -5.678000 17.580000 1.876000
+ -3.325000 21.592000 -0.335000 -3.325000 21.592000 -0.335000
+ -0.974000 25.605000 -2.546000 -0.974000 25.605000 -2.546000
+-18.679000 -13.695001 15.254000 -18.679000 -13.695001 15.254000
+-16.327000 -9.682001 13.043000 -16.327000 -9.682001 13.043000
+-13.976000 -5.669001 10.832000 -13.976000 -5.669001 10.832000
+-11.624000 -1.656001 8.621000 -11.624000 -1.656001 8.621000
+ -9.272000 2.355999 6.410000 -9.272000 2.355999 6.410000
+ -6.920000 6.369000 4.199000 -6.920000 6.369000 4.199000
+ -4.569000 10.382000 1.988000 -4.569000 10.382000 1.988000
+ -2.217000 14.395000 -0.224000 -2.217000 14.395000 -0.224000
+ 0.134000 18.408000 -2.434000 0.134000 18.408000 -2.434000
+ 2.487000 22.420001 -4.646000 2.487000 22.420001 -4.646000
+ 4.838000 26.433001 -6.857000 4.838000 26.433001 -6.857000
+-15.219000 -16.880001 13.155000 -15.219000 -16.880001 13.155000
+-12.867000 -12.867001 10.944000 -12.867000 -12.867001 10.944000
+-10.515000 -8.854001 8.733000 -10.515000 -8.854001 8.733000
+ -8.164000 -4.841000 6.522000 -8.164000 -4.841000 6.522000
+ -5.812000 -0.828000 4.310000 -5.812000 -0.828000 4.310000
+ -3.461000 3.184000 2.100000 -3.461000 3.184000 2.100000
+ -1.108000 7.197000 -0.112000 -1.108000 7.197000 -0.112000
+ 1.243000 11.211000 -2.323000 1.243000 11.211000 -2.323000
+ 3.595000 15.223001 -4.534000 3.595000 15.223001 -4.534000
+ 5.946000 19.236001 -6.745000 5.946000 19.236001 -6.745000
+ 8.298000 23.248001 -8.956000 8.298000 23.248001 -8.956000
+-11.758000 -20.064001 11.055000 -11.758000 -20.064001 11.055000
+ -9.407000 -16.051001 8.844000 -9.407000 -16.051001 8.844000
+ -7.055000 -12.039000 6.633000 -7.055000 -12.039000 6.633000
+ -4.703000 -8.025000 4.422000 -4.703000 -8.025000 4.422000
+ -2.351000 -4.013000 2.211000 -2.351000 -4.013000 2.211000
+ 2.351000 4.013001 -2.211000 2.351000 4.013001 -2.211000
+ 4.703000 8.025001 -4.422000 4.703000 8.025001 -4.422000
+ 7.055000 12.039001 -6.633000 7.055000 12.039001 -6.633000
+ 9.407000 16.051001 -8.844000 9.407000 16.051001 -8.844000
+ 11.758000 20.064001 -11.055000 11.758000 20.064001 -11.055000
+ -8.298000 -23.248000 8.956000 -8.298000 -23.248000 8.956000
+ -5.946000 -19.236000 6.745000 -5.946000 -19.236000 6.745000
+ -3.595000 -15.223000 4.534000 -3.595000 -15.223000 4.534000
+ -1.243000 -11.211000 2.323000 -1.243000 -11.211000 2.323000
+ 1.108000 -7.197000 0.112000 1.108000 -7.197000 0.112000
+ 3.461000 -3.183999 -2.100000 3.461000 -3.183999 -2.100000
+ 5.812000 0.828001 -4.310000 5.812000 0.828001 -4.310000
+ 8.164000 4.841001 -6.522000 8.164000 4.841001 -6.522000
+ 10.515000 8.854001 -8.733000 10.515000 8.854001 -8.733000
+ 12.867000 12.867002 -10.944000 12.867000 12.867002 -10.944000
+ 15.219000 16.880002 -13.155000 15.219000 16.880002 -13.155000
+ -4.838000 -26.433000 6.857000 -4.838000 -26.433000 6.857000
+ -2.487000 -22.420000 4.646000 -2.487000 -22.420000 4.646000
+ -0.134000 -18.408000 2.434000 -0.134000 -18.408000 2.434000
+ 2.217000 -14.394999 0.224000 2.217000 -14.394999 0.224000
+ 4.569000 -10.381999 -1.988000 4.569000 -10.381999 -1.988000
+ 6.920000 -6.368999 -4.199000 6.920000 -6.368999 -4.199000
+ 9.272000 -2.355999 -6.410000 9.272000 -2.355999 -6.410000
+ 11.624000 1.656001 -8.621000 11.624000 1.656001 -8.621000
+ 13.976000 5.669002 -10.832000 13.976000 5.669002 -10.832000
+ 16.327000 9.682002 -13.043000 16.327000 9.682002 -13.043000
+ 18.679000 13.695002 -15.254000 18.679000 13.695002 -15.254000
+ 0.974000 -25.605000 2.546000 0.974000 -25.605000 2.546000
+ 3.325000 -21.591999 0.335000 3.325000 -21.591999 0.335000
+ 5.678000 -17.579999 -1.876000 5.678000 -17.579999 -1.876000
+ 8.029000 -13.566999 -4.087000 8.029000 -13.566999 -4.087000
+ 10.381000 -9.552999 -6.298000 10.381000 -9.552999 -6.298000
+ 12.732000 -5.540998 -8.509000 12.732000 -5.540998 -8.509000
+ 15.084000 -1.527998 -10.720000 15.084000 -1.527998 -10.720000
+ 17.436000 2.484002 -12.931000 17.436000 2.484002 -12.931000
+ 19.788000 6.498002 -15.143000 19.788000 6.498002 -15.143000
+ 6.786000 -24.776999 -1.764000 6.786000 -24.776999 -1.764000
+ 9.137000 -20.763999 -3.975000 9.137000 -20.763999 -3.975000
+ 11.489000 -16.751999 -6.186000 11.489000 -16.751999 -6.186000
+ 13.841000 -12.738998 -8.397000 13.841000 -12.738998 -8.397000
+ 16.193000 -8.724998 -10.609000 16.193000 -8.724998 -10.609000
+ 18.544000 -4.712998 -12.819000 18.544000 -4.712998 -12.819000
+ 20.896000 -0.699998 -15.031000 20.896000 -0.699998 -15.031000
+ 12.598000 -23.948998 -6.075000 12.598000 -23.948998 -6.075000
+ 14.950000 -19.935998 -8.286000 14.950000 -19.935998 -8.286000
+ 17.301000 -15.922998 -10.497000 17.301000 -15.922998 -10.497000
+ 19.653000 -11.909998 -12.708000 19.653000 -11.909998 -12.708000
+ 22.005000 -7.896998 -14.919000 22.005000 -7.896998 -14.919000
+]
diff --git a/MAC/Deployment/data/StaticMetaData/AntennaFields/PL611-AntennaField.conf b/MAC/Deployment/data/StaticMetaData/AntennaFields/PL611-AntennaField.conf
new file mode 100644
index 00000000000..0b3c98d52d1
--- /dev/null
+++ b/MAC/Deployment/data/StaticMetaData/AntennaFields/PL611-AntennaField.conf
@@ -0,0 +1,227 @@
+#
+# DUMMY AntennaPositions for PL611
+# ITRF2005 target_date = 2015.5
+# Created: 2015-06-23 19:24:00
+#
+
+NORMAL_VECTOR LBA
+3 [ 0.582686 0.102609 0.806194 ]
+
+ROTATION_MATRIX LBA
+3 x 3 [
+ -0.1384370000 -0.8008200000 0.5826860000
+ 0.9900310000 -0.0964870000 0.1026090000
+ -0.0259500000 0.5910820000 0.8061940000
+]
+
+LBA
+3 [ 3727207.315770000 655185.261474000 5117000.905 ]
+96 x 2 x 3 [
+ -2.385000 -13.491000 3.440990 -2.385000 -13.491000 3.440990
+ -7.901000 -8.415000 6.781990 -7.901000 -8.415000 6.781990
+ -5.396000 -14.324000 5.722990 -5.396000 -14.324000 5.722990
+-10.186000 -10.119001 8.649990 -10.186000 -10.119001 8.649990
+ -0.124000 20.778000 -2.555010 -0.124000 20.778000 -2.555010
+ -4.701000 24.130000 0.326990 -4.701000 24.130000 0.326990
+ 5.326000 25.166001 -7.052000 5.326000 25.166001 -7.052000
+ 2.267000 28.650001 -5.285010 2.267000 28.650001 -5.285010
+ -3.652000 -19.315000 5.097990 -3.652000 -19.315000 5.097990
+ 0.017010 -24.962000 3.165000 0.017010 -24.962000 3.165000
+ -1.878990 -28.049000 4.928000 -1.878990 -28.049000 4.928000
+ -5.235990 -27.314000 7.260990 -5.235990 -27.314000 7.260990
+ -0.077000 5.075000 -0.590010 -0.077000 5.075000 -0.590010
+ -3.595000 1.673000 2.385990 -3.595000 1.673000 2.385990
+ -5.802000 3.931000 3.692990 -5.802000 3.931000 3.692990
+ -4.688000 8.401000 2.318990 -4.688000 8.401000 2.318990
+ -8.598000 3.337999 5.788990 -8.598000 3.337999 5.788990
+ -9.953000 -1.251001 7.352990 -9.953000 -1.251001 7.352990
+-14.253000 5.953999 9.543990 -14.253000 5.953999 9.543990
+-16.063000 2.028999 11.351990 -16.063000 2.028999 11.351990
+ 17.152000 -3.302998 -11.976000 17.152000 -3.302998 -11.976000
+ 17.566000 -12.263998 -11.135000 17.566000 -12.263998 -11.135000
+ 24.669000 -4.977997 -17.197000 24.669000 -4.977997 -17.197000
+ 21.351010 -15.981998 -13.397000 21.351010 -15.981998 -13.397000
+ 3.774000 -12.505999 -1.136000 3.774000 -12.505999 -1.136000
+ 2.166000 -15.975999 0.468000 2.166000 -15.975999 0.468000
+ 6.991000 -14.399999 -3.220000 6.991000 -14.399999 -3.220000
+ 9.106010 -17.695999 -4.329000 9.106010 -17.695999 -4.329000
+ 12.973000 11.788002 -10.876000 12.973000 11.788002 -10.876000
+ 14.427000 15.115002 -12.351000 14.427000 15.115002 -12.351000
+ 19.444000 6.923002 -14.935000 19.444000 6.923002 -14.935000
+ 21.584000 12.082002 -17.138000 21.584000 12.082002 -17.138000
+-15.908000 9.413999 10.299990 -15.908000 9.413999 10.299990
+-15.203000 15.815999 8.974990 -15.203000 15.815999 8.974990
+-22.162000 21.404998 13.292990 -22.162000 21.404998 13.292990
+-23.439000 15.267998 14.997990 -23.439000 15.267998 14.997990
+ 3.902000 5.015001 -3.459000 3.902000 5.015001 -3.459000
+ 1.084000 9.810000 -2.032000 1.084000 9.810000 -2.032000
+ 6.434000 7.284001 -5.577000 6.434000 7.284001 -5.577000
+ 5.172000 10.742001 -5.105000 5.172000 10.742001 -5.105000
+-13.778000 -17.568001 12.193990 -13.778000 -17.568001 12.193990
+-17.938000 -8.584001 14.056990 -17.938000 -8.584001 14.056990
+-20.272000 -16.947002 16.808990 -20.272000 -16.947002 16.808990
+-23.586000 -7.012002 17.938990 -23.586000 -7.012002 17.938990
+ 5.987000 -1.138999 -4.182000 5.987000 -1.138999 -4.182000
+ 8.740000 0.756001 -6.413000 8.740000 0.756001 -6.413000
+ 7.442000 -5.689999 -4.655000 7.442000 -5.689999 -4.655000
+ 10.784000 -2.740999 -7.446000 10.784000 -2.740999 -7.446000
+ -4.781000 17.368000 1.244990 -4.781000 17.368000 1.244990
+ 16.096000 3.398002 -12.066000 16.096000 3.398002 -12.066000
+ -7.428000 28.024000 1.801990 -7.428000 28.024000 1.801990
+-12.607000 16.980999 6.950990 -12.607000 16.980999 6.950990
+ 13.586010 -19.721998 -7.309000 13.586010 -19.721998 -7.309000
+ 25.303000 2.547003 -18.612000 25.303000 2.547003 -18.612000
+-14.951000 24.151999 7.731990 -14.951000 24.151999 7.731990
+ -1.321000 8.211000 -0.090010 -1.321000 8.211000 -0.090010
+-11.883000 -24.202001 11.668990 -11.883000 -24.202001 11.668990
+ 13.136000 -12.848998 -7.859000 13.136000 -12.848998 -7.859000
+ 2.007000 -19.867999 1.078000 2.007000 -19.867999 1.078000
+ 17.313000 20.669002 -15.144000 17.313000 20.669002 -15.144000
+ 12.879000 4.744002 -9.912000 12.879000 4.744002 -9.912000
+ 12.378000 -9.072998 -7.792000 12.378000 -9.072998 -7.792000
+-14.753000 -7.090001 11.565990 -14.753000 -7.090001 11.565990
+ -5.135000 -2.138000 3.982990 -5.135000 -2.138000 3.982990
+ -0.165000 15.213000 -1.817010 -0.165000 15.213000 -1.817010
+ -1.108000 -7.354000 1.736990 -1.108000 -7.354000 1.736990
+ 10.425000 18.470001 -9.886000 10.425000 18.470001 -9.886000
+ 8.508000 11.428001 -7.603000 8.508000 11.428001 -7.603000
+ 7.179010 -22.427999 -2.334000 7.179010 -22.427999 -2.334000
+ 5.515010 -24.913999 -0.815000 5.515010 -24.913999 -0.815000
+ 2.140000 -4.021999 -1.035000 2.140000 -4.021999 -1.035000
+-10.858000 -14.764001 9.726990 -10.858000 -14.764001 9.726990
+ 9.646000 4.173001 -7.503000 9.646000 4.173001 -7.503000
+ -9.835000 12.923999 5.462990 -9.835000 12.923999 5.462990
+ 2.586000 15.972001 -3.902000 2.586000 15.972001 -3.902000
+-12.249000 25.292999 5.633990 -12.249000 25.292999 5.633990
+ 2.427000 -9.188999 -0.584000 2.427000 -9.188999 -0.584000
+ 13.400000 -0.374998 -9.637000 13.400000 -0.374998 -9.637000
+ -6.464000 -20.643000 7.298990 -6.464000 -20.643000 7.298990
+-20.268000 -0.557002 14.719990 -20.268000 -0.557002 14.719990
+-13.727000 -1.982001 10.172990 -13.727000 -1.982001 10.172990
+ -7.039000 20.879000 2.429990 -7.039000 20.879000 2.429990
+-11.211000 6.636999 7.257990 -11.211000 6.636999 7.257990
+ -1.969000 -3.827000 1.909990 -1.969000 -3.827000 1.909990
+ 21.307000 -1.589998 -15.198000 21.307000 -1.589998 -15.198000
+ 3.916000 1.518001 -3.024000 3.916000 1.518001 -3.024000
+ 10.477000 28.105001 -11.149000 10.477000 28.105001 -11.149000
+ 7.760000 19.672001 -8.112000 7.760000 19.672001 -8.112000
+ -7.629000 -3.386000 5.944990 -7.629000 -3.386000 5.944990
+-25.785000 3.657998 18.170990 -25.785000 3.657998 18.170990
+ -2.791000 -9.971000 3.286990 -2.791000 -9.971000 3.286990
+ 16.219010 -19.770998 -9.206000 16.219010 -19.770998 -9.206000
+-19.359000 7.202998 13.074990 -19.359000 7.202998 13.074990
+ 4.190000 -6.577999 -2.191000 4.190000 -6.577999 -2.191000
+ 8.585000 -8.974999 -5.063000 8.585000 -8.974999 -5.063000
+ -6.503000 11.141000 3.281990 -6.503000 11.141000 3.281990
+]
+
+NORMAL_VECTOR HBA
+3 [ 0.583126 0.102326 0.805912 ]
+
+ROTATION_MATRIX HBA
+3 x 3 [
+ -0.1382550000 -0.8005310000 0.5831260000
+ 0.9900640000 -0.0964520000 0.1023260000
+ -0.0256710000 0.5914790000 0.8059120000
+]
+
+HBA
+3 [ 3727217.665780000 655109.182475000 5117003.127 ]
+96 x 2 x 3 [
+-22.005000 7.896998 14.918990 -22.005000 7.896998 14.918990
+-19.653000 11.909998 12.707990 -19.653000 11.909998 12.707990
+-17.301000 15.922999 10.496990 -17.301000 15.922999 10.496990
+-14.950000 19.935999 8.286000 -14.950000 19.935999 8.286000
+-12.598000 23.948999 6.075000 -12.598000 23.948999 6.075000
+-20.896000 0.699998 15.030990 -20.896000 0.699998 15.030990
+-18.544000 4.712999 12.818990 -18.544000 4.712999 12.818990
+-16.193000 8.724999 10.609000 -16.193000 8.724999 10.609000
+-13.841000 12.738999 8.397000 -13.841000 12.738999 8.397000
+-11.489000 16.751999 6.186000 -11.489000 16.751999 6.186000
+ -9.137000 20.763999 3.975000 -9.137000 20.763999 3.975000
+ -6.786000 24.777000 1.764000 -6.786000 24.777000 1.764000
+-19.788000 -6.498002 15.142990 -19.788000 -6.498002 15.142990
+-17.436000 -2.484001 12.931000 -17.436000 -2.484001 12.931000
+-15.084000 1.527999 10.720000 -15.084000 1.527999 10.720000
+-12.732000 5.540999 8.509000 -12.732000 5.540999 8.509000
+-10.381000 9.552999 6.298000 -10.381000 9.552999 6.298000
+ -8.029000 13.567000 4.087000 -8.029000 13.567000 4.087000
+ -5.678000 17.580000 1.876000 -5.678000 17.580000 1.876000
+ -3.325000 21.592000 -0.335000 -3.325000 21.592000 -0.335000
+ -0.974000 25.605000 -2.546000 -0.974000 25.605000 -2.546000
+-18.679000 -13.695001 15.254000 -18.679000 -13.695001 15.254000
+-16.327000 -9.682001 13.043000 -16.327000 -9.682001 13.043000
+-13.976000 -5.669001 10.832000 -13.976000 -5.669001 10.832000
+-11.624000 -1.656001 8.621000 -11.624000 -1.656001 8.621000
+ -9.272000 2.355999 6.410000 -9.272000 2.355999 6.410000
+ -6.920000 6.369000 4.199000 -6.920000 6.369000 4.199000
+ -4.569000 10.382000 1.988000 -4.569000 10.382000 1.988000
+ -2.217000 14.395000 -0.224000 -2.217000 14.395000 -0.224000
+ 0.134000 18.408000 -2.434000 0.134000 18.408000 -2.434000
+ 2.487000 22.420001 -4.646000 2.487000 22.420001 -4.646000
+ 4.838000 26.433001 -6.857000 4.838000 26.433001 -6.857000
+-15.219000 -16.880001 13.155000 -15.219000 -16.880001 13.155000
+-12.867000 -12.867001 10.944000 -12.867000 -12.867001 10.944000
+-10.515000 -8.854001 8.733000 -10.515000 -8.854001 8.733000
+ -8.164000 -4.841000 6.522000 -8.164000 -4.841000 6.522000
+ -5.812000 -0.828000 4.310000 -5.812000 -0.828000 4.310000
+ -3.461000 3.184000 2.100000 -3.461000 3.184000 2.100000
+ -1.108000 7.197000 -0.112000 -1.108000 7.197000 -0.112000
+ 1.243000 11.211000 -2.323000 1.243000 11.211000 -2.323000
+ 3.595000 15.223001 -4.534000 3.595000 15.223001 -4.534000
+ 5.946000 19.236001 -6.745000 5.946000 19.236001 -6.745000
+ 8.298000 23.248001 -8.956000 8.298000 23.248001 -8.956000
+-11.758000 -20.064001 11.055000 -11.758000 -20.064001 11.055000
+ -9.407000 -16.051001 8.844000 -9.407000 -16.051001 8.844000
+ -7.055000 -12.039000 6.633000 -7.055000 -12.039000 6.633000
+ -4.703000 -8.025000 4.422000 -4.703000 -8.025000 4.422000
+ -2.351000 -4.013000 2.211000 -2.351000 -4.013000 2.211000
+ 2.351000 4.013001 -2.211000 2.351000 4.013001 -2.211000
+ 4.703000 8.025001 -4.422000 4.703000 8.025001 -4.422000
+ 7.055000 12.039001 -6.633000 7.055000 12.039001 -6.633000
+ 9.407000 16.051001 -8.844000 9.407000 16.051001 -8.844000
+ 11.758000 20.064001 -11.055000 11.758000 20.064001 -11.055000
+ -8.298000 -23.248000 8.956000 -8.298000 -23.248000 8.956000
+ -5.946000 -19.236000 6.745000 -5.946000 -19.236000 6.745000
+ -3.595000 -15.223000 4.534000 -3.595000 -15.223000 4.534000
+ -1.243000 -11.211000 2.323000 -1.243000 -11.211000 2.323000
+ 1.108000 -7.197000 0.112000 1.108000 -7.197000 0.112000
+ 3.461000 -3.183999 -2.100000 3.461000 -3.183999 -2.100000
+ 5.812000 0.828001 -4.310000 5.812000 0.828001 -4.310000
+ 8.164000 4.841001 -6.522000 8.164000 4.841001 -6.522000
+ 10.515000 8.854001 -8.733000 10.515000 8.854001 -8.733000
+ 12.867000 12.867002 -10.944000 12.867000 12.867002 -10.944000
+ 15.219000 16.880002 -13.155000 15.219000 16.880002 -13.155000
+ -4.838000 -26.433000 6.857000 -4.838000 -26.433000 6.857000
+ -2.487000 -22.420000 4.646000 -2.487000 -22.420000 4.646000
+ -0.134000 -18.408000 2.434000 -0.134000 -18.408000 2.434000
+ 2.217000 -14.394999 0.224000 2.217000 -14.394999 0.224000
+ 4.569000 -10.381999 -1.988000 4.569000 -10.381999 -1.988000
+ 6.920000 -6.368999 -4.199000 6.920000 -6.368999 -4.199000
+ 9.272000 -2.355999 -6.410000 9.272000 -2.355999 -6.410000
+ 11.624000 1.656001 -8.621000 11.624000 1.656001 -8.621000
+ 13.976000 5.669002 -10.832000 13.976000 5.669002 -10.832000
+ 16.327000 9.682002 -13.043000 16.327000 9.682002 -13.043000
+ 18.679000 13.695002 -15.254000 18.679000 13.695002 -15.254000
+ 0.974000 -25.605000 2.546000 0.974000 -25.605000 2.546000
+ 3.325000 -21.591999 0.335000 3.325000 -21.591999 0.335000
+ 5.678000 -17.579999 -1.876000 5.678000 -17.579999 -1.876000
+ 8.029000 -13.566999 -4.087000 8.029000 -13.566999 -4.087000
+ 10.381000 -9.552999 -6.298000 10.381000 -9.552999 -6.298000
+ 12.732000 -5.540998 -8.509000 12.732000 -5.540998 -8.509000
+ 15.084000 -1.527998 -10.720000 15.084000 -1.527998 -10.720000
+ 17.436000 2.484002 -12.931000 17.436000 2.484002 -12.931000
+ 19.788000 6.498002 -15.143000 19.788000 6.498002 -15.143000
+ 6.786000 -24.776999 -1.764000 6.786000 -24.776999 -1.764000
+ 9.137000 -20.763999 -3.975000 9.137000 -20.763999 -3.975000
+ 11.489000 -16.751999 -6.186000 11.489000 -16.751999 -6.186000
+ 13.841000 -12.738998 -8.397000 13.841000 -12.738998 -8.397000
+ 16.193000 -8.724998 -10.609000 16.193000 -8.724998 -10.609000
+ 18.544000 -4.712998 -12.819000 18.544000 -4.712998 -12.819000
+ 20.896000 -0.699998 -15.031000 20.896000 -0.699998 -15.031000
+ 12.598000 -23.948998 -6.075000 12.598000 -23.948998 -6.075000
+ 14.950000 -19.935998 -8.286000 14.950000 -19.935998 -8.286000
+ 17.301000 -15.922998 -10.497000 17.301000 -15.922998 -10.497000
+ 19.653000 -11.909998 -12.708000 19.653000 -11.909998 -12.708000
+ 22.005000 -7.896998 -14.919000 22.005000 -7.896998 -14.919000
+]
diff --git a/MAC/Deployment/data/StaticMetaData/AntennaFields/PL612-AntennaField.conf b/MAC/Deployment/data/StaticMetaData/AntennaFields/PL612-AntennaField.conf
new file mode 100644
index 00000000000..ab84688c73e
--- /dev/null
+++ b/MAC/Deployment/data/StaticMetaData/AntennaFields/PL612-AntennaField.conf
@@ -0,0 +1,227 @@
+#
+# DUMMY AntennaPositions for PL612
+# ITRF2005 target_date = 2015.5
+# Created: 2015-06-23 19:24:00
+#
+
+NORMAL_VECTOR LBA
+3 [ 0.582686 0.102609 0.806194 ]
+
+ROTATION_MATRIX LBA
+3 x 3 [
+ -0.1384370000 -0.8008200000 0.5826860000
+ 0.9900310000 -0.0964870000 0.1026090000
+ -0.0259500000 0.5910820000 0.8061940000
+]
+
+LBA
+3 [ 3727207.315770000 655185.261474000 5117000.905 ]
+96 x 2 x 3 [
+ -2.385000 -13.491000 3.440990 -2.385000 -13.491000 3.440990
+ -7.901000 -8.415000 6.781990 -7.901000 -8.415000 6.781990
+ -5.396000 -14.324000 5.722990 -5.396000 -14.324000 5.722990
+-10.186000 -10.119001 8.649990 -10.186000 -10.119001 8.649990
+ -0.124000 20.778000 -2.555010 -0.124000 20.778000 -2.555010
+ -4.701000 24.130000 0.326990 -4.701000 24.130000 0.326990
+ 5.326000 25.166001 -7.052000 5.326000 25.166001 -7.052000
+ 2.267000 28.650001 -5.285010 2.267000 28.650001 -5.285010
+ -3.652000 -19.315000 5.097990 -3.652000 -19.315000 5.097990
+ 0.017010 -24.962000 3.165000 0.017010 -24.962000 3.165000
+ -1.878990 -28.049000 4.928000 -1.878990 -28.049000 4.928000
+ -5.235990 -27.314000 7.260990 -5.235990 -27.314000 7.260990
+ -0.077000 5.075000 -0.590010 -0.077000 5.075000 -0.590010
+ -3.595000 1.673000 2.385990 -3.595000 1.673000 2.385990
+ -5.802000 3.931000 3.692990 -5.802000 3.931000 3.692990
+ -4.688000 8.401000 2.318990 -4.688000 8.401000 2.318990
+ -8.598000 3.337999 5.788990 -8.598000 3.337999 5.788990
+ -9.953000 -1.251001 7.352990 -9.953000 -1.251001 7.352990
+-14.253000 5.953999 9.543990 -14.253000 5.953999 9.543990
+-16.063000 2.028999 11.351990 -16.063000 2.028999 11.351990
+ 17.152000 -3.302998 -11.976000 17.152000 -3.302998 -11.976000
+ 17.566000 -12.263998 -11.135000 17.566000 -12.263998 -11.135000
+ 24.669000 -4.977997 -17.197000 24.669000 -4.977997 -17.197000
+ 21.351010 -15.981998 -13.397000 21.351010 -15.981998 -13.397000
+ 3.774000 -12.505999 -1.136000 3.774000 -12.505999 -1.136000
+ 2.166000 -15.975999 0.468000 2.166000 -15.975999 0.468000
+ 6.991000 -14.399999 -3.220000 6.991000 -14.399999 -3.220000
+ 9.106010 -17.695999 -4.329000 9.106010 -17.695999 -4.329000
+ 12.973000 11.788002 -10.876000 12.973000 11.788002 -10.876000
+ 14.427000 15.115002 -12.351000 14.427000 15.115002 -12.351000
+ 19.444000 6.923002 -14.935000 19.444000 6.923002 -14.935000
+ 21.584000 12.082002 -17.138000 21.584000 12.082002 -17.138000
+-15.908000 9.413999 10.299990 -15.908000 9.413999 10.299990
+-15.203000 15.815999 8.974990 -15.203000 15.815999 8.974990
+-22.162000 21.404998 13.292990 -22.162000 21.404998 13.292990
+-23.439000 15.267998 14.997990 -23.439000 15.267998 14.997990
+ 3.902000 5.015001 -3.459000 3.902000 5.015001 -3.459000
+ 1.084000 9.810000 -2.032000 1.084000 9.810000 -2.032000
+ 6.434000 7.284001 -5.577000 6.434000 7.284001 -5.577000
+ 5.172000 10.742001 -5.105000 5.172000 10.742001 -5.105000
+-13.778000 -17.568001 12.193990 -13.778000 -17.568001 12.193990
+-17.938000 -8.584001 14.056990 -17.938000 -8.584001 14.056990
+-20.272000 -16.947002 16.808990 -20.272000 -16.947002 16.808990
+-23.586000 -7.012002 17.938990 -23.586000 -7.012002 17.938990
+ 5.987000 -1.138999 -4.182000 5.987000 -1.138999 -4.182000
+ 8.740000 0.756001 -6.413000 8.740000 0.756001 -6.413000
+ 7.442000 -5.689999 -4.655000 7.442000 -5.689999 -4.655000
+ 10.784000 -2.740999 -7.446000 10.784000 -2.740999 -7.446000
+ -4.781000 17.368000 1.244990 -4.781000 17.368000 1.244990
+ 16.096000 3.398002 -12.066000 16.096000 3.398002 -12.066000
+ -7.428000 28.024000 1.801990 -7.428000 28.024000 1.801990
+-12.607000 16.980999 6.950990 -12.607000 16.980999 6.950990
+ 13.586010 -19.721998 -7.309000 13.586010 -19.721998 -7.309000
+ 25.303000 2.547003 -18.612000 25.303000 2.547003 -18.612000
+-14.951000 24.151999 7.731990 -14.951000 24.151999 7.731990
+ -1.321000 8.211000 -0.090010 -1.321000 8.211000 -0.090010
+-11.883000 -24.202001 11.668990 -11.883000 -24.202001 11.668990
+ 13.136000 -12.848998 -7.859000 13.136000 -12.848998 -7.859000
+ 2.007000 -19.867999 1.078000 2.007000 -19.867999 1.078000
+ 17.313000 20.669002 -15.144000 17.313000 20.669002 -15.144000
+ 12.879000 4.744002 -9.912000 12.879000 4.744002 -9.912000
+ 12.378000 -9.072998 -7.792000 12.378000 -9.072998 -7.792000
+-14.753000 -7.090001 11.565990 -14.753000 -7.090001 11.565990
+ -5.135000 -2.138000 3.982990 -5.135000 -2.138000 3.982990
+ -0.165000 15.213000 -1.817010 -0.165000 15.213000 -1.817010
+ -1.108000 -7.354000 1.736990 -1.108000 -7.354000 1.736990
+ 10.425000 18.470001 -9.886000 10.425000 18.470001 -9.886000
+ 8.508000 11.428001 -7.603000 8.508000 11.428001 -7.603000
+ 7.179010 -22.427999 -2.334000 7.179010 -22.427999 -2.334000
+ 5.515010 -24.913999 -0.815000 5.515010 -24.913999 -0.815000
+ 2.140000 -4.021999 -1.035000 2.140000 -4.021999 -1.035000
+-10.858000 -14.764001 9.726990 -10.858000 -14.764001 9.726990
+ 9.646000 4.173001 -7.503000 9.646000 4.173001 -7.503000
+ -9.835000 12.923999 5.462990 -9.835000 12.923999 5.462990
+ 2.586000 15.972001 -3.902000 2.586000 15.972001 -3.902000
+-12.249000 25.292999 5.633990 -12.249000 25.292999 5.633990
+ 2.427000 -9.188999 -0.584000 2.427000 -9.188999 -0.584000
+ 13.400000 -0.374998 -9.637000 13.400000 -0.374998 -9.637000
+ -6.464000 -20.643000 7.298990 -6.464000 -20.643000 7.298990
+-20.268000 -0.557002 14.719990 -20.268000 -0.557002 14.719990
+-13.727000 -1.982001 10.172990 -13.727000 -1.982001 10.172990
+ -7.039000 20.879000 2.429990 -7.039000 20.879000 2.429990
+-11.211000 6.636999 7.257990 -11.211000 6.636999 7.257990
+ -1.969000 -3.827000 1.909990 -1.969000 -3.827000 1.909990
+ 21.307000 -1.589998 -15.198000 21.307000 -1.589998 -15.198000
+ 3.916000 1.518001 -3.024000 3.916000 1.518001 -3.024000
+ 10.477000 28.105001 -11.149000 10.477000 28.105001 -11.149000
+ 7.760000 19.672001 -8.112000 7.760000 19.672001 -8.112000
+ -7.629000 -3.386000 5.944990 -7.629000 -3.386000 5.944990
+-25.785000 3.657998 18.170990 -25.785000 3.657998 18.170990
+ -2.791000 -9.971000 3.286990 -2.791000 -9.971000 3.286990
+ 16.219010 -19.770998 -9.206000 16.219010 -19.770998 -9.206000
+-19.359000 7.202998 13.074990 -19.359000 7.202998 13.074990
+ 4.190000 -6.577999 -2.191000 4.190000 -6.577999 -2.191000
+ 8.585000 -8.974999 -5.063000 8.585000 -8.974999 -5.063000
+ -6.503000 11.141000 3.281990 -6.503000 11.141000 3.281990
+]
+
+NORMAL_VECTOR HBA
+3 [ 0.583126 0.102326 0.805912 ]
+
+ROTATION_MATRIX HBA
+3 x 3 [
+ -0.1382550000 -0.8005310000 0.5831260000
+ 0.9900640000 -0.0964520000 0.1023260000
+ -0.0256710000 0.5914790000 0.8059120000
+]
+
+HBA
+3 [ 3727217.665780000 655109.182475000 5117003.127 ]
+96 x 2 x 3 [
+-22.005000 7.896998 14.918990 -22.005000 7.896998 14.918990
+-19.653000 11.909998 12.707990 -19.653000 11.909998 12.707990
+-17.301000 15.922999 10.496990 -17.301000 15.922999 10.496990
+-14.950000 19.935999 8.286000 -14.950000 19.935999 8.286000
+-12.598000 23.948999 6.075000 -12.598000 23.948999 6.075000
+-20.896000 0.699998 15.030990 -20.896000 0.699998 15.030990
+-18.544000 4.712999 12.818990 -18.544000 4.712999 12.818990
+-16.193000 8.724999 10.609000 -16.193000 8.724999 10.609000
+-13.841000 12.738999 8.397000 -13.841000 12.738999 8.397000
+-11.489000 16.751999 6.186000 -11.489000 16.751999 6.186000
+ -9.137000 20.763999 3.975000 -9.137000 20.763999 3.975000
+ -6.786000 24.777000 1.764000 -6.786000 24.777000 1.764000
+-19.788000 -6.498002 15.142990 -19.788000 -6.498002 15.142990
+-17.436000 -2.484001 12.931000 -17.436000 -2.484001 12.931000
+-15.084000 1.527999 10.720000 -15.084000 1.527999 10.720000
+-12.732000 5.540999 8.509000 -12.732000 5.540999 8.509000
+-10.381000 9.552999 6.298000 -10.381000 9.552999 6.298000
+ -8.029000 13.567000 4.087000 -8.029000 13.567000 4.087000
+ -5.678000 17.580000 1.876000 -5.678000 17.580000 1.876000
+ -3.325000 21.592000 -0.335000 -3.325000 21.592000 -0.335000
+ -0.974000 25.605000 -2.546000 -0.974000 25.605000 -2.546000
+-18.679000 -13.695001 15.254000 -18.679000 -13.695001 15.254000
+-16.327000 -9.682001 13.043000 -16.327000 -9.682001 13.043000
+-13.976000 -5.669001 10.832000 -13.976000 -5.669001 10.832000
+-11.624000 -1.656001 8.621000 -11.624000 -1.656001 8.621000
+ -9.272000 2.355999 6.410000 -9.272000 2.355999 6.410000
+ -6.920000 6.369000 4.199000 -6.920000 6.369000 4.199000
+ -4.569000 10.382000 1.988000 -4.569000 10.382000 1.988000
+ -2.217000 14.395000 -0.224000 -2.217000 14.395000 -0.224000
+ 0.134000 18.408000 -2.434000 0.134000 18.408000 -2.434000
+ 2.487000 22.420001 -4.646000 2.487000 22.420001 -4.646000
+ 4.838000 26.433001 -6.857000 4.838000 26.433001 -6.857000
+-15.219000 -16.880001 13.155000 -15.219000 -16.880001 13.155000
+-12.867000 -12.867001 10.944000 -12.867000 -12.867001 10.944000
+-10.515000 -8.854001 8.733000 -10.515000 -8.854001 8.733000
+ -8.164000 -4.841000 6.522000 -8.164000 -4.841000 6.522000
+ -5.812000 -0.828000 4.310000 -5.812000 -0.828000 4.310000
+ -3.461000 3.184000 2.100000 -3.461000 3.184000 2.100000
+ -1.108000 7.197000 -0.112000 -1.108000 7.197000 -0.112000
+ 1.243000 11.211000 -2.323000 1.243000 11.211000 -2.323000
+ 3.595000 15.223001 -4.534000 3.595000 15.223001 -4.534000
+ 5.946000 19.236001 -6.745000 5.946000 19.236001 -6.745000
+ 8.298000 23.248001 -8.956000 8.298000 23.248001 -8.956000
+-11.758000 -20.064001 11.055000 -11.758000 -20.064001 11.055000
+ -9.407000 -16.051001 8.844000 -9.407000 -16.051001 8.844000
+ -7.055000 -12.039000 6.633000 -7.055000 -12.039000 6.633000
+ -4.703000 -8.025000 4.422000 -4.703000 -8.025000 4.422000
+ -2.351000 -4.013000 2.211000 -2.351000 -4.013000 2.211000
+ 2.351000 4.013001 -2.211000 2.351000 4.013001 -2.211000
+ 4.703000 8.025001 -4.422000 4.703000 8.025001 -4.422000
+ 7.055000 12.039001 -6.633000 7.055000 12.039001 -6.633000
+ 9.407000 16.051001 -8.844000 9.407000 16.051001 -8.844000
+ 11.758000 20.064001 -11.055000 11.758000 20.064001 -11.055000
+ -8.298000 -23.248000 8.956000 -8.298000 -23.248000 8.956000
+ -5.946000 -19.236000 6.745000 -5.946000 -19.236000 6.745000
+ -3.595000 -15.223000 4.534000 -3.595000 -15.223000 4.534000
+ -1.243000 -11.211000 2.323000 -1.243000 -11.211000 2.323000
+ 1.108000 -7.197000 0.112000 1.108000 -7.197000 0.112000
+ 3.461000 -3.183999 -2.100000 3.461000 -3.183999 -2.100000
+ 5.812000 0.828001 -4.310000 5.812000 0.828001 -4.310000
+ 8.164000 4.841001 -6.522000 8.164000 4.841001 -6.522000
+ 10.515000 8.854001 -8.733000 10.515000 8.854001 -8.733000
+ 12.867000 12.867002 -10.944000 12.867000 12.867002 -10.944000
+ 15.219000 16.880002 -13.155000 15.219000 16.880002 -13.155000
+ -4.838000 -26.433000 6.857000 -4.838000 -26.433000 6.857000
+ -2.487000 -22.420000 4.646000 -2.487000 -22.420000 4.646000
+ -0.134000 -18.408000 2.434000 -0.134000 -18.408000 2.434000
+ 2.217000 -14.394999 0.224000 2.217000 -14.394999 0.224000
+ 4.569000 -10.381999 -1.988000 4.569000 -10.381999 -1.988000
+ 6.920000 -6.368999 -4.199000 6.920000 -6.368999 -4.199000
+ 9.272000 -2.355999 -6.410000 9.272000 -2.355999 -6.410000
+ 11.624000 1.656001 -8.621000 11.624000 1.656001 -8.621000
+ 13.976000 5.669002 -10.832000 13.976000 5.669002 -10.832000
+ 16.327000 9.682002 -13.043000 16.327000 9.682002 -13.043000
+ 18.679000 13.695002 -15.254000 18.679000 13.695002 -15.254000
+ 0.974000 -25.605000 2.546000 0.974000 -25.605000 2.546000
+ 3.325000 -21.591999 0.335000 3.325000 -21.591999 0.335000
+ 5.678000 -17.579999 -1.876000 5.678000 -17.579999 -1.876000
+ 8.029000 -13.566999 -4.087000 8.029000 -13.566999 -4.087000
+ 10.381000 -9.552999 -6.298000 10.381000 -9.552999 -6.298000
+ 12.732000 -5.540998 -8.509000 12.732000 -5.540998 -8.509000
+ 15.084000 -1.527998 -10.720000 15.084000 -1.527998 -10.720000
+ 17.436000 2.484002 -12.931000 17.436000 2.484002 -12.931000
+ 19.788000 6.498002 -15.143000 19.788000 6.498002 -15.143000
+ 6.786000 -24.776999 -1.764000 6.786000 -24.776999 -1.764000
+ 9.137000 -20.763999 -3.975000 9.137000 -20.763999 -3.975000
+ 11.489000 -16.751999 -6.186000 11.489000 -16.751999 -6.186000
+ 13.841000 -12.738998 -8.397000 13.841000 -12.738998 -8.397000
+ 16.193000 -8.724998 -10.609000 16.193000 -8.724998 -10.609000
+ 18.544000 -4.712998 -12.819000 18.544000 -4.712998 -12.819000
+ 20.896000 -0.699998 -15.031000 20.896000 -0.699998 -15.031000
+ 12.598000 -23.948998 -6.075000 12.598000 -23.948998 -6.075000
+ 14.950000 -19.935998 -8.286000 14.950000 -19.935998 -8.286000
+ 17.301000 -15.922998 -10.497000 17.301000 -15.922998 -10.497000
+ 19.653000 -11.909998 -12.708000 19.653000 -11.909998 -12.708000
+ 22.005000 -7.896998 -14.919000 22.005000 -7.896998 -14.919000
+]
diff --git a/MAC/Deployment/data/StaticMetaData/RSPImage.conf b/MAC/Deployment/data/StaticMetaData/RSPImage.conf
index 58786a949a9..dca5dd030da 100644
--- a/MAC/Deployment/data/StaticMetaData/RSPImage.conf
+++ b/MAC/Deployment/data/StaticMetaData/RSPImage.conf
@@ -55,3 +55,6 @@ SE607C 4
UK608C 4
DE609C 4
FI609C 4
+PL610C 4
+PL611C 4
+PL612C 4
diff --git a/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh b/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh
index ae12a3cf7fc..ecc5fb80b21 100755
--- a/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh
+++ b/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh
@@ -65,7 +65,7 @@ function usage {
# It wait on the processes finish (using the PID) with a
# succesfull return value
# - On signals kill the child process
-# - On non zero return value of the command it will retry
+# - On ssh and bash errors it will retry
# with increasingly larger wait periods between tries
function command_retry {
COMMAND="$1"
@@ -78,8 +78,22 @@ function command_retry {
# Trap 'all' signals and forward to ssh process
TRAP_COMMAND="kill $SSH_PID; break"
trap "$TRAP_COMMAND" SIGTERM SIGINT SIGQUIT SIGHUP 2> /dev/null
- wait $SSH_PID && break # wait while the ssh command is up
- # Break the loop if the command returned with exit value 0
+
+ # wait for ssh to finish
+ wait $SSH_PID
+
+ # Return codes:
+ # 255: SSH fails
+ # 127: BASH 'command not found'
+ # 126: BASH 'command not executable'
+ # smaller: outputProc fails
+ # 0: success
+
+ # Break the loop if the command was started -- there is no need
+ # to keep starting it if the command itself returned an error.
+ if [ "$?" -lt 126 ]; then
+ break
+ fi
sleep $SLEEP_DURATION # Sleep if ssh failed
SLEEP_DURATION=$((SLEEP_DURATION + 1)) # Increase duration
diff --git a/SAS/Scheduler/src/Controller.cpp b/SAS/Scheduler/src/Controller.cpp
index e59b2ccdac2..55e23f1ec3a 100644
--- a/SAS/Scheduler/src/Controller.cpp
+++ b/SAS/Scheduler/src/Controller.cpp
@@ -3321,10 +3321,6 @@ std::pair<unscheduled_reasons, QString> Controller::doPreScheduleChecks(Task *ta
}
}
- // TODO: setInputFilesForPipeline should probably not be done here. Only set the input files when a task is downloaded from SAS or when it is just loaded from disk
- // now the enabled flags (user selection) gets reset by calling setInputFilesForPipeline which is also a bug. This should not be the case
- error = setInputFilesForPipeline(pPipe);
-
if (pPipe->isCalibrationPipeline() &&
!task->storage()->getEqualityInputOutputProducts())
{
diff --git a/SAS/Scheduler/src/taskstorage.cpp b/SAS/Scheduler/src/taskstorage.cpp
index 8da804abba1..c83a8090074 100644
--- a/SAS/Scheduler/src/taskstorage.cpp
+++ b/SAS/Scheduler/src/taskstorage.cpp
@@ -297,6 +297,7 @@ QDataStream& operator>> (QDataStream &in, TaskStorage &storage) {
return in;
}
+<<<<<<< .working
// Returns True if the input and output node locations are equal for
// all the input and output products
// THis function should be moved to the pipeline class?
@@ -335,6 +336,53 @@ bool TaskStorage::getEqualityInputOutputProducts()const
return true;
}
+=======
+// Returns True if the input and output node locations are equal for
+// all the input and output products
+// THis function should be moved to the pipeline class?
+bool TaskStorage::getEqualityInputOutputProducts()const
+{
+ // Check we have the same number of dataproduct types
+ //if (itsInputDataProducts.size() != itsOutputDataProducts.size())
+ // return false;
+
+ //loop over the input and output data types
+ std::map<dataProductTypes, inputDataProduct >::const_iterator inputTypePair;
+ std::map<dataProductTypes, outputDataProduct >::const_iterator outputTypePair;
+ for (outputTypePair = itsOutputDataProducts.begin();
+ outputTypePair != itsOutputDataProducts.end(); // length is the same
+ ++outputTypePair )
+ {
+ if(itsInputDataProducts.find(outputTypePair->first) !=
+ itsInputDataProducts.end())
+ continue;
+ inputDataProduct input = itsInputDataProducts.at(outputTypePair->first);
+
+
+ // Check if we have the same number of input and output entries
+ if (input.locations.size() !=
+ outputTypePair->second.locations.size())
+ return false;
+
+ // Loop over all the input and output locations
+ QStringList::const_iterator inputLoc;
+ QStringList::const_iterator outputLoc;
+ for (inputLoc = input.locations.begin(),
+ outputLoc = outputTypePair->second.locations.begin();
+ inputLoc != input.locations.end();
+ ++inputLoc , ++outputLoc)
+ {
+ //return false if the nodes are not the same
+ if (inputLoc->split(":").at(0) != outputLoc->split(":").at(0))
+ return false;
+ }
+ }
+ return true;
+}
+
+
+
+>>>>>>> .merge-right.r32112
void TaskStorage::setInputFileSizes(dataProductTypes dpType, const std::pair<double, unsigned> &inputFileSizes) {
itsInputDataFiles[dpType] = inputFileSizes;
itsRecalcStorageNeeded = true;
diff --git a/SubSystems/CMakeLists.txt b/SubSystems/CMakeLists.txt
index 0b138640c09..92561d79208 100644
--- a/SubSystems/CMakeLists.txt
+++ b/SubSystems/CMakeLists.txt
@@ -9,5 +9,6 @@ lofar_add_package(Online_OutputProc)
lofar_add_package(ST_MAC)
lofar_add_package(Offline)
lofar_add_package(SAS_OTDB)
+lofar_add_package(SAS_Tools)
lofar_add_package(PVSS_DB)
lofar_add_package(LAPS_CEP)
diff --git a/SubSystems/SAS_Tools/CMakeLists.txt b/SubSystems/SAS_Tools/CMakeLists.txt
new file mode 100644
index 00000000000..8ed2f41e74c
--- /dev/null
+++ b/SubSystems/SAS_Tools/CMakeLists.txt
@@ -0,0 +1,4 @@
+# $Id: CMakeLists.txt 31909 2015-06-23 20:58:05Z schoenmakers $
+
+lofar_package(SAS_Tools
+ DEPENDS Scheduler CleanupTool)
--
GitLab