diff --git a/.gitattributes b/.gitattributes
index af2242b14cd72f4805ed8df8780c5d3a27a98cc2..87d79ca583e4d6597cd2e331e7d91efedf5e9dd4 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1087,6 +1087,9 @@ CEP/Pipeline/recipes/sip/helpers/WritableParmDB.py -text
CEP/Pipeline/recipes/sip/helpers/__init__.py eol=lf
CEP/Pipeline/recipes/sip/helpers/data_quality.py eol=lf
CEP/Pipeline/recipes/sip/helpers/metadata.py eol=lf
+CEP/Pipeline/recipes/sip/helpers/test/CMakeLists.txt -text
+CEP/Pipeline/recipes/sip/helpers/test/t_metadata.py -text
+CEP/Pipeline/recipes/sip/helpers/test/t_metadata.sh -text
CEP/Pipeline/recipes/sip/master/__init__.py eol=lf
CEP/Pipeline/recipes/sip/master/copier.py -text
CEP/Pipeline/recipes/sip/master/deprecated/bbs.py eol=lf
diff --git a/CEP/Pipeline/recipes/sip/CMakeLists.txt b/CEP/Pipeline/recipes/sip/CMakeLists.txt
index e52af408f9598e286becfc725e886711e1ed31a0..87cdd00cf499e483d4c98183c457abe45b49490a 100644
--- a/CEP/Pipeline/recipes/sip/CMakeLists.txt
+++ b/CEP/Pipeline/recipes/sip/CMakeLists.txt
@@ -143,3 +143,4 @@ configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/tasks.cfg.CEP4.in
${CMAKE_CURRENT_BINARY_DIR}/tasks.cfg.CEP4)
+add_subdirectory(helpers/test/)
diff --git a/CEP/Pipeline/recipes/sip/helpers/metadata.py b/CEP/Pipeline/recipes/sip/helpers/metadata.py
index 247761de26ea56938e31469d793e1797a7b8226b..fdf83778f1ecbca87d9ab6073ebcef66cfe93387 100644
--- a/CEP/Pipeline/recipes/sip/helpers/metadata.py
+++ b/CEP/Pipeline/recipes/sip/helpers/metadata.py
@@ -79,6 +79,65 @@ def to_parset(data, prefix=''):
result.replace(fullkey, str(value))
return result
+class StorageWriterTypes:
+ '''
+ type definitions for the used storage writer for the dataproducts,
+ and method to get the type and version from an MS table
+ '''
+ CASA = 'CASA' # any default casa storage manager in MS datasets
+ LOFAR = 'LOFAR' # the lofar storage manager in MS datasets
+ DYSCO = 'DYSCO' # the dyscostorage manager in MS datasets
+ HDF5DEFAULT = 'HDF5DEFAULT' # normal hdf5
+ UNKNOWN = 'UNKNOWN' # miscellaneous/unknown
+
+ UNKNOWN_VERSION = 'UNKNOWN'
+
+ @staticmethod
+ def get_type_and_version(main_table, logger=None):
+ '''
+ tries to determine the used StorageWriterType and version from a MS main table
+ :param main_table: the main table of a casacore Measurement Set
+ :param logger: a logging.logger instance
+ :return: a tuple of (StorageWriterTypes.<constant>, '<version>')
+ '''
+ try:
+ dminfo = main_table.getdminfo('DATA')
+ dataManagerType = dminfo.get('TYPE')
+
+ if dataManagerType in ['StandardStMan', 'IncrementalStMan',
+ 'TiledColumnStMan', 'TiledCellStMan', 'TiledShapeStMan']:
+ return StorageWriterTypes.CASA, StorageWriterTypes.get_casa_version()
+
+ if dataManagerType == 'LofarStMan':
+ try:
+ version = dminfo['SPEC']['version']
+ except:
+ version = StorageWriterTypes.UNKNOWN_VERSION
+ return StorageWriterTypes.LOFAR, version
+
+ if dataManagerType == 'DyscoStMan':
+ version = os.environ.get('DYSCO_VERSION', StorageWriterTypes.UNKNOWN_VERSION)
+ return StorageWriterTypes.DYSCO, StorageWriterTypes.get_dysco_version()
+
+ except Exception as e:
+ if logger:
+ logger.error('Could not determine the used storageWriter type and version: %s', e)
+
+ return StorageWriterTypes.UNKNOWN, StorageWriterTypes.UNKNOWN_VERSION
+
+ @staticmethod
+ def get_casa_version():
+ '''
+ :return the casa version from the environment, or unknown
+ '''
+ return os.environ.get('CASACORE_VERSION', StorageWriterTypes.UNKNOWN_VERSION)
+
+ @staticmethod
+ def get_dysco_version():
+ '''
+ :return the dysco version from the environment, or unknown
+ '''
+ return os.environ.get('DYSCO_VERSION', StorageWriterTypes.UNKNOWN_VERSION)
class DataProduct(object):
@@ -91,7 +150,10 @@ class DataProduct(object):
'fileFormat' : "",
'filename' : "",
'location' : "",
- 'percentageWritten' : 0
+ 'percentageWritten' : 0,
+ # by default the type of storagewriter is unknown and overridden in each subclass if needed
+ 'storageWriter': StorageWriterTypes.UNKNOWN,
+ 'storageWriterVersion': StorageWriterTypes.UNKNOWN_VERSION
}
self.logger = logger
@@ -163,16 +225,21 @@ class Correlated(DataProduct):
pyrap.tables.taql('calc ctod($startTime s)')[0]
.replace('/', '-', 2).replace('/', 'T')
)
+
+ sw_type, sw_version = StorageWriterTypes.get_type_and_version(main, self.logger)
+
self._data.update({
'percentageWritten' : 100,
+ 'storageWriter': sw_type,
+ 'storageWriterVersion': sw_version,
'startTime' : startTimeString,
'duration' : endTime - startTime,
'integrationInterval' : exposure,
'centralFrequency' : spw.getcell('REF_FREQUENCY', 0),
- 'channelWidth' : spw.getcell('RESOLUTION', 0)[0],
+ 'channelWidth' : spw.getcell('RESOLUTION', [0])[0],
'channelsPerSubband' : spw.getcell('NUM_CHAN', 0),
# Assume subband name has format 'SB-nn'
- 'subband' : int(spw.getcell('NAME', 0)[3:]),
+ 'subband' : int(spw.getcell('NAME', 'SB000')[3:]),
'stationSubband' : 0 ### NOT CORRECT! ###
})
except Exception, error:
@@ -202,11 +269,12 @@ class InstrumentModel(DataProduct):
Collect instrument model metadata from the Measurement Set `filename`.
"""
super(InstrumentModel, self).collect(filename)
+ self._data['storageWriter'] = StorageWriterTypes.CASA
+ self._data['storageWriterVersion'] = StorageWriterTypes.get_casa_version()
if self._data['size'] > 0:
self._data['percentageWritten'] = 100
-
class SkyImage(DataProduct):
"""
Class representing the metadata associated with a sky image.
@@ -242,6 +310,16 @@ class SkyImage(DataProduct):
"""
super(SkyImage, self).collect(filename)
try:
+ if filename.endswith('.h5'):
+ self._data['storageWriter'] = StorageWriterTypes.HDF5DEFAULT
+ self._data['storageWriterVersion'] = StorageWriterTypes.UNKNOWN_VERSION
+ elif filename.endswith('.IM'):
+ self._data['storageWriter'] = StorageWriterTypes.CASA
+ self._data['storageWriterVersion'] = StorageWriterTypes.get_casa_version()
+ else:
+ self._data['storageWriter'] = StorageWriterTypes.UNKNOWN
+ self._data['storageWriterVersion'] = StorageWriterTypes.UNKNOWN_VERSION
+
image = pyrap.images.image(filename)
coord = image.coordinates()
beaminfo = image.imageinfo()['restoringbeam']
diff --git a/CEP/Pipeline/recipes/sip/helpers/test/CMakeLists.txt b/CEP/Pipeline/recipes/sip/helpers/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f566fde1ae2292337318648b14ed1805886363b5
--- /dev/null
+++ b/CEP/Pipeline/recipes/sip/helpers/test/CMakeLists.txt
@@ -0,0 +1,6 @@
+# $Id$
+include(LofarCTest)
+
+lofar_add_test(t_metadata)
+
+
diff --git a/CEP/Pipeline/recipes/sip/helpers/test/t_metadata.py b/CEP/Pipeline/recipes/sip/helpers/test/t_metadata.py
new file mode 100755
index 0000000000000000000000000000000000000000..c62fee90f82bc9b25376f9abefba7436e41b85cc
--- /dev/null
+++ b/CEP/Pipeline/recipes/sip/helpers/test/t_metadata.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2017
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from lofarpipe.recipes.helpers.metadata import *
+from numpy import *
+import mock
+
+import logging
+logger = logging.getLogger(__name__)
+
+class AbstractMockTable:
+ '''a mocked version of the pyrap.tables.table class,
+ can be used with dependency injection in the tests'''
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def getkeyword(self, *args, **kwargs):
+ return ''
+
+ def getcell(self, name, default=None):
+ return default
+
+ def nrows(self):
+ return 0
+
+ def getdminfo(self, columnname):
+ return {}
+
+class LofarMockTable(AbstractMockTable):
+ def getdminfo(self, columnname):
+ assert columnname == 'DATA'
+ # return a real world example of datamanager info from a MS with a LofarStMan
+ return {'NAME': 'LofarStMan',
+ 'SEQNR': 0,
+ 'SPEC': {'alignment': 512,
+ 'bigEndian': False,
+ 'maxNrSample': 3056.0,
+ 'nbaseline': 741,
+ 'nrBytesPerNrValidSamples': 2,
+ 'startTime': 5022530520.500695,
+ 'timeInterval': 1.00139008,
+ 'useSeqnrFile': True,
+ 'version': 3},
+ 'TYPE': 'LofarStMan'}
+
+class DyscoMockTable(AbstractMockTable):
+ def getdminfo(self, columnname):
+ assert columnname == 'DATA'
+ # return a real world example of datamanager info from a MS with a DyscoStMan
+ return {'NAME': 'DyscoData',
+ 'SEQNR': 3,
+ 'SPEC': {'dataBitCount': 10,
+ 'distribution': 'TruncatedGaussian',
+ 'distributionTruncation': 2.5,
+ 'normalization': 'AF',
+ 'studentTNu': 0.0,
+ 'weightBitCount': 12},
+ 'TYPE': 'DyscoStMan'}
+
+class CasaTiledMockTable(AbstractMockTable):
+ def getdminfo(self, columnname):
+ assert columnname == 'DATA'
+ # return a real world example of datamanager info from a MS with a TiledColumnStMan
+ return {'NAME': 'TiledFlag',
+ 'SEQNR': 4,
+ 'SPEC': {'ActualMaxCacheSize': 0,
+ 'DEFAULTTILESHAPE': array([4, 4, 65536], dtype=int32),
+ 'HYPERCUBES': {'*1': {'BucketSize': 131072,
+ 'CellShape': array([4, 4], dtype=int32),
+ 'CubeShape': array([4, 4, 5993949], dtype=int32),
+ 'ID': {},
+ 'TileShape': array([4, 4, 65536], dtype=int32)}},
+ 'MAXIMUMCACHESIZE': 0,
+ 'SEQNR': 4},
+ 'TYPE': 'TiledColumnStMan'}
+
+class CasaStandardMockTable(AbstractMockTable):
+ def getdminfo(self, columnname):
+ assert columnname == 'DATA'
+ # return a real world example of datamanager info from a MS with a StandardStMan
+ return {'NAME': 'SSMVar',
+ 'SEQNR': 0,
+ 'SPEC': {'ActualCacheSize': 2,
+ 'BUCKETSIZE': 32768,
+ 'IndexLength': 11830,
+ 'PERSCACHESIZE': 2},
+ 'TYPE': 'StandardStMan'}
+
+
+# for some reason, the casa and dysco versions are 'encoded' in the running environment
+# define the here and set them for this test in the enviroment
+CASA_VERSION = "2.2.0"
+DYSCO_VERSION = "1.01"
+os.environ['CASACORE_VERSION'] = CASA_VERSION
+os.environ['DYSCO_VERSION'] = DYSCO_VERSION
+
+
+class StorageWriterTypesTest(unittest.TestCase):
+ '''
+ Tests the StorageWriterTypes class
+ '''
+
+ def test_get_type_and_version_casa_standard(self):
+ main = CasaStandardMockTable()
+ sw_type, sw_version = StorageWriterTypes.get_type_and_version(main)
+ self.assertEqual(StorageWriterTypes.CASA, sw_type)
+ self.assertEqual(CASA_VERSION, sw_version)
+
+ def test_get_type_and_version_casa_tiled(self):
+ main = CasaTiledMockTable()
+ sw_type, sw_version = StorageWriterTypes.get_type_and_version(main)
+ self.assertEqual(StorageWriterTypes.CASA, sw_type)
+ self.assertEqual(CASA_VERSION, sw_version)
+
+ def test_get_type_and_version_dysco(self):
+ main = DyscoMockTable()
+ sw_type, sw_version = StorageWriterTypes.get_type_and_version(main)
+ self.assertEqual(StorageWriterTypes.DYSCO, sw_type)
+ self.assertEqual(DYSCO_VERSION, sw_version)
+
+ def test_get_type_and_version_lofar(self):
+ main = LofarMockTable()
+ sw_type, sw_version = StorageWriterTypes.get_type_and_version(main)
+ self.assertEqual(StorageWriterTypes.LOFAR, sw_type)
+ self.assertEqual(3, sw_version)
+
+ def test_get_type_and_version_unknown(self):
+ main = AbstractMockTable()
+ sw_type, sw_version = StorageWriterTypes.get_type_and_version(main)
+ self.assertEqual(StorageWriterTypes.UNKNOWN, sw_type)
+ self.assertEqual(StorageWriterTypes.UNKNOWN_VERSION, sw_version)
+
+
+class MetaDataTest(unittest.TestCase):
+ '''
+ Tests the creation of correct meta data parsets
+ '''
+
+ def test_correlated_casa_standard(self):
+ with mock.patch('pyrap.tables.table', new=CasaStandardMockTable):
+ dataproduct_metadata = Correlated(logger=None, filename='casa-standard')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('casa_standard metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.CASA, metadata_parset.getString('storageWriter'))
+ self.assertEqual(CASA_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_correlated_casa_tiled(self):
+ with mock.patch('pyrap.tables.table', new=CasaTiledMockTable):
+ dataproduct_metadata = Correlated(logger=None, filename='casa-tiled')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('casa_lofar metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.CASA, metadata_parset.getString('storageWriter'))
+ self.assertEqual(CASA_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_correlated_lofar(self):
+ with mock.patch('pyrap.tables.table', new=LofarMockTable):
+ dataproduct_metadata = Correlated(logger=None, filename='lofar')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('lofar metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.LOFAR, metadata_parset.getString('storageWriter'))
+ self.assertEqual('3', metadata_parset.getString('storageWriterVersion'))
+
+ def test_correlated_dysco(self):
+ with mock.patch('pyrap.tables.table', new=DyscoMockTable):
+ dataproduct_metadata = Correlated(logger=None, filename='dysco')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('dysco metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.DYSCO, metadata_parset.getString('storageWriter'))
+ self.assertEqual(DYSCO_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_correlated_unknown(self):
+ with mock.patch('pyrap.tables.table', new=AbstractMockTable):
+ dataproduct_metadata = Correlated(logger=None, filename='foo.bar')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('unknown metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.UNKNOWN, metadata_parset.getString('storageWriter'))
+ self.assertEqual(StorageWriterTypes.UNKNOWN_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_instrument_model(self):
+ with mock.patch('pyrap.images.image'):
+ dataproduct_metadata = InstrumentModel(logger=None, filename='foo.INST')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('instrument model metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.CASA, metadata_parset.getString('storageWriter'))
+ self.assertEqual(CASA_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_skyimage_h5(self):
+ with mock.patch('pyrap.images.image'):
+ dataproduct_metadata = SkyImage(logger=None, filename='foo.h5')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('instrument model metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.HDF5DEFAULT, metadata_parset.getString('storageWriter'))
+ self.assertEqual(StorageWriterTypes.UNKNOWN_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_skyimage_casa(self):
+ with mock.patch('pyrap.images.image'):
+ dataproduct_metadata = SkyImage(logger=None, filename='foo.IM')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('instrument model metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.CASA, metadata_parset.getString('storageWriter'))
+ self.assertEqual(CASA_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+ def test_skyimage_other(self):
+ with mock.patch('pyrap.images.image'):
+ dataproduct_metadata = SkyImage(logger=None, filename='foo.fits')
+ metadata_parset = dataproduct_metadata.as_parameterset()
+ logger.info('instrument model metadata parset:\n%s', metadata_parset)
+ self.assertEqual(StorageWriterTypes.UNKNOWN, metadata_parset.getString('storageWriter'))
+ self.assertEqual(StorageWriterTypes.UNKNOWN_VERSION, metadata_parset.getString('storageWriterVersion'))
+
+
+if __name__ == '__main__':
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG)
+ unittest.main()
+
diff --git a/CEP/Pipeline/recipes/sip/helpers/test/t_metadata.sh b/CEP/Pipeline/recipes/sip/helpers/test/t_metadata.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f5b276f5ce3931415e054a13e4ae816f3577573b
--- /dev/null
+++ b/CEP/Pipeline/recipes/sip/helpers/test/t_metadata.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_metadata