Skip to content
Snippets Groups Projects
Commit 4b3f7ee2 authored by Jörn Künsemöller's avatar Jörn Künsemöller
Browse files

TMSS-281: Update SIP library to work against latest XSD

parent 56fc4bd8
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
lib/constants.py
+ 8
− 0
View file @ 4b3f7ee2
...@@ -40,6 +40,7 @@ DATAPRODUCTTYPE_SKY_IMAGE="Sky Image" ...@@ -40,6 +40,7 @@ DATAPRODUCTTYPE_SKY_IMAGE="Sky Image"
DATAPRODUCTTYPE_SKY_MODEL="Sky Model" DATAPRODUCTTYPE_SKY_MODEL="Sky Model"
DATAPRODUCTTYPE_TRANSIENT_BUFFER_BOARD_DATA="Transient Buffer Board data" DATAPRODUCTTYPE_TRANSIENT_BUFFER_BOARD_DATA="Transient Buffer Board data"
DATAPRODUCTTYPE_UNKNOWN="Unknown" DATAPRODUCTTYPE_UNKNOWN="Unknown"
EQUINOXTYPE_AZELGEO="AZELGEO"
EQUINOXTYPE_B1950="B1950" EQUINOXTYPE_B1950="B1950"
EQUINOXTYPE_J2000="J2000" EQUINOXTYPE_J2000="J2000"
EQUINOXTYPE_JUPITER="JUPITER" EQUINOXTYPE_JUPITER="JUPITER"
...@@ -47,7 +48,9 @@ EQUINOXTYPE_SUN="SUN" ...@@ -47,7 +48,9 @@ EQUINOXTYPE_SUN="SUN"
FILEFORMATTYPE_AIPS___CASA="AIPS++/CASA" FILEFORMATTYPE_AIPS___CASA="AIPS++/CASA"
FILEFORMATTYPE_FITS="FITS" FILEFORMATTYPE_FITS="FITS"
FILEFORMATTYPE_HDF5="HDF5" FILEFORMATTYPE_HDF5="HDF5"
FILEFORMATTYPE_PREFACTOR="PREFACTOR"
FILEFORMATTYPE_PULP="PULP" FILEFORMATTYPE_PULP="PULP"
FILEFORMATTYPE_UNDOCUMENTED="UNDOCUMENTED"
FILTERSELECTIONTYPE_10_70_MHZ="10-70 MHz" FILTERSELECTIONTYPE_10_70_MHZ="10-70 MHz"
FILTERSELECTIONTYPE_10_90_MHZ="10-90 MHz" FILTERSELECTIONTYPE_10_90_MHZ="10-90 MHz"
FILTERSELECTIONTYPE_110_190_MHZ="110-190 MHz" FILTERSELECTIONTYPE_110_190_MHZ="110-190 MHz"
...@@ -146,6 +149,11 @@ STATIONSELECTIONTYPE_SINGLE="Single" ...@@ -146,6 +149,11 @@ STATIONSELECTIONTYPE_SINGLE="Single"
STATIONTYPETYPE_CORE="Core" STATIONTYPETYPE_CORE="Core"
STATIONTYPETYPE_INTERNATIONAL="International" STATIONTYPETYPE_INTERNATIONAL="International"
STATIONTYPETYPE_REMOTE="Remote" STATIONTYPETYPE_REMOTE="Remote"
STORAGEWRITERTYPE_CASASTORAGEMANAGERS="CasaStorageManagers"
STORAGEWRITERTYPE_DYSCOSTORAGEMANAGER="DyscoStorageManager"
STORAGEWRITERTYPE_HDF5DEFAULT="HDF5Default"
STORAGEWRITERTYPE_LOFARSTORAGEMANAGER="LofarStorageManager"
STORAGEWRITERTYPE_UNKNOWN="Unknown"
TELESCOPE_LOFAR="LOFAR" TELESCOPE_LOFAR="LOFAR"
TIMESYSTEMTYPE_LST="LST" TIMESYSTEMTYPE_LST="LST"
TIMESYSTEMTYPE_UTC="UTC" TIMESYSTEMTYPE_UTC="UTC"
......
lib/constants_generator.py
+ 4
− 2
View file @ 4b3f7ee2
...@@ -4,10 +4,12 @@ ...@@ -4,10 +4,12 @@
# enumerations in the XSD). These are dynamically retrieved from the pyxb-generated API module and in most cases can # enumerations in the XSD). These are dynamically retrieved from the pyxb-generated API module and in most cases can
# just be rerun after pyxb to update the constants module after something has changed in the XSD schema definition. # just be rerun after pyxb to update the constants module after something has changed in the XSD schema definition.
from . import ltasip import ltasip
import inspect import inspect
import pyxb import pyxb
#from collections import namedtuple #from collections import namedtuple
import logging
logger = logging.getLogger(__name__)
VERSION = "SIPlib Constants Generator 0.1" VERSION = "SIPlib Constants Generator 0.1"
...@@ -75,7 +77,7 @@ def main(path): ...@@ -75,7 +77,7 @@ def main(path):
else: else:
value = str(value) value = str(value)
line = key+"="+value+"\n" line = key+"="+value+"\n"
print(line, end=' ') logger.info(line, end=' ')
f.write(line) f.write(line)
......
lib/feedback.py
+ 48
− 15
View file @ 4b3f7ee2
#!/usr/bin/env python3 #!/usr/bin/env python3
# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id: $
#
# !!! This is a stub and needs significant work to be usable in practice !!!
import sys import sys
import pprint import pprint
from . import siplib from . import siplib
...@@ -8,14 +30,22 @@ from ast import literal_eval ...@@ -8,14 +30,22 @@ from ast import literal_eval
import datetime import datetime
import copy import copy
import uuid import uuid
import logging
logger = logging.getLogger(__name__)
# Unfortunately storageWriter choices in XSD and Feedback differ...
storage_writer_mapping = {"CASA": constants.STORAGEWRITERTYPE_CASASTORAGEMANAGERS,
"DYSCO": constants.STORAGEWRITERTYPE_DYSCOSTORAGEMANAGER,
"HDF5DEFAULT": constants.STORAGEWRITERTYPE_HDF5DEFAULT,
"LOFAR": constants.STORAGEWRITERTYPE_LOFARSTORAGEMANAGER,
"UNKNOWN": constants.STORAGEWRITERTYPE_UNKNOWN}
class Feedback(): class Feedback():
def __init__(self, feedback): def __init__(self, feedback):
self.__inputstrings = feedback self.__inputstrings = feedback
self.__tree = {} self.__tree = {}
print("parsing",len(feedback),"lines of feedback") logger.info("parsing",len(feedback),"lines of feedback")
for line in feedback: for line in feedback:
if line.strip() and not line.startswith("#"): if line.strip() and not line.startswith("#"):
try: try:
...@@ -30,7 +60,7 @@ class Feedback(): ...@@ -30,7 +60,7 @@ class Feedback():
except: except:
t[key.split('.')[-1]] = value.strip() t[key.split('.')[-1]] = value.strip()
except: except:
print("Skipping line:", line) logger.info("Skipping line:", line)
# Now self.__tree holds nested dicts according to the dot-encoded key hierarchy # Now self.__tree holds nested dicts according to the dot-encoded key hierarchy
#pprint.pprint(self.__tree) #pprint.pprint(self.__tree)
...@@ -73,13 +103,11 @@ class Feedback(): ...@@ -73,13 +103,11 @@ class Feedback():
prefixes = prefix.split(".") prefixes = prefix.split(".")
dps = self.__get_tree_elem(prefix) dps = self.__get_tree_elem(prefix)
#print dps.items()
dataproducts = [] dataproducts = []
dps = [(k, dp) for (k, dp) in list(dps.items()) if k.startswith("Output_")] dps = [(k, dp) for (k, dp) in list(dps.items()) if k.startswith("Output_")]
for k, dp in dps: for k, dp in dps:
print("Parsing",k,"...") logger.info("Parsing",k,"...")
# correct timestamp format # correct timestamp format
startt=dp.get("startTime") startt=dp.get("startTime")
...@@ -95,6 +123,8 @@ class Feedback(): ...@@ -95,6 +123,8 @@ class Feedback():
size=dp.get("size"), size=dp.get("size"),
filename=dp.get("filename"), filename=dp.get("filename"),
fileformat=dp.get("fileFormat"), fileformat=dp.get("fileFormat"),
storage_writer=storage_writer_mapping[dp.get("storageWriter")],
storage_writer_version=dp.get("storageWriterVersion"),
process_identifier=process_identifier, process_identifier=process_identifier,
), ),
subarraypointing_identifier=subarraypointing_identifier, subarraypointing_identifier=subarraypointing_identifier,
...@@ -114,17 +144,19 @@ class Feedback(): ...@@ -114,17 +144,19 @@ class Feedback():
elif k.startswith("Output_Beamformed_["): elif k.startswith("Output_Beamformed_["):
beamlist=None beamlist=None
dataproduct = siplib.BeamFormedDataProduct( dataproducts.append(siplib.BeamFormedDataProduct(
siplib.DataProductMap( siplib.DataProductMap(
type="Correlator data", type="Correlator data",
identifier=siplib.Identifier('identifier_source'), identifier=siplib.Identifier('identifier_source'),
size=dp.get("size"), size=dp.get("size"),
filename=dp.get("filename"), filename=dp.get("filename"),
fileformat=dp.get("fileFormat"), fileformat=dp.get("fileFormat"),
storage_writer=dp.get("storageWriter"),
storage_writer_version=dp.get("storageWriterVersion"),
process_identifier=process_identifier process_identifier=process_identifier
), ),
beams=beamlist beams=beamlist
) ))
# todo other dataproduct types (if helpful, this is kind of prefactor specific for now) # todo other dataproduct types (if helpful, this is kind of prefactor specific for now)
return dataproducts return dataproducts
...@@ -137,7 +169,7 @@ class Feedback(): ...@@ -137,7 +169,7 @@ class Feedback():
if elem.get(prefix): if elem.get(prefix):
elem = elem.get(prefix) elem = elem.get(prefix)
else: else:
print("provided prefix seems to be wrong: '"+prefix+"' not in", list(elem.keys())) logger.info("provided prefix seems to be wrong: '"+prefix+"' not in", list(elem.keys()))
return elem return elem
...@@ -151,7 +183,7 @@ class Feedback(): ...@@ -151,7 +183,7 @@ class Feedback():
# todo: After evaluation, if still applicable, check assumptions made for missing attributes, assign new IDs, etc. # todo: After evaluation, if still applicable, check assumptions made for missing attributes, assign new IDs, etc.
def get_dataproduct_sips(self, obs_prefix="ObsSW.Observation", dp_prefix="ObsSW.Observation.DataProducts"): def get_dataproduct_sips(self, obs_prefix="ObsSW.Observation", dp_prefix="ObsSW.Observation.DataProducts"):
print("Generating SIPs for all dataproducts") logger.info("Generating SIPs for all dataproducts")
obs = self.__get_tree_elem(obs_prefix) obs = self.__get_tree_elem(obs_prefix)
dps = self.__get_tree_elem(dp_prefix) dps = self.__get_tree_elem(dp_prefix)
...@@ -271,9 +303,10 @@ class Feedback(): ...@@ -271,9 +303,10 @@ class Feedback():
# create sip for each dataproduct # create sip for each dataproduct
sips = {} sips = {}
for dataproduct in self.get_dataproducts(prefix=dp_prefix): for dataproduct in self.get_dataproducts(prefix=dp_prefix):
filename = None
try: try:
filename = dataproduct.get_pyxb_dataproduct().fileName filename = dataproduct.get_pyxb_dataproduct().fileName
print("Creating SIP for", filename) logger.info("Creating SIP for", filename)
# create SIP document for dataproduct # create SIP document for dataproduct
sip = self.__get_basic_sip(dataproduct) sip = self.__get_basic_sip(dataproduct)
...@@ -322,7 +355,7 @@ class Feedback(): ...@@ -322,7 +355,7 @@ class Feedback():
except Exception as err: except Exception as err:
if not filename: if not filename:
filename = "UNDEFINED" filename = "UNDEFINED"
print("Could not create SIP for", filename,"->",err) logger.info("Could not create SIP for", filename,"->",err)
if sips: if sips:
return sips return sips
...@@ -331,7 +364,7 @@ class Feedback(): ...@@ -331,7 +364,7 @@ class Feedback():
def example(fil): def example(fil):
print("Now running example on file", fil) logger.info("Now running example on file", fil)
with open(fil) as f: with open(fil) as f:
text = f.readlines() text = f.readlines()
...@@ -340,7 +373,7 @@ def example(fil): ...@@ -340,7 +373,7 @@ def example(fil):
# A) Parse complete SIP: # A) Parse complete SIP:
sips = feedback.get_dataproduct_sips(obs_prefix="ObsSW.Observation", dp_prefix="Observation.DataProducts") sips = feedback.get_dataproduct_sips(obs_prefix="ObsSW.Observation", dp_prefix="Observation.DataProducts")
for key in list(sips.keys()): for key in list(sips.keys()):
print("Created SIP for file "+ str(key)) logger.info("Created SIP for file "+ str(key))
# B) Alternatively: Parse dataproducts from pseudo-feedback (specialty of Leiden group): # B) Alternatively: Parse dataproducts from pseudo-feedback (specialty of Leiden group):
...@@ -378,8 +411,8 @@ def example(fil): ...@@ -378,8 +411,8 @@ def example(fil):
def main(argv): def main(argv):
print("! This is a stub, the feedback to SIP conversion is not correctly working at this point.") logger.warning("! This is a stub, the feedback to SIP conversion is not correctly working at this point.")
print("! You may use this as a module to do some feedback parsing, but unfortunately not all information can be determined from feedback to create a valid SIP.") logger.warning("! You may use this as a module to do some feedback parsing, but unfortunately not all information can be determined from feedback to create a valid SIP.")
if argv[1] is not None: if argv[1] is not None:
example(argv[1]) example(argv[1])
......
lib/ltasip.py
+ 1516
− 1241
View file @ 4b3f7ee2
This diff is collapsed.
lib/query.py
+ 3
− 1
View file @ 4b3f7ee2
...@@ -8,6 +8,8 @@ import xml.etree.ElementTree as ET ...@@ -8,6 +8,8 @@ import xml.etree.ElementTree as ET
import xmlrpc.client import xmlrpc.client
import uuid import uuid
import copy import copy
import logging
logger = logging.getLogger(__name__)
path = expanduser("~/.siplibrc") path = expanduser("~/.siplibrc")
user = None user = None
...@@ -23,7 +25,7 @@ if not exists(path): ...@@ -23,7 +25,7 @@ if not exists(path):
file.write("host=\n") file.write("host=\n")
with open(path,'r') as file: with open(path,'r') as file:
print("Parsing user credentials from",path) logger.info("Parsing user credentials from",path)
for line in file: for line in file:
if line.startswith("user"): if line.startswith("user"):
user = line.split('=')[1].strip() user = line.split('=')[1].strip()
......
lib/siplib.py
+ 24
− 21
View file @ 4b3f7ee2
...@@ -34,7 +34,8 @@ import xml.dom.minidom ...@@ -34,7 +34,8 @@ import xml.dom.minidom
from pyxb.namespace import XMLSchema_instance as xsi from pyxb.namespace import XMLSchema_instance as xsi
from pyxb.namespace import XMLNamespaces as xmlns from pyxb.namespace import XMLNamespaces as xmlns
from . import query from . import query
import logging
logger = logging.getLogger(__name__)
VERSION = "SIPlib 0.4" VERSION = "SIPlib 0.4"
d = os.path.dirname(os.path.realpath(__file__)) d = os.path.dirname(os.path.realpath(__file__))
...@@ -51,14 +52,15 @@ ltasip.Namespace.setPrefix('sip') ...@@ -51,14 +52,15 @@ ltasip.Namespace.setPrefix('sip')
def print_user_warning(): def print_user_warning():
print("!!! You are accessing an object, which is based on code that was auto-generated with the pyxb package.") logger.warning('''
print("!!! We strongly advise you to only use the datatypes of the siplib wrapper to create your SIP file.") !!! You are accessing an object, which is based on code that was auto-generated with the pyxb package.
print("!!! If you choose to alter pyxb/ltasip objects or their values directly for some reason, your SIP may ") !!! We strongly advise you to only use the datatypes of the siplib wrapper to create your SIP file.
print("!!! become invalid. - Please make sure to validate your SIP before submission! ") !!! If you choose to alter pyxb/ltasip objects or their values directly for some reason, your SIP may
print("!!! Note that the pyxb portion of the code is subject to change whitout being backwards compatible.") !!! become invalid. - Please make sure to validate your SIP before submission!
print("!!! This means that, should you choose to access pyxb objects e.g. to parse an existing SIP file, things") !!! Note that the pyxb portion of the code is subject to change whitout being backwards compatible.
print("!!! might break for you without further warning.") !!! This means that, should you choose to access pyxb objects e.g. to parse an existing SIP file, things
print("!!! (You may suppress this warning by setting the flag in the pyxb-related getter/setter functions.)") !!! might break for you without further warning.
!!! You may suppress this warning by setting the flag in the pyxb-related getter/setter functions.''')
# =============================== # ===============================
# Identifier definition (used for LTA entities, i-e- processes and dataproducts): # Identifier definition (used for LTA entities, i-e- processes and dataproducts):
...@@ -111,6 +113,7 @@ class Identifier(object): ...@@ -111,6 +113,7 @@ class Identifier(object):
print_user_warning() print_user_warning()
return self.__pyxb_identifier return self.__pyxb_identifier
# =============================== # ===============================
# Station definitions: # Station definitions:
...@@ -433,6 +436,8 @@ class DataProductMap(): ...@@ -433,6 +436,8 @@ class DataProductMap():
size, size,
filename, filename,
fileformat, fileformat,
storage_writer,
storage_writer_version,
process_identifier, process_identifier,
checksum_md5=None, checksum_md5=None,
checksum_adler32=None, checksum_adler32=None,
...@@ -441,7 +446,10 @@ class DataProductMap(): ...@@ -441,7 +446,10 @@ class DataProductMap():
self.dataproduct_map= dict(dataProductType=type, self.dataproduct_map= dict(dataProductType=type,
dataProductIdentifier=identifier._get_pyxb_identifier(suppress_warning=True), dataProductIdentifier=identifier._get_pyxb_identifier(suppress_warning=True),
size=size, size=size,
fileName=filename, fileFormat=fileformat, fileName=filename,
fileFormat=fileformat,
storageWriter=storage_writer,
storageWriterVersion=storage_writer_version,
processIdentifier=process_identifier._get_pyxb_identifier(suppress_warning=True), processIdentifier=process_identifier._get_pyxb_identifier(suppress_warning=True),
storageTicket=storageticket) storageTicket=storageticket)
...@@ -672,7 +680,7 @@ class SpectralCoordinate(): ...@@ -672,7 +680,7 @@ class SpectralCoordinate():
elif isinstance(axis, TabularAxis): elif isinstance(axis, TabularAxis):
args.update(dict(spectralTabularAxis=axis._get_pyxb_axis(suppress_warning=True))) args.update(dict(spectralTabularAxis=axis._get_pyxb_axis(suppress_warning=True)))
else: else:
print("wrong axis type:",type(axis)) logger.error("wrong axis type:",type(axis))
self.__pyxb_coordinate=ltasip.SpectralCoordinate(**args) self.__pyxb_coordinate=ltasip.SpectralCoordinate(**args)
...@@ -695,7 +703,7 @@ class TimeCoordinate(): ...@@ -695,7 +703,7 @@ class TimeCoordinate():
elif isinstance(axis, TabularAxis): elif isinstance(axis, TabularAxis):
args.update(dict(timeTabularAxis=axis._get_pyxb_axis(suppress_warning=True))) args.update(dict(timeTabularAxis=axis._get_pyxb_axis(suppress_warning=True)))
else: else:
print("wrong axis type:",type(axis)) logger.error("wrong axis type:",type(axis))
self.__pyxb_coordinate=ltasip.TimeCoordinate(**args) self.__pyxb_coordinate=ltasip.TimeCoordinate(**args)
...@@ -1311,9 +1319,7 @@ class Sip(object): ...@@ -1311,9 +1319,7 @@ class Sip(object):
The main Sip object. Instantiate this with the dataproduct you want to describe/ingest. The main Sip object. Instantiate this with the dataproduct you want to describe/ingest.
Then add all related items to it, like observation, pipeline runs, and intermediate dataproducts. """ Then add all related items to it, like observation, pipeline runs, and intermediate dataproducts. """
print("\n################") logger.info("SIP library v.%s" %VERSION)
print(VERSION)
print("################\n")
#----------- #-----------
# Base document # Base document
...@@ -1423,7 +1429,7 @@ class Sip(object): ...@@ -1423,7 +1429,7 @@ class Sip(object):
if not any(x.dataProductIdentifier.identifier == relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier for x in self.__sip.relatedDataProduct): if not any(x.dataProductIdentifier.identifier == relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier for x in self.__sip.relatedDataProduct):
self.__sip.relatedDataProduct.append(relateddataproduct_sip.__sip.dataProduct) self.__sip.relatedDataProduct.append(relateddataproduct_sip.__sip.dataProduct)
else: else:
print("WARNING: There already exists a dataproduct with id", relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier," - Will try to add any new related items anyway.") logger.warning("There already exists a dataproduct with id", relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier," - Will try to add any new related items anyway.")
if relateddataproduct_sip.__sip.relatedDataProduct: if relateddataproduct_sip.__sip.relatedDataProduct:
# add related dataproducts (if not there already) # add related dataproducts (if not there already)
for dp in relateddataproduct_sip.__sip.relatedDataProduct: for dp in relateddataproduct_sip.__sip.relatedDataProduct:
...@@ -1473,18 +1479,15 @@ class Sip(object): ...@@ -1473,18 +1479,15 @@ class Sip(object):
def get_prettyxml(self): def get_prettyxml(self):
try: try:
dom = self.__sip.toDOM() dom = self.__sip.toDOM()
#print "1) "+dom.toprettyxml()
dom.documentElement.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") dom.documentElement.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
dom.documentElement.setAttribute('xsi:schemaLocation', "http://www.astron.nl/SIP-Lofar LTA-SIP-2.7.0.xsd") dom.documentElement.setAttribute('xsi:schemaLocation', "http://www.astron.nl/SIP-Lofar LTA-SIP-2.7.0.xsd")
#print "2) "+dom.toprettyxml()
return dom.toprettyxml() return dom.toprettyxml()
except pyxb.ValidationError as err: except pyxb.ValidationError as err:
#print str(err) logger.error(err.details())
print(err.details())
raise err raise err
def prettyprint(self): def prettyprint(self):
print(self.get_prettyxml()) logger.info(self.get_prettyxml())
def save_to_file(self, path): def save_to_file(self, path):
......
lib/testmetadata_file.Correlated.modified deleted 100644 → 0
+ 0
− 31
View file @ 56fc4bd8
test.prefix.Output_Correlated_[0].centralFrequency=119042968.75
test.prefix.Output_Correlated_[0].channelWidth=48828.125
test.prefix.Output_Correlated_[0].channelsPerSubband=8
test.prefix.Output_Correlated_[0].duration=3600.99872684
test.prefix.Output_Correlated_[0].fileFormat=AIPS++/CASA
test.prefix.Output_Correlated_[0].filename=L74759_SAP000_SB001_uv.copySubtracted
test.prefix.Output_Correlated_[0].integrationInterval=4.00556032
test.prefix.Output_Correlated_[0].job_duration=5.0088031292
test.prefix.Output_Correlated_[0].location=jrc0636:/work/htb00/htb003/cookbook_working_dir/Cookbook-Tutorial
test.prefix.Output_Correlated_[0].percentageWritten=100
test.prefix.Output_Correlated_[0].returncode=0
test.prefix.Output_Correlated_[0].size=1381625751
test.prefix.Output_Correlated_[0].startTime=2012-11-12T12:47:00.000
test.prefix.Output_Correlated_[0].stationSubband=0
test.prefix.Output_Correlated_[0].subband=0
test.prefix.Output_Correlated_[1].centralFrequency=119042968.75
test.prefix.Output_Correlated_[1].channelWidth=48828.125
test.prefix.Output_Correlated_[1].channelsPerSubband=8
test.prefix.Output_Correlated_[1].duration=3600.99872684
test.prefix.Output_Correlated_[1].fileFormat=AIPS++/CASA
test.prefix.Output_Correlated_[1].filename=L74759_SAP000_SB001_uv.copySubtracted.2
test.prefix.Output_Correlated_[1].integrationInterval=4.00556032
test.prefix.Output_Correlated_[1].job_duration=5.0088031292
test.prefix.Output_Correlated_[1].location=jrc0636:/work/htb00/htb003/cookbook_working_dir/Cookbook-Tutorial
test.prefix.Output_Correlated_[1].percentageWritten=100
test.prefix.Output_Correlated_[1].returncode=1
test.prefix.Output_Correlated_[1].size=1381625751
test.prefix.Output_Correlated_[1].startTime=2012-11-12T12:47:00.000
test.prefix.Output_Correlated_[1].stationSubband=1
test.prefix.Output_Correlated_[1].subband=1
test.prefix.nrOfOutput_Correlated_=2
\ No newline at end of file
lib/validator.py
+ 8
− 7
View file @ 4b3f7ee2
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
from lxml import etree from lxml import etree
import os import os
from . import ltasip from . import ltasip
import logging
logger = logging.getLogger(__name__)
d = os.path.dirname(os.path.realpath(__file__)) d = os.path.dirname(os.path.realpath(__file__))
XSDPATH = d+"/LTA-SIP.xsd" XSDPATH = d+"/LTA-SIP.xsd"
...@@ -11,7 +13,7 @@ DEFAULT_SIP_XSD_PATH = os.path.join(os.environ.get('LOFARROOT', '/opt/lofar'), ' ...@@ -11,7 +13,7 @@ DEFAULT_SIP_XSD_PATH = os.path.join(os.environ.get('LOFARROOT', '/opt/lofar'), '
def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH): def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH):
'''validates given xml file against given xsd file''' '''validates given xml file against given xsd file'''
print("validating", xmlpath, "against", xsdpath) logger.info("validating", xmlpath, "against", xsdpath)
with open(xsdpath) as xsd: with open(xsdpath) as xsd:
xmlschema_doc = etree.parse(xsd) xmlschema_doc = etree.parse(xsd)
...@@ -25,9 +27,9 @@ def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH): ...@@ -25,9 +27,9 @@ def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH):
try: try:
xmlschema.assertValid(doc) xmlschema.assertValid(doc)
except Exception as err: except Exception as err:
print(err) logger.error(err)
print("SIP is valid according to schema definition!") logger.info("SIP is valid according to schema definition!")
return valid return valid
...@@ -38,7 +40,7 @@ def check_consistency(xmlpath): ...@@ -38,7 +40,7 @@ def check_consistency(xmlpath):
Are the input dataproducts for these processes present? Are the input dataproducts for these processes present?
""" """
print("Checking", xmlpath, "for structural consistency") logger.info("Checking", xmlpath, "for structural consistency")
with open(xmlpath) as f: with open(xmlpath) as f:
xml = f.read() xml = f.read()
...@@ -94,7 +96,7 @@ def check_consistency(xmlpath): ...@@ -94,7 +96,7 @@ def check_consistency(xmlpath):
if not id_from in linkstodataproduct: if not id_from in linkstodataproduct:
raise Exception("The input dataproduct for pipeline '"+ id +"' seems to be missing! -> ", id_from) raise Exception("The input dataproduct for pipeline '"+ id +"' seems to be missing! -> ", id_from)
print("General SIP structure seems ok!") logger.info("General SIP structure seems ok!")
return True # already raised Exception if there was a problem... return True # already raised Exception if there was a problem...
...@@ -110,5 +112,4 @@ def main(xml): ...@@ -110,5 +112,4 @@ def main(xml):
consistent = check_consistency(xml) consistent = check_consistency(xml)
return valid and consistent return valid and consistent
except Exception as err: except Exception as err:
print("An error occurred:") logger.error(err)
print(err)
lib/visualizer.py
+ 9
− 10
View file @ 4b3f7ee2
...@@ -5,6 +5,8 @@ import sys ...@@ -5,6 +5,8 @@ import sys
from . import siplib from . import siplib
from . import ltasip from . import ltasip
import tempfile import tempfile
import logging
logger = logging.getLogger(__name__)
ltasip.Namespace.setPrefix('sip') ltasip.Namespace.setPrefix('sip')
...@@ -45,7 +47,7 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -45,7 +47,7 @@ def visualize_sip(sip, path=None, format="svg", view=False):
data_out = sip.dataProduct data_out = sip.dataProduct
id_out = str(data_out.dataProductIdentifier.identifier) id_out = str(data_out.dataProductIdentifier.identifier)
dot.node(id_out, id_out +": "+data_out.fileName,style="filled",fillcolor="cadetblue", shape="note") dot.node(id_out, id_out +": "+data_out.fileName,style="filled",fillcolor="cadetblue", shape="note")
print("adding node for final dataproduct ", id_out) logger.info("adding node for final dataproduct ", id_out)
id_process = str(data_out.processIdentifier.identifier) id_process = str(data_out.processIdentifier.identifier)
# keep reference to originating pipeline run / observation: # keep reference to originating pipeline run / observation:
linkstodataproduct.setdefault(id_out,[]).append(id_process) linkstodataproduct.setdefault(id_out,[]).append(id_process)
...@@ -54,7 +56,7 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -54,7 +56,7 @@ def visualize_sip(sip, path=None, format="svg", view=False):
for data_in in sip.relatedDataProduct: for data_in in sip.relatedDataProduct:
id_in = str(data_in.dataProductIdentifier.identifier) id_in = str(data_in.dataProductIdentifier.identifier)
dot.node(id_in, id_in +": "+data_in.fileName, style="filled", shape="note",fillcolor="cadetblue2") dot.node(id_in, id_in +": "+data_in.fileName, style="filled", shape="note",fillcolor="cadetblue2")
print("adding node for dataproduct ", id_in) logger.info("adding node for dataproduct ", id_in)
id_process = str(data_in.processIdentifier.identifier) id_process = str(data_in.processIdentifier.identifier)
# keep reference to originating pipeline run / observation: # keep reference to originating pipeline run / observation:
linkstodataproduct.setdefault(id_in,[]).append(id_process) linkstodataproduct.setdefault(id_in,[]).append(id_process)
...@@ -64,7 +66,7 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -64,7 +66,7 @@ def visualize_sip(sip, path=None, format="svg", view=False):
id_obs = str(obs.observationId.identifier) id_obs = str(obs.observationId.identifier)
id_process = str(obs.processIdentifier.identifier) id_process = str(obs.processIdentifier.identifier)
dot.node(id_process, id_process + ": "+ id_obs, style="filled", fillcolor="gold",shape="octagon") dot.node(id_process, id_process + ": "+ id_obs, style="filled", fillcolor="gold",shape="octagon")
print("adding node for observation ", id_process) logger.info("adding node for observation ", id_process)
# no incoming data here, but register node as present: # no incoming data here, but register node as present:
linkstoprocess.setdefault(id_process,[]) linkstoprocess.setdefault(id_process,[])
...@@ -72,7 +74,7 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -72,7 +74,7 @@ def visualize_sip(sip, path=None, format="svg", view=False):
for pipe in sip.pipelineRun: for pipe in sip.pipelineRun:
id_pipe = str(pipe.processIdentifier.identifier) id_pipe = str(pipe.processIdentifier.identifier)
dot.node(id_pipe, id_pipe+" ", style="filled", fillcolor="chartreuse", shape="cds") dot.node(id_pipe, id_pipe+" ", style="filled", fillcolor="chartreuse", shape="cds")
print("adding node for pipelinerun ", id_pipe) logger.info("adding node for pipelinerun ", id_pipe)
# keep reference to input dataproducts: # keep reference to input dataproducts:
id_in = [] id_in = []
for elem in pipe.sourceData.orderedContent(): for elem in pipe.sourceData.orderedContent():
...@@ -83,7 +85,7 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -83,7 +85,7 @@ def visualize_sip(sip, path=None, format="svg", view=False):
for unspec in sip.unspecifiedProcess: for unspec in sip.unspecifiedProcess:
id_unspec = str(unspec.processIdentifier.identifier) id_unspec = str(unspec.processIdentifier.identifier)
dot.node(id_unspec, id_unspec, style="filled", fillcolor="orange", shape="hexagon") dot.node(id_unspec, id_unspec, style="filled", fillcolor="orange", shape="hexagon")
print("adding node for unspecified process ", id_unspec) logger.info("adding node for unspecified process ", id_unspec)
# no incoming data here, but register node as present: # no incoming data here, but register node as present:
linkstoprocess.setdefault(id_unspec,[]) linkstoprocess.setdefault(id_unspec,[])
...@@ -91,9 +93,6 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -91,9 +93,6 @@ def visualize_sip(sip, path=None, format="svg", view=False):
# todo: online processing # todo: online processing
# todo: parsets (?) # todo: parsets (?)
# print linkstoprocess
# print linkstodataproduct
# add edges: # add edges:
for id in linkstodataproduct: for id in linkstodataproduct:
for id_from in linkstodataproduct.get(id): for id_from in linkstodataproduct.get(id):
...@@ -119,7 +118,7 @@ def visualize_sip(sip, path=None, format="svg", view=False): ...@@ -119,7 +118,7 @@ def visualize_sip(sip, path=None, format="svg", view=False):
dot_wrapper.format = format dot_wrapper.format = format
dot_wrapper.engine = 'dot' dot_wrapper.engine = 'dot'
if path is not None: if path is not None:
print("writing rendering to", path) logger.info("writing rendering to", path)
dot_wrapper.render(path, view=view) dot_wrapper.render(path, view=view)
else: else:
with tempfile.NamedTemporaryFile(mode='wt') as temp: with tempfile.NamedTemporaryFile(mode='wt') as temp:
...@@ -166,7 +165,7 @@ def stylize(graph): ...@@ -166,7 +165,7 @@ def stylize(graph):
def main(xmlpath): def main(xmlpath):
print("Reading xml from file", xmlpath) logger.info("Reading xml from file", xmlpath)
with open(xmlpath) as f: with open(xmlpath) as f:
xml = f.read() xml = f.read()
sip = ltasip.CreateFromDocument(xml) sip = ltasip.CreateFromDocument(xml)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment