Skip to content
Snippets Groups Projects
Commit fccef0c0 authored by Jörn Künsemöller's avatar Jörn Künsemöller
Browse files

Task #9091 - separated test for validator, some cleanup

parent 80a018c8
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
lib/feedback.py
+ 34
− 28
View file @ fccef0c0
......@@ -60,8 +60,8 @@ class Feedback():
# return dataproducts objects from 'pseudo feedback', which can be generated from the MS's by some existing code.
def get_dataproducts(self,
prefix="ObsSW.Observation",
process_identifier=siplib.Identifier('???',uuid.uuid1().int>>64),
subarraypointing_identifier=siplib.Identifier('???',uuid.uuid1().int>>64),
process_identifier=siplib.Identifier('mysource'),
subarraypointing_identifier=siplib.Identifier('mysource'),
):
prefixes = prefix.split(".")
dps = self.__get_tree_elem(prefix)
......@@ -84,7 +84,7 @@ class Feedback():
siplib.CorrelatedDataProduct(
siplib.DataProductMap(
type="Correlator data",
identifier=siplib.Identifier('???',uuid.uuid1().int>>64),
identifier=siplib.Identifier('mysource'),
size=dp.get("size"),
filename=dp.get("filename"),
fileformat=dp.get("fileFormat"),
......@@ -110,7 +110,7 @@ class Feedback():
dataproduct = siplib.BeamFormedDataProduct(
siplib.DataProductMap(
type="Correlator data",
identifier=siplib.Identifier('???',uuid.uuid1().int>>64),
identifier=siplib.Identifier('myproject'),
size=dp.get("size"),
filename=dp.get("filename"),
fileformat=dp.get("fileFormat"),
......@@ -255,7 +255,7 @@ class Feedback():
numberofcorrelateddataproducts=2, # todo
numberofbeamformeddataproducts=1, # todo
relations=[siplib.ProcessRelation(
identifier=siplib.Identifier('???','???') # todo
identifier=siplib.Identifier('myproject') # todo
)],
#todo: optional kwargs
)
......@@ -295,12 +295,12 @@ class Feedback():
strategydescription="awesome strategy", # todo
starttime=obs.get("startTime").replace(" ","T"),
duration= duration,
identifier=siplib.Identifier(source="SAS", id="SAS VIC Tree Id"), # todo: Not possible to obtain this?
process_identifier=siplib.Identifier(source="MoM",id="momid"), #obs.get(some_beam).get("momID"), # todo: Not possible to obtain this?
identifier=siplib.Identifier(source="myproject"), # todo: Not possible to obtain the ID that this has in the catalog based on the feedback?
process_identifier=siplib.Identifier(source="myproject"), #obs.get(some_beam).get("momID"), # todo: Not possible to obtain the ID that this has in the catalog based on the feedback?
#parset_source="parsource", # todo
#parset_id="parid", #todo,
relations=[siplib.ProcessRelation(
identifier=siplib.Identifier(source="groupid source", id='groupid') #todo: Not possible to obtain this?
identifier=siplib.Identifier(source="myproject") #todo: Not possible to obtain this?
)],
),
observationdescription=campaign.get("title"), #todo
......@@ -323,36 +323,32 @@ class Feedback():
return None
def example(fil):
print "Now running example on file", fil
def main(argv):
#do the proper calls to the feedback/SIP API
# parse cmdline args etc
with open(argv[1]) as f:
print argv[1]
with open(fil) as f:
text = f.readlines()
feedback = Feedback(text)
# Parse complete SIP:
#sips = feedback.get_dataproduct_sips(obs_prefix="ObsSW.Observation", dp_prefix="Observation.DataProducts")
#for sip in sips.values():
# print sip.sip.dataProduct.fileName
# A) Parse complete SIP:
sips = feedback.get_dataproduct_sips(obs_prefix="ObsSW.Observation", dp_prefix="Observation.DataProducts")
for key in sips.keys():
print "Created SIP for file "+ str(key)
# Alternatively: Parse dataproducts from pseudo-feedback:
beamsrc = 'MoM'
beamid = '400869'
# B) Alternatively: Parse dataproducts from pseudo-feedback (specialty of Leiden group):
identifier = siplib.Identifier(id='test', source='constructor')
dataproducts = feedback.get_dataproducts(process_identifier=identifier,subarraypointing_identifier=identifier, prefix="test.prefix" )
process_identifier = siplib.Identifier(source='myproject') # create process identifier
sapointing_identifier = siplib.Identifier(source='myproject') # create subarra ypointing identifier
procids = {"keyatt":"pipe_"+str(uuid.uuid4())}
# either provide identifiers as args...
dataproducts = feedback.get_dataproducts(process_identifier=process_identifier, subarraypointing_identifier=sapointing_identifier, prefix="test.prefix" )
# ...or set them explicitely
for dp in dataproducts:
procid = procids.get("keyatt") # somehow identify correct process id for this dp
dp.set_identifier(siplib.Identifier('Leiden',"data_"+str(uuid.uuid4())))
dp.set_process_identifier(siplib.Identifier('Leiden',procid))
dp.set_subarraypointing_identifier(siplib.Identifier('test',"42"))
dp.set_identifier(siplib.Identifier('myproject')) # create new unique ID for dataproduct
dp.set_process_identifier(process_identifier) # set the identifier of the creating process
dp.set_subarraypointing_identifier(sapointing_identifier) # assign the pointing identifier
# Create example doc:
sip = siplib.Sip(
......@@ -372,6 +368,16 @@ def main(argv):
sip.prettyprint()
def main(argv):
print "! This is a stub, the feedback to SIP conversion is not correctly working at this point."
print "! You may use this as a module to do some feedback parsing, but unfortunately not all information can be determined from feedback to create a valid SIP."
if argv[1] is not None:
example(argv[1])
if __name__ == '__main__':
main(sys.argv)
lib/siplib.py
+ 168
− 83
View file @ fccef0c0
......@@ -64,19 +64,23 @@ def print_user_warning():
# Identifier definition (used for LTA entities, i-e- processes and dataproducts):
class Identifier():
"""Identifier for LTA entities"""
def __init__(self,
source,
id,
name=None
):
self.__pyxb_identifier=ltasip.IdentifierType(source=source, identifier=id, name=name)
@classmethod
def request_unique_identifier(cls, source, name=None):
unique_id = query.get_unique_id(source)
return cls(source, unique_id, name)
self.__pyxb_identifier=ltasip.IdentifierType(source=str(source), identifier=str(unique_id), name=str(name))
def _setpyxb_identifier(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self
def get_pyxb_identifier(self):
def _get_pyxb_identifier(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_identifier
# ===============================
......@@ -90,9 +94,9 @@ class Station():
antennafield2=None
):
__afields=[antennafield1.get_pyxb_antennafield()]
__afields=[antennafield1._get_pyxb_antennafield(suppress_warning=True)]
if antennafield2:
__afields.append(antennafield2.get_pyxb_antennafield())
__afields.append(antennafield2._get_pyxb_antennafield(suppress_warning=True))
self.__pyxb_station=ltasip.Station(
name=name,
stationType=type,
......@@ -109,7 +113,6 @@ class Station():
__afield1=None
__afield2=None
print os.path.abspath(STATION_CONFIG_PATH)
with open(STATION_CONFIG_PATH, 'r') as f:
for line in f.readlines():
if line.strip():
......@@ -145,7 +148,9 @@ class Station():
antennafield2=__afield2,
)
def get_pyxb_station(self):
def _get_pyxb_station(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_station
......@@ -168,7 +173,9 @@ class AntennafieldXYZ():
)
)
def get_pyxb_antennafield(self):
def _get_pyxb_antennafield(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_antennafield
class AntennafieldRadLonLat():
......@@ -191,7 +198,9 @@ class AntennafieldRadLonLat():
)
)
def get_pyxb_antennafield(self):
def _get_pyxb_antennafield(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_antennafield
......@@ -208,7 +217,7 @@ class PipelineMap():
__sourcedata=ltasip.DataSources()
for id in sourcedata_identifiers:
__sourcedata.append(id.get_pyxb_identifier())
__sourcedata.append(id._get_pyxb_identifier(suppress_warning=True))
self.pipeline_map=dict(
pipelineName=name,
......@@ -225,7 +234,9 @@ class SimplePipeline():
def __init__(self, pipeline_map):
self.__pyxb_pipeline=ltasip.PipelineRun(**pipeline_map.get_dict())
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
......@@ -256,7 +267,9 @@ class ImagingPipeline():
**pipeline_map.get_dict()
)
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
......@@ -281,7 +294,7 @@ class CalibrationPipeline():
**pipeline_map.get_dict()
)
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
return self.__pyxb_pipeline
class AveragingPipeline():
......@@ -302,7 +315,9 @@ class AveragingPipeline():
)
#self.__pyxb_pipeline._setAttribute("xsi:type","ns1:AveragingPipeline")
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
......@@ -338,14 +353,18 @@ class PulsarPipeline():
skipPreFold=skipprefold,
**pipeline_map.get_dict()
)
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
class CosmicRayPipeline():
def __init__(self, pipeline_map):
self.__pyxb_pipeline=ltasip.CosmicRayPipeline(**pipeline_map.get_dict())
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
class LongBaselinePipeline():
......@@ -359,14 +378,18 @@ class LongBaselinePipeline():
subbandGroupsPerMS=subbandgroupspermS,
**pipeline_map.get_dict())
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
class GenericPipeline():
def __init__(self, pipeline_map):
self.__pyxb_pipeline=ltasip.GenericPipeline(**pipeline_map.get_dict())
def get_pyxb_pipeline(self):
def _get_pyxb_pipeline(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pipeline
......@@ -388,10 +411,10 @@ class DataProductMap():
storageticket=None
):
self.dataproduct_map= dict(dataProductType=type,
dataProductIdentifier=identifier.get_pyxb_identifier(),
dataProductIdentifier=identifier._get_pyxb_identifier(suppress_warning=True),
size=size,
fileName=filename, fileFormat=fileformat,
processIdentifier=process_identifier.get_pyxb_identifier(),
processIdentifier=process_identifier._get_pyxb_identifier(suppress_warning=True),
storageTicket=storageticket)
checksums = []
......@@ -411,15 +434,17 @@ class __DataProduct(object):
self.__pyxb_dataproduct=pyxb_dataproduct
def set_identifier(self, identifier):
self.__pyxb_dataproduct.dataProductIdentifier = identifier.get_pyxb_identifier()
self.__pyxb_dataproduct.dataProductIdentifier = identifier._get_pyxb_identifier(suppress_warning=True)
def set_process_identifier(self, identifier):
self.__pyxb_dataproduct.processIdentifier = identifier.get_pyxb_identifier()
self.__pyxb_dataproduct.processIdentifier = identifier._get_pyxb_identifier(suppress_warning=True)
def set_subarraypointing_identifier(self, identifier):
self.__pyxb_dataproduct.subArrayPointingIdentifier = identifier.get_pyxb_identifier()
self.__pyxb_dataproduct.subArrayPointingIdentifier = identifier._get_pyxb_identifier(suppress_warning=True)
def get_pyxb_dataproduct(self):
def _get_pyxb_dataproduct(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_dataproduct
......@@ -458,7 +483,7 @@ class PixelMapDataProduct(__DataProduct):
ltasip.PixelMapDataProduct(
numberOfAxes=numberofaxes,
numberOfCoordinates=len(coordinates),
coordinate = [x.get_pyxb_coordinate() for x in coordinates],
coordinate = [x._get_pyxb_coordinate(suppress_warning=True) for x in coordinates],
**dataproduct_map.get_dict())
)
......@@ -482,10 +507,10 @@ class SkyImageDataProduct(__DataProduct):
ltasip.SkyImageDataProduct(
numberOfAxes=numberofaxes,
numberOfCoordinates=len(coordinates),
coordinate = [x.get_pyxb_coordinate() for x in coordinates],
coordinate = [x._get_pyxb_coordinate(suppress_warning=True) for x in coordinates],
locationFrame=locationframe,
timeFrame=timeframe,
observationPointing=observationpointing.get_pyxb_pointing(),
observationPointing=observationpointing._get_pyxb_pointing(suppress_warning=True),
restoringBeamMajor=ltasip.Angle(restoringbeammajor_angle, units=restoringbeammajor_angleunit),
restoringBeamMinor=ltasip.Angle(restoringbeamminor_angle, units=restoringbeamminor_angleunit),
rmsNoise=ltasip.Pixel(rmsnoise, units="Jy/beam"),
......@@ -511,7 +536,7 @@ class CorrelatedDataProduct(__DataProduct):
stationsubband=None):
super(CorrelatedDataProduct, self).__init__(
ltasip.CorrelatedDataProduct(
subArrayPointingIdentifier=subarraypointing_identifier.get_pyxb_identifier(),
subArrayPointingIdentifier=subarraypointing_identifier._get_pyxb_identifier(suppress_warning=True),
centralFrequency=ltasip.Frequency(central_frequency, units=central_frequencyunit),
channelWidth=ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit),
subband=subband,
......@@ -573,7 +598,7 @@ class PulpDataProduct(__DataProduct):
ltasip.PulpDataProduct(
fileContent=filecontent,
dataType=datatype,
arrayBeam=arraybeam.get_pyxb_beam(),
arrayBeam=arraybeam._get_pyxb_beam(suppress_warning=True),
**dataproduct_map.get_dict())
)
......@@ -590,7 +615,7 @@ class BeamFormedDataProduct(__DataProduct):
__beams=ltasip.ArrayBeams()
__nbeams = len(beams)
for beam in beams:
__beams.append(beam.get_pyxb_beam())
__beams.append(beam._get_pyxb_beam(suppress_warning=True))
super(BeamFormedDataProduct, self).__init__(
ltasip.BeamFormedDataProduct(
numberOfBeams=__nbeams,
......@@ -614,15 +639,17 @@ class SpectralCoordinate():
args = dict(spectralQuantity=ltasip.SpectralQuantity(value_=quantity_value, type=quantity_type))
if isinstance(axis, LinearAxis):
args.update(dict(spectralLinearAxis=axis.get_pyxb_axis()))
args.update(dict(spectralLinearAxis=axis._get_pyxb_axis(suppress_warning=True)))
elif isinstance(axis, TabularAxis):
args.update(dict(spectralTabularAxis=axis.get_pyxb_axis()))
args.update(dict(spectralTabularAxis=axis._get_pyxb_axis(suppress_warning=True)))
else:
print "wrong axis type:",type(axis)
self.__pyxb_coordinate=ltasip.SpectralCoordinate(**args)
def get_pyxb_coordinate(self):
def _get_pyxb_coordinate(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_coordinate
......@@ -635,16 +662,18 @@ class TimeCoordinate():
args = dict(equinox=equinox)
if isinstance(axis, LinearAxis):
args.update(dict(timeLinearAxis=axis.get_pyxb_axis()))
args.update(dict(timeLinearAxis=axis._get_pyxb_axis(suppress_warning=True)))
elif isinstance(axis, TabularAxis):
args.update(dict(timeTabularAxis=axis.get_pyxb_axis()))
args.update(dict(timeTabularAxis=axis._get_pyxb_axis(suppress_warning=True)))
else:
print "wrong axis type:",type(axis)
self.__pyxb_coordinate=ltasip.TimeCoordinate(**args)
def get_pyxb_coordinate(self):
def _get_pyxb_coordinate(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_coordinate
......@@ -654,10 +683,12 @@ class PolarizationCoordinate():
polarizations
):
self.__pyxb_coordinate=ltasip.PolarizationCoordinate(
polarizationTabularAxis=tabularaxis.get_pyxb_axis(),
polarizationTabularAxis=tabularaxis._get_pyxb_axis(suppress_warning=True),
polarization=polarizations)
def get_pyxb_coordinate(self):
def _get_pyxb_coordinate(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_coordinate
......@@ -679,7 +710,7 @@ class DirectionCoordinate():
latitudepole_angleunit):
self.__pyxb_coordinate=ltasip.DirectionCoordinate(
directionLinearAxis=[linearaxis_a.get_pyxb_axis(), linearaxis_b.get_pyxb_axis()],
directionLinearAxis=[linearaxis_a._get_pyxb_axis(suppress_warning=True), linearaxis_b._get_pyxb_axis(suppress_warning=True)],
PC0_0=pc0_0,
PC0_1=pc0_1,
PC1_0=pc1_0,
......@@ -692,7 +723,9 @@ class DirectionCoordinate():
latitudePole=ltasip.Angle(latitudepole_angle, units=latitudepole_angleunit)
)
def get_pyxb_coordinate(self):
def _get_pyxb_coordinate(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_coordinate
# ###########
......@@ -714,7 +747,7 @@ class ArrayBeamMap():
channelspersubband,
stokes):
self.arraybeam_map=dict(
subArrayPointingIdentifier=subarraypointing_identifier.get_pyxb_identifier(),
subArrayPointingIdentifier=subarraypointing_identifier._get_pyxb_identifier(suppress_warning=True),
beamNumber=beamnumber,
dispersionMeasure=dispersionmeasure,
numberOfSubbands=numberofsubbands,
......@@ -733,7 +766,9 @@ class SimpleArrayBeam():
def __init__(self, arraybeam_map):
self.__pyxb_beam=ltasip.ArrayBeam(**arraybeam_map.get_dict())
def get_pyxb_beam(self):
def _get_pyxb_beam(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_beam
......@@ -747,7 +782,9 @@ class CoherentStokesBeam():
offset=offset,
**arraybeam_map.get_dict())
def get_pyxb_beam(self):
def _get_pyxb_beam(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_beam
......@@ -755,7 +792,9 @@ class IncoherentStokesBeam():
def __init__(self, arraybeam_map):
self.__pyxb_beam=ltasip.IncoherentStokesBeam(**arraybeam_map.get_dict())
def get_pyxb_beam(self):
def _get_pyxb_beam(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_beam
......@@ -764,10 +803,12 @@ class FlysEyeBeam():
arraybeam_map,
station):
self.__pyxb_beam=ltasip.FlysEyeBeam(
station=station.get_pyxb_station(),
station=station._get_pyxb_station(suppress_warning=True),
**arraybeam_map.get_dict())
def get_pyxb_beam(self):
def _get_pyxb_beam(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_beam
......@@ -798,7 +839,9 @@ class CorrelatorProcessing():
# Somehow this does not work in the constructor:
self.__pyxb_rtprocessing.channelwidth=__channelwidth
def get_pyxb_rtprocessing(self):
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_rtprocessing
class CoherentStokesProcessing():
......@@ -821,7 +864,7 @@ class CoherentStokesProcessing():
__stations = ltasip.Stations()
for station in stations:
__stations.append(station.get_pyxb_station())
__stations.append(station._get_pyxb_station(suppress_warning=True))
__channelwidth=None
if channelwidth_frequency and channelwidth_frequencyunit:
......@@ -844,7 +887,9 @@ class CoherentStokesProcessing():
# Somehow this does not work in the constructor:
self.__pyxb_rtprocessing.channelwidth=__channelwidth
def get_pyxb_rtprocessing(self):
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_rtprocessing
# This is identical to coherent stokes. Redundancy already in the SIP schema...
......@@ -867,7 +912,7 @@ class IncoherentStokesProcessing():
):
__stations = ltasip.Stations()
for station in stations:
__stations.append(station.get_pyxb_station())
__stations.append(station._get_pyxb_station(suppress_warning=True))
__channelwidth=None
if channelwidth_frequency and channelwidth_frequencyunit:
......@@ -889,7 +934,9 @@ class IncoherentStokesProcessing():
# Somehow this does not work in the constructor:
self.__pyxb_rtprocessing.channelwidth=__channelwidth
def get_pyxb_rtprocessing(self):
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_rtprocessing
class FlysEyeProcessing():
......@@ -924,7 +971,9 @@ class FlysEyeProcessing():
self.__pyxb_rtprocessing.channelwidth=__channelwidth
def get_pyxb_rtprocessing(self):
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_rtprocessing
class NonStandardProcessing():
......@@ -940,7 +989,9 @@ class NonStandardProcessing():
channelWidth=ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit)
)
def get_pyxb_rtprocessing(self):
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_rtprocessing
......@@ -961,15 +1012,15 @@ class ProcessMap():
__relations=ltasip.ProcessRelations()
for rel in relations:
__relations.append(rel.get_pyxb_processrelation())
self.process_map = dict(processIdentifier=identifier.get_pyxb_identifier(),
observationId=observation_identifier.get_pyxb_identifier(),
__relations.append(rel._get_pyxb_processrelation(suppress_warning=True))
self.process_map = dict(processIdentifier=identifier._get_pyxb_identifier(suppress_warning=True),
observationId=observation_identifier._get_pyxb_identifier(suppress_warning=True),
relations=__relations,
strategyName=strategyname, strategyDescription=strategydescription, startTime=starttime,
duration=duration)
if parset_identifier:
self.process_map["parset"]=parset_identifier.get_pyxb_identifier()
self.process_map["parset"]=parset_identifier._get_pyxb_identifier(suppress_warning=True)
def get_dict(self):
return self.process_map
......@@ -982,10 +1033,12 @@ class ProcessRelation():
self.__pyxb_processrelation=ltasip.ProcessRelation(
relationType=ltasip.ProcessRelationType(type),
identifier=identifier.get_pyxb_identifier()
identifier=identifier._get_pyxb_identifier(suppress_warning=True)
)
def get_pyxb_processrelation(self):
def _get_pyxb_processrelation(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_processrelation
......@@ -993,7 +1046,9 @@ class SimpleProcess():
def __init__(self, process_map):
self.pyxb_process = ltasip.Process(**process_map.get_dict())
def get_pyxb_process(self):
def _get_pyxb_process(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.pyxb_process
......@@ -1015,7 +1070,9 @@ class PointingRaDec():
declination=ltasip.Angle(dec_angle, units=dec_angleunit),
equinox=equinox)
def get_pyxb_pointing(self):
def _get_pyxb_pointing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pointing
......@@ -1033,7 +1090,9 @@ class PointingAltAz():
altitude=ltasip.Angle(alt_angle, units=alt_angleunit),
equinox=equinox)
def get_pyxb_pointing(self):
def _get_pyxb_pointing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_pointing
......@@ -1056,7 +1115,9 @@ class LinearAxis():
referencePixel=referencepixel,
referenceValue=referencevalue
)
def get_pyxb_axis(self):
def _get_pyxb_axis(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_axis
......@@ -1075,7 +1136,9 @@ class TabularAxis():
length=length,
)
def get_pyxb_axis(self):
def _get_pyxb_axis(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_axis
......@@ -1101,21 +1164,21 @@ class SubArrayPointing():
__relations=ltasip.ProcessRelations()
for rel in relations:
__relations.append(rel.get_pyxb_processrelation())
__relations.append(rel._get_pyxb_processrelation(suppress_warning=True))
__processing=None
for processing in [correlatorprocessing, coherentstokesprocessing, incoherentstokesprocessing, flyseyeprocessing, nonstandardprocessing]:
if processing:
if __processing is None:
__processing=ltasip.Processing()
__processing.append(processing.get_pyxb_rtprocessing()
__processing.append(processing._get_pyxb_rtprocessing(suppress_warning=True)
)
self.__pyxb_subarraypointing = ltasip.SubArrayPointing(
pointing=pointing.get_pyxb_pointing(),
pointing=pointing._get_pyxb_pointing(suppress_warning=True),
beamNumber=beamnumber,
subArrayPointingIdentifier=identifier.get_pyxb_identifier(),
subArrayPointingIdentifier=identifier._get_pyxb_identifier(suppress_warning=True),
measurementType=measurementtype,
targetName=targetname,
startTime=starttime,
......@@ -1127,7 +1190,9 @@ class SubArrayPointing():
relations=__relations,
measurementDescription=measurementdescription)
def get_pyxb_subarraypointing(self):
def _get_pyxb_subarraypointing(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_subarraypointing
......@@ -1158,7 +1223,7 @@ class Observation():
__stations = ltasip.Stations()
for station in stations:
__stations.append(station.get_pyxb_station())
__stations.append(station._get_pyxb_station(suppress_warning=True))
__tbbevents=None
if(transientbufferboardevents):
......@@ -1170,7 +1235,7 @@ class Observation():
if(subarraypointings):
__pointings = ltasip.SubArrayPointings()
for point in subarraypointings:
__pointings.append(point.get_pyxb_subarraypointing())
__pointings.append(point._get_pyxb_subarraypointing(suppress_warning=True))
self.__pyxb_observation = ltasip.Observation(
observingMode=observingmode,
......@@ -1198,7 +1263,9 @@ class Observation():
if channelwidth_frequency and channelwidth_frequencyunit:
self.__pyxb_observation.channelwidth=ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit),
def get_pyxb_observation(self):
def _get_pyxb_observation(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__pyxb_observation
......@@ -1210,6 +1277,9 @@ class Observation():
# SIP document
class Sip(object):
"""
The main Sip object. Instantiate this with the dataproduct you want to describe/ingest.
Then add all related items to it, like observation, pipeline runs, and intermediate dataproducts. """
print "\n################"
print VERSION
......@@ -1242,18 +1312,25 @@ class Sip(object):
coInvestigator=project_coinvestigators,
)
self.__sip.dataProduct=dataproduct.get_pyxb_dataproduct()
self.__sip.dataProduct=dataproduct._get_pyxb_dataproduct(suppress_warning=True)
self.get_prettyxml() # for validation
@classmethod
def from_xml(cls, xml):
"""
If you are processing data that is already described by a SIP (e.g. LTA data), you can instantiate a Sip object
from that SIP XML string. When adding related dataproducts based on the SIP that describes it, this add all the
SIP content to your new SIP. This way, you only have to fill in the gaps, e.g. add your pipeline run.
-> add_related_dataproduct_with_history()
"""
newsip = Sip.__new__(Sip)
newsip._set_pyxb_sip(ltasip.CreateFromDocument(xml), suppress_warning=True)
newsip.get_prettyxml() # for validation
return newsip
def get_pyxb_sip(self, suppress_warning=False):
def _get_pyxb_sip(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__sip
......@@ -1269,16 +1346,16 @@ class Sip(object):
#---
def add_related_dataproduct(self, dataproduct):
self.__sip.relatedDataProduct.append(dataproduct.get_pyxb_dataproduct())
self.__sip.relatedDataProduct.append(dataproduct._get_pyxb_dataproduct(suppress_warning=True))
return self.get_prettyxml()
def add_observation(self, observation):
self.__sip.observation.append(observation.get_pyxb_observation())
self.__sip.observation.append(observation._get_pyxb_observation(suppress_warning=True))
return self.get_prettyxml()
def add_pipelinerun(self, pipeline):
self.__sip.pipelineRun.append(pipeline.get_pyxb_pipeline())
self.__sip.pipelineRun.append(pipeline._get_pyxb_pipeline(suppress_warning=True))
return self.get_prettyxml()
......@@ -1303,7 +1380,7 @@ class Sip(object):
contents):
self.__sip.parset.append(ltasip.Parset(
identifier=identifier.get_pyxb_identifier(),
identifier=identifier._get_pyxb_identifier(suppress_warning=True),
contents=contents
))
......@@ -1342,6 +1419,15 @@ class Sip(object):
self.__sip.parset.append(par)
return self.get_prettyxml()
def get_dataproduct_identifier(self):
"""
Get the identifier of the dataproduct that is described by this Sip, e.g. for reference by your pipeline run..
"""
identifier = Identifier.__new__()
identifier._set_pyxb_identifier(self.__sip.dataproduct.identifier)
# this will also validate the document so far
def get_prettyxml(self):
try:
......@@ -1356,7 +1442,6 @@ class Sip(object):
print err.details()
raise err
def prettyprint(self):
print self.get_prettyxml()
......
lib/validator.py
+ 83
− 7
View file @ fccef0c0
from lxml import etree
import os
import ltasip
d = os.path.dirname(os.path.realpath(__file__))
XSDPATH = d+"/LTA-SIP.xsd"
def validate(xmlpath, xsdpath):
def validate(xmlpath, xsdpath=XSDPATH):
'''validates given xml file against given xsd file'''
print "validating", xmlpath, "against", xsdpath
with open(xsdpath) as xsd:
xmlschema_doc = etree.parse(xsd)
xmlschema = etree.XMLSchema(xmlschema_doc)
......@@ -20,17 +25,88 @@ def validate(xmlpath, xsdpath):
except Exception as err:
print err
print "SIP is valid according to schema definition!"
return valid
#expose a main method in this lib module which can be used by external programs
#or by the bin/validatesip 'program'
def check_consistency(xmlpath):
"""
Checks the general structure of the provided SIP XML. E.g.:
Is/Are the processes/es present that created the described dataproduct / related dataproducts?
Are the input dataproducts for these processes present?
"""
print "Checking", xmlpath, "for structural consistency"
with open(xmlpath) as f:
xml = f.read()
sip = ltasip.CreateFromDocument(xml)
linkstodataproduct = {}
linkstoprocess = {}
# the dataproduct that is described by the sip
data_out = sip.dataProduct
id_out = str(data_out.dataProductIdentifier.identifier)
id_process = str(data_out.processIdentifier.identifier)
linkstodataproduct.setdefault(id_out,[]).append(id_process)
# the input / intermediate dataproducts
for data_in in sip.relatedDataProduct:
id_in = str(data_in.dataProductIdentifier.identifier)
id_process = str(data_in.processIdentifier.identifier)
linkstodataproduct.setdefault(id_in,[]).append(id_process)
# the observations
for obs in sip.observation:
id_obs = str(obs.observationId.identifier)
id_process = str(obs.processIdentifier.identifier)
linkstoprocess.setdefault(id_process,[])
# the data processing steps
for pipe in sip.pipelineRun:
id_pipe = str(pipe.processIdentifier.identifier)
id_in = []
for id in pipe.sourceData.content():
id_in.append(str(id.identifier))
linkstoprocess.setdefault(id_pipe,[]).append(id_in)
# the data processing steps
for unspec in sip.unspecifiedProcess:
id_unspec = str(unspec.processIdentifier.identifier)
linkstoprocess.setdefault(id_unspec,[])
# todo: online processing
# todo: parsets (?)
for id in linkstodataproduct:
for id_from in linkstodataproduct.get(id):
if not id_from in linkstoprocess:
raise Exception("The pipeline or observation that created dataproduct '"+ id + "' seems to be missing! -> ", id_from)
for id in linkstoprocess:
for ids_from in linkstoprocess.get(id):
for id_from in ids_from:
if not id_from in linkstodataproduct:
raise Exception("The input dataproduct for pipeline '"+ id +"' seems to be missing! -> ", id_from)
print "General SIP structure seems ok!"
return True # already raised Exception if there was a problem...
def main(xml):
#do the proper calls to the SIP API
# parse cmdline args etc
"""
validates given xml against the SIP XSD and does consistency check
"""
try:
xml = xml
xsd = d+"/LTA-SIP.xsd"
return validate(xml, xsd)
xsd = XSDPATH
valid = validate(xml, xsd)
consistent = check_consistency(xml)
return valid and consistent
except Exception as err:
print "An error occurred:"
print err
lib/visualizer.py
+ 5
− 5
View file @ fccef0c0
......@@ -9,7 +9,7 @@ ltasip.Namespace.setPrefix('sip')
def visualize_sip(sip, path="sip.visualize", format="svg", view=False):
if type(sip) == siplib.Sip:
sip = sip.sip
sip = sip._get_pyxb_sip(supress_warning=True)
linkstodataproduct = {}
linkstoprocess = {}
......@@ -161,12 +161,12 @@ def stylize(graph):
def main(argv):
print "Reading xml from file", argv[1]
with open(argv[1]) as f:
def main(xmlpath):
print "Reading xml from file", xmlpath
with open(xmlpath) as f:
xml = f.read()
sip = ltasip.CreateFromDocument(xml)
path = argv[1]+".visualize"
path = xmlpath+".visualize"
format = 'svg'
visualize_sip(sip, path, format)
......
test/CMakeLists.txt
+ 2
− 0
View file @ fccef0c0
# $Id: CMakeLists.txt 32909 2015-11-18 10:26:41Z schaap $
include(LofarCTest)
lofar_add_test(test_validator)
lofar_add_test(test_siplib)
lofar_add_test(test_feedback)
lofar_add_test(test_visualizer)
set(test_resource_files
sipfrommom.xml
valid_sip.xml
......
test/test_feedback.py
+ 5
− 10
View file @ fccef0c0
......@@ -27,7 +27,6 @@ from lofar.lta.sip import feedback
import uuid
TMPFILE_PATH = "/tmp/test_siplib.xml"# todo: how to deploy in testdir?
VALIDFILE_PATH ="/tmp/valid_sip.xml" # todo: how to deploy in testdir?
FEEDBACK_PATH="/tmp/testmetadata_file.Correlated.modified" # todo: how to deploy in testdir?
def create_basicdoc():
......@@ -41,11 +40,11 @@ def create_basicdoc():
dataproduct=siplib.SimpleDataProduct(
siplib.DataProductMap(
type="Unknown",
identifier=siplib.Identifier("source","1"),
identifier=siplib.Identifier("test"),
size=1024,
filename="/home/paulus/test.h5",
fileformat="HDF5",
process_identifier=siplib.Identifier("source","1"),
process_identifier=siplib.Identifier("source"),
checksum_md5="hash1",
checksum_adler32= "hash2",
storageticket="ticket"
......@@ -53,10 +52,6 @@ def create_basicdoc():
)
)
class TestSIPvalidator(unittest.TestCase):
def test_validate(self):
self.assertTrue(validator.main(VALIDFILE_PATH))
class TestSIPfeedback(unittest.TestCase):
......@@ -65,7 +60,7 @@ class TestSIPfeedback(unittest.TestCase):
print "===\nCreating basic document:\n"
mysip = create_basicdoc()
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_dataproducts(self):
mysip = create_basicdoc()
......@@ -74,14 +69,14 @@ class TestSIPfeedback(unittest.TestCase):
text = f.readlines()
fb = feedback.Feedback(text)
dataproducts = fb.get_dataproducts(prefix="test.prefix")
product_identifier = siplib.Identifier(id=uuid.uuid1().int>>64, source='testquelle')
product_identifier = siplib.Identifier('test')
dataproducts[0].set_identifier(product_identifier)
for dp in dataproducts:
print "...adding:",dp
mysip.add_related_dataproduct(dp)
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
......
test/test_siplib.py
+ 30
− 32
View file @ fccef0c0
......@@ -27,10 +27,15 @@ import os
#d = os.path.dirname(os.path.realpath(__file__))
TMPFILE_PATH = "/tmp/test_siplib.xml"
VALIDFILE_PATH = "/tmp/valid_sip.xml" # todo: how to deploy in testdir?
RELATEDSIP = '/tmp/sipfrommom.xml' # todo: how to deploy in testdir?
print os.environ
dp_id = siplib.Identifier("test")
in_dpid1 = siplib.Identifier("test")
in_dpid2 = siplib.Identifier("test")
proc_id = siplib.Identifier("test")
pipe_id = siplib.Identifier("test")
obs_id = siplib.Identifier("test")
point_id = siplib.Identifier("test")
def create_basicdoc():
return siplib.Sip(
......@@ -43,11 +48,11 @@ def create_basicdoc():
dataproduct=siplib.SimpleDataProduct(
siplib.DataProductMap(
type="Unknown",
identifier=siplib.Identifier("source",42),
identifier=dp_id,
size=1024,
filename="/home/paulus/test.h5",
fileformat="HDF5",
process_identifier=siplib.Identifier("SIPlib", 1),
process_identifier=pipe_id,
checksum_md5="hash1",
checksum_adler32= "hash2",
storageticket="ticket"
......@@ -62,15 +67,15 @@ def create_processmap():
strategydescription="awesome strategy",
starttime="1980-03-23T10:20:15",
duration= "P6Y3M10DT15H",
identifier=siplib.Identifier("MoM",1),
observation_identifier=siplib.Identifier("SAS",1),
parset_identifier=siplib.Identifier("source",42),
identifier=proc_id,
observation_identifier=obs_id,
parset_identifier=siplib.Identifier("test"),
relations=[
siplib.ProcessRelation(
identifier=siplib.Identifier("source",45),
identifier=siplib.Identifier("test"),
),
siplib.ProcessRelation(
identifier=siplib.Identifier("source",46),
identifier=siplib.Identifier("test"),
)
]
)
......@@ -80,26 +85,21 @@ def create_pipelinemap():
return siplib.PipelineMap(
name="simple",
version="version",
sourcedata_identifiers=[siplib.Identifier('source',1),siplib.Identifier('source',1)],
sourcedata_identifiers=[in_dpid1, in_dpid2],
process_map=create_processmap(),
)
def create_dataproductmap():
return siplib.DataProductMap(
type="Unknown",
identifier=siplib.Identifier("source",1),
identifier=dp_id,
size=2048,
filename="/home/paulus/test.h5",
fileformat="HDF5",
process_identifier=siplib.Identifier("SIPlib", 1),
process_identifier=proc_id,
)
class TestSIPvalidator(unittest.TestCase):
def test_validate(self):
self.assertTrue(validator.main(VALIDFILE_PATH))
class TestSIPlib(unittest.TestCase):
def test_basic_doc(self):
......@@ -107,7 +107,7 @@ class TestSIPlib(unittest.TestCase):
print "===\nCreating basic document:\n"
mysip = create_basicdoc()
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_dataproducts(self):
mysip = create_basicdoc()
......@@ -139,7 +139,7 @@ class TestSIPlib(unittest.TestCase):
filecontent=["content_a","content_b"],
datatype="CoherentStokes",
arraybeam=siplib.SimpleArrayBeam(siplib.ArrayBeamMap(
subarraypointing_identifier=siplib.Identifier("source",42),
subarraypointing_identifier=point_id,
beamnumber=4,
dispersionmeasure=16,
numberofsubbands=3,
......@@ -163,7 +163,7 @@ class TestSIPlib(unittest.TestCase):
create_dataproductmap(),
beams=[siplib.FlysEyeBeam(
arraybeam_map=siplib.ArrayBeamMap(
subarraypointing_identifier=siplib.Identifier("source",42),
subarraypointing_identifier=point_id,
beamnumber=4,
dispersionmeasure=16,
numberofsubbands=3,
......@@ -281,7 +281,7 @@ class TestSIPlib(unittest.TestCase):
print mysip.add_related_dataproduct(
siplib.CorrelatedDataProduct(
create_dataproductmap(),
subarraypointing_identifier=siplib.Identifier("source",42),
subarraypointing_identifier=siplib.Identifier("test"),
subband="1",
starttime="1980-03-23T10:20:15",
duration= "P6Y3M10DT15H",
......@@ -392,8 +392,7 @@ class TestSIPlib(unittest.TestCase):
)
)
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_observation(self):
mysip = create_basicdoc()
......@@ -428,7 +427,7 @@ class TestSIPlib(unittest.TestCase):
equinox="SUN"
),
beamnumber=5,
identifier=siplib.Identifier("source",42),
identifier=point_id,
measurementtype="All Sky",
targetname="Sun",
starttime="1980-03-23T10:20:15",
......@@ -437,7 +436,7 @@ class TestSIPlib(unittest.TestCase):
numberofcorrelateddataproducts=2,
numberofbeamformeddataproducts=1,
relations=[siplib.ProcessRelation(
identifier=siplib.Identifier("source",42)
identifier=siplib.Identifier("test")
)],
correlatorprocessing=siplib.CorrelatorProcessing(
integrationinterval=0.5,
......@@ -493,7 +492,7 @@ class TestSIPlib(unittest.TestCase):
))
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
......@@ -501,11 +500,11 @@ class TestSIPlib(unittest.TestCase):
mysip = create_basicdoc()
print "===\nAdding parset:\n"
print mysip.add_parset(
identifier=siplib.Identifier("source",42),
identifier=siplib.Identifier("test"),
contents="blabla")
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_unspecifiedprocess(self):
mysip = create_basicdoc()
......@@ -516,8 +515,7 @@ class TestSIPlib(unittest.TestCase):
process_map=create_processmap()
)
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_pipelines(self):
mysip = create_basicdoc()
......@@ -613,7 +611,7 @@ class TestSIPlib(unittest.TestCase):
)
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
......@@ -625,7 +623,7 @@ class TestSIPlib(unittest.TestCase):
sip = siplib.Sip.from_xml(xml)
mysip.add_related_dataproduct_with_history(sip)
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.main(TMPFILE_PATH))
self.assertTrue(validator.validate(TMPFILE_PATH))
# run tests if main
if __name__ == '__main__':
......
test/test_validator.py 0 → 100644
+ 36
− 0
View file @ fccef0c0
#!/usr/bin/python
# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id: $
import unittest
from lofar.lta.sip import validator
VALIDFILE_PATH = "/tmp/valid_sip.xml" # todo: how to deploy in testdir?
class TestSIPvalidator(unittest.TestCase):
def test_validate(self):
self.assertTrue(validator.validate(VALIDFILE_PATH))
self.assertTrue(validator.check_consistency(VALIDFILE_PATH))
self.assertTrue(validator.main(VALIDFILE_PATH))
# run tests if main
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
test/test_validator.sh 0 → 100755
+ 3
− 0
View file @ fccef0c0
#!/bin/sh
./runctest.sh test_validator
\ No newline at end of file
test/test_visualizer.py
+ 0
− 1
View file @ fccef0c0
......@@ -27,7 +27,6 @@ from lofar.lta.sip import ltasip
INPUTFILE_PATH = "/tmp/valid_sip.xml"
class TestSIPvisualizer(unittest.TestCase):
def test_visualize(self):
with open(INPUTFILE_PATH) as f:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment