diff --git a/lib/feedback.py b/lib/feedback.py index ec1474166c2248b924f6f4c39f74ec691a056b4e..b9d2d63db359de4e97fae90f404959941538f68e 100644 --- a/lib/feedback.py +++ b/lib/feedback.py @@ -60,8 +60,8 @@ class Feedback(): # return dataproducts objects from 'pseudo feedback', which can be generated from the MS's by some existing code. def get_dataproducts(self, prefix="ObsSW.Observation", - process_identifier=siplib.Identifier('???',uuid.uuid1().int>>64), - subarraypointing_identifier=siplib.Identifier('???',uuid.uuid1().int>>64), + process_identifier=siplib.Identifier('mysource'), + subarraypointing_identifier=siplib.Identifier('mysource'), ): prefixes = prefix.split(".") dps = self.__get_tree_elem(prefix) @@ -84,7 +84,7 @@ class Feedback(): siplib.CorrelatedDataProduct( siplib.DataProductMap( type="Correlator data", - identifier=siplib.Identifier('???',uuid.uuid1().int>>64), + identifier=siplib.Identifier('mysource'), size=dp.get("size"), filename=dp.get("filename"), fileformat=dp.get("fileFormat"), @@ -110,7 +110,7 @@ class Feedback(): dataproduct = siplib.BeamFormedDataProduct( siplib.DataProductMap( type="Correlator data", - identifier=siplib.Identifier('???',uuid.uuid1().int>>64), + identifier=siplib.Identifier('myproject'), size=dp.get("size"), filename=dp.get("filename"), fileformat=dp.get("fileFormat"), @@ -255,7 +255,7 @@ class Feedback(): numberofcorrelateddataproducts=2, # todo numberofbeamformeddataproducts=1, # todo relations=[siplib.ProcessRelation( - identifier=siplib.Identifier('???','???') # todo + identifier=siplib.Identifier('myproject') # todo )], #todo: optional kwargs ) @@ -295,12 +295,12 @@ class Feedback(): strategydescription="awesome strategy", # todo starttime=obs.get("startTime").replace(" ","T"), duration= duration, - identifier=siplib.Identifier(source="SAS", id="SAS VIC Tree Id"), # todo: Not possible to obtain this? - process_identifier=siplib.Identifier(source="MoM",id="momid"), #obs.get(some_beam).get("momID"), # todo: Not possible to obtain this? + identifier=siplib.Identifier(source="myproject"), # todo: Not possible to obtain the ID that this has in the catalog based on the feedback? + process_identifier=siplib.Identifier(source="myproject"), #obs.get(some_beam).get("momID"), # todo: Not possible to obtain the ID that this has in the catalog based on the feedback? #parset_source="parsource", # todo #parset_id="parid", #todo, relations=[siplib.ProcessRelation( - identifier=siplib.Identifier(source="groupid source", id='groupid') #todo: Not possible to obtain this? + identifier=siplib.Identifier(source="myproject") #todo: Not possible to obtain this? )], ), observationdescription=campaign.get("title"), #todo @@ -323,36 +323,32 @@ class Feedback(): return None +def example(fil): + print "Now running example on file", fil -def main(argv): - #do the proper calls to the feedback/SIP API - # parse cmdline args etc - with open(argv[1]) as f: - print argv[1] + with open(fil) as f: text = f.readlines() feedback = Feedback(text) - # Parse complete SIP: - #sips = feedback.get_dataproduct_sips(obs_prefix="ObsSW.Observation", dp_prefix="Observation.DataProducts") - #for sip in sips.values(): - # print sip.sip.dataProduct.fileName + # A) Parse complete SIP: + sips = feedback.get_dataproduct_sips(obs_prefix="ObsSW.Observation", dp_prefix="Observation.DataProducts") + for key in sips.keys(): + print "Created SIP for file "+ str(key) - # Alternatively: Parse dataproducts from pseudo-feedback: - beamsrc = 'MoM' - beamid = '400869' + # B) Alternatively: Parse dataproducts from pseudo-feedback (specialty of Leiden group): - identifier = siplib.Identifier(id='test', source='constructor') - dataproducts = feedback.get_dataproducts(process_identifier=identifier,subarraypointing_identifier=identifier, prefix="test.prefix" ) + process_identifier = siplib.Identifier(source='myproject') # create process identifier + sapointing_identifier = siplib.Identifier(source='myproject') # create subarra ypointing identifier - procids = {"keyatt":"pipe_"+str(uuid.uuid4())} + # either provide identifiers as args... + dataproducts = feedback.get_dataproducts(process_identifier=process_identifier, subarraypointing_identifier=sapointing_identifier, prefix="test.prefix" ) + # ...or set them explicitely for dp in dataproducts: - procid = procids.get("keyatt") # somehow identify correct process id for this dp - dp.set_identifier(siplib.Identifier('Leiden',"data_"+str(uuid.uuid4()))) - dp.set_process_identifier(siplib.Identifier('Leiden',procid)) - dp.set_subarraypointing_identifier(siplib.Identifier('test',"42")) - + dp.set_identifier(siplib.Identifier('myproject')) # create new unique ID for dataproduct + dp.set_process_identifier(process_identifier) # set the identifier of the creating process + dp.set_subarraypointing_identifier(sapointing_identifier) # assign the pointing identifier # Create example doc: sip = siplib.Sip( @@ -372,6 +368,16 @@ def main(argv): sip.prettyprint() + +def main(argv): + + print "! This is a stub, the feedback to SIP conversion is not correctly working at this point." + print "! You may use this as a module to do some feedback parsing, but unfortunately not all information can be determined from feedback to create a valid SIP." + + if argv[1] is not None: + example(argv[1]) + + if __name__ == '__main__': main(sys.argv) diff --git a/lib/siplib.py b/lib/siplib.py index 58684580de6fd33384f6ee56a3d6dbd7ba8c30e2..5fbf90aece6438e6824dbe7133e800d76c108134 100644 --- a/lib/siplib.py +++ b/lib/siplib.py @@ -64,19 +64,23 @@ def print_user_warning(): # Identifier definition (used for LTA entities, i-e- processes and dataproducts): class Identifier(): + """Identifier for LTA entities""" def __init__(self, source, - id, name=None ): - self.__pyxb_identifier=ltasip.IdentifierType(source=source, identifier=id, name=name) - - @classmethod - def request_unique_identifier(cls, source, name=None): unique_id = query.get_unique_id(source) - return cls(source, unique_id, name) + self.__pyxb_identifier=ltasip.IdentifierType(source=str(source), identifier=str(unique_id), name=str(name)) + + + def _setpyxb_identifier(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() + return self - def get_pyxb_identifier(self): + def _get_pyxb_identifier(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_identifier # =============================== @@ -90,9 +94,9 @@ class Station(): antennafield2=None ): - __afields=[antennafield1.get_pyxb_antennafield()] + __afields=[antennafield1._get_pyxb_antennafield(suppress_warning=True)] if antennafield2: - __afields.append(antennafield2.get_pyxb_antennafield()) + __afields.append(antennafield2._get_pyxb_antennafield(suppress_warning=True)) self.__pyxb_station=ltasip.Station( name=name, stationType=type, @@ -109,7 +113,6 @@ class Station(): __afield1=None __afield2=None - print os.path.abspath(STATION_CONFIG_PATH) with open(STATION_CONFIG_PATH, 'r') as f: for line in f.readlines(): if line.strip(): @@ -145,7 +148,9 @@ class Station(): antennafield2=__afield2, ) - def get_pyxb_station(self): + def _get_pyxb_station(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_station @@ -168,7 +173,9 @@ class AntennafieldXYZ(): ) ) - def get_pyxb_antennafield(self): + def _get_pyxb_antennafield(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_antennafield class AntennafieldRadLonLat(): @@ -191,7 +198,9 @@ class AntennafieldRadLonLat(): ) ) - def get_pyxb_antennafield(self): + def _get_pyxb_antennafield(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_antennafield @@ -208,7 +217,7 @@ class PipelineMap(): __sourcedata=ltasip.DataSources() for id in sourcedata_identifiers: - __sourcedata.append(id.get_pyxb_identifier()) + __sourcedata.append(id._get_pyxb_identifier(suppress_warning=True)) self.pipeline_map=dict( pipelineName=name, @@ -225,7 +234,9 @@ class SimplePipeline(): def __init__(self, pipeline_map): self.__pyxb_pipeline=ltasip.PipelineRun(**pipeline_map.get_dict()) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline @@ -256,7 +267,9 @@ class ImagingPipeline(): **pipeline_map.get_dict() ) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline @@ -281,7 +294,7 @@ class CalibrationPipeline(): **pipeline_map.get_dict() ) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): return self.__pyxb_pipeline class AveragingPipeline(): @@ -302,7 +315,9 @@ class AveragingPipeline(): ) #self.__pyxb_pipeline._setAttribute("xsi:type","ns1:AveragingPipeline") - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline @@ -338,14 +353,18 @@ class PulsarPipeline(): skipPreFold=skipprefold, **pipeline_map.get_dict() ) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline class CosmicRayPipeline(): def __init__(self, pipeline_map): self.__pyxb_pipeline=ltasip.CosmicRayPipeline(**pipeline_map.get_dict()) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline class LongBaselinePipeline(): @@ -359,14 +378,18 @@ class LongBaselinePipeline(): subbandGroupsPerMS=subbandgroupspermS, **pipeline_map.get_dict()) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline class GenericPipeline(): def __init__(self, pipeline_map): self.__pyxb_pipeline=ltasip.GenericPipeline(**pipeline_map.get_dict()) - def get_pyxb_pipeline(self): + def _get_pyxb_pipeline(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pipeline @@ -388,10 +411,10 @@ class DataProductMap(): storageticket=None ): self.dataproduct_map= dict(dataProductType=type, - dataProductIdentifier=identifier.get_pyxb_identifier(), + dataProductIdentifier=identifier._get_pyxb_identifier(suppress_warning=True), size=size, fileName=filename, fileFormat=fileformat, - processIdentifier=process_identifier.get_pyxb_identifier(), + processIdentifier=process_identifier._get_pyxb_identifier(suppress_warning=True), storageTicket=storageticket) checksums = [] @@ -411,15 +434,17 @@ class __DataProduct(object): self.__pyxb_dataproduct=pyxb_dataproduct def set_identifier(self, identifier): - self.__pyxb_dataproduct.dataProductIdentifier = identifier.get_pyxb_identifier() + self.__pyxb_dataproduct.dataProductIdentifier = identifier._get_pyxb_identifier(suppress_warning=True) def set_process_identifier(self, identifier): - self.__pyxb_dataproduct.processIdentifier = identifier.get_pyxb_identifier() + self.__pyxb_dataproduct.processIdentifier = identifier._get_pyxb_identifier(suppress_warning=True) def set_subarraypointing_identifier(self, identifier): - self.__pyxb_dataproduct.subArrayPointingIdentifier = identifier.get_pyxb_identifier() + self.__pyxb_dataproduct.subArrayPointingIdentifier = identifier._get_pyxb_identifier(suppress_warning=True) - def get_pyxb_dataproduct(self): + def _get_pyxb_dataproduct(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_dataproduct @@ -458,7 +483,7 @@ class PixelMapDataProduct(__DataProduct): ltasip.PixelMapDataProduct( numberOfAxes=numberofaxes, numberOfCoordinates=len(coordinates), - coordinate = [x.get_pyxb_coordinate() for x in coordinates], + coordinate = [x._get_pyxb_coordinate(suppress_warning=True) for x in coordinates], **dataproduct_map.get_dict()) ) @@ -482,10 +507,10 @@ class SkyImageDataProduct(__DataProduct): ltasip.SkyImageDataProduct( numberOfAxes=numberofaxes, numberOfCoordinates=len(coordinates), - coordinate = [x.get_pyxb_coordinate() for x in coordinates], + coordinate = [x._get_pyxb_coordinate(suppress_warning=True) for x in coordinates], locationFrame=locationframe, timeFrame=timeframe, - observationPointing=observationpointing.get_pyxb_pointing(), + observationPointing=observationpointing._get_pyxb_pointing(suppress_warning=True), restoringBeamMajor=ltasip.Angle(restoringbeammajor_angle, units=restoringbeammajor_angleunit), restoringBeamMinor=ltasip.Angle(restoringbeamminor_angle, units=restoringbeamminor_angleunit), rmsNoise=ltasip.Pixel(rmsnoise, units="Jy/beam"), @@ -511,7 +536,7 @@ class CorrelatedDataProduct(__DataProduct): stationsubband=None): super(CorrelatedDataProduct, self).__init__( ltasip.CorrelatedDataProduct( - subArrayPointingIdentifier=subarraypointing_identifier.get_pyxb_identifier(), + subArrayPointingIdentifier=subarraypointing_identifier._get_pyxb_identifier(suppress_warning=True), centralFrequency=ltasip.Frequency(central_frequency, units=central_frequencyunit), channelWidth=ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit), subband=subband, @@ -573,7 +598,7 @@ class PulpDataProduct(__DataProduct): ltasip.PulpDataProduct( fileContent=filecontent, dataType=datatype, - arrayBeam=arraybeam.get_pyxb_beam(), + arrayBeam=arraybeam._get_pyxb_beam(suppress_warning=True), **dataproduct_map.get_dict()) ) @@ -590,7 +615,7 @@ class BeamFormedDataProduct(__DataProduct): __beams=ltasip.ArrayBeams() __nbeams = len(beams) for beam in beams: - __beams.append(beam.get_pyxb_beam()) + __beams.append(beam._get_pyxb_beam(suppress_warning=True)) super(BeamFormedDataProduct, self).__init__( ltasip.BeamFormedDataProduct( numberOfBeams=__nbeams, @@ -614,15 +639,17 @@ class SpectralCoordinate(): args = dict(spectralQuantity=ltasip.SpectralQuantity(value_=quantity_value, type=quantity_type)) if isinstance(axis, LinearAxis): - args.update(dict(spectralLinearAxis=axis.get_pyxb_axis())) + args.update(dict(spectralLinearAxis=axis._get_pyxb_axis(suppress_warning=True))) elif isinstance(axis, TabularAxis): - args.update(dict(spectralTabularAxis=axis.get_pyxb_axis())) + args.update(dict(spectralTabularAxis=axis._get_pyxb_axis(suppress_warning=True))) else: print "wrong axis type:",type(axis) self.__pyxb_coordinate=ltasip.SpectralCoordinate(**args) - def get_pyxb_coordinate(self): + def _get_pyxb_coordinate(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_coordinate @@ -635,16 +662,18 @@ class TimeCoordinate(): args = dict(equinox=equinox) if isinstance(axis, LinearAxis): - args.update(dict(timeLinearAxis=axis.get_pyxb_axis())) + args.update(dict(timeLinearAxis=axis._get_pyxb_axis(suppress_warning=True))) elif isinstance(axis, TabularAxis): - args.update(dict(timeTabularAxis=axis.get_pyxb_axis())) + args.update(dict(timeTabularAxis=axis._get_pyxb_axis(suppress_warning=True))) else: print "wrong axis type:",type(axis) self.__pyxb_coordinate=ltasip.TimeCoordinate(**args) - def get_pyxb_coordinate(self): + def _get_pyxb_coordinate(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_coordinate @@ -654,10 +683,12 @@ class PolarizationCoordinate(): polarizations ): self.__pyxb_coordinate=ltasip.PolarizationCoordinate( - polarizationTabularAxis=tabularaxis.get_pyxb_axis(), + polarizationTabularAxis=tabularaxis._get_pyxb_axis(suppress_warning=True), polarization=polarizations) - def get_pyxb_coordinate(self): + def _get_pyxb_coordinate(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_coordinate @@ -679,7 +710,7 @@ class DirectionCoordinate(): latitudepole_angleunit): self.__pyxb_coordinate=ltasip.DirectionCoordinate( - directionLinearAxis=[linearaxis_a.get_pyxb_axis(), linearaxis_b.get_pyxb_axis()], + directionLinearAxis=[linearaxis_a._get_pyxb_axis(suppress_warning=True), linearaxis_b._get_pyxb_axis(suppress_warning=True)], PC0_0=pc0_0, PC0_1=pc0_1, PC1_0=pc1_0, @@ -692,7 +723,9 @@ class DirectionCoordinate(): latitudePole=ltasip.Angle(latitudepole_angle, units=latitudepole_angleunit) ) - def get_pyxb_coordinate(self): + def _get_pyxb_coordinate(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_coordinate # ########### @@ -714,7 +747,7 @@ class ArrayBeamMap(): channelspersubband, stokes): self.arraybeam_map=dict( - subArrayPointingIdentifier=subarraypointing_identifier.get_pyxb_identifier(), + subArrayPointingIdentifier=subarraypointing_identifier._get_pyxb_identifier(suppress_warning=True), beamNumber=beamnumber, dispersionMeasure=dispersionmeasure, numberOfSubbands=numberofsubbands, @@ -733,7 +766,9 @@ class SimpleArrayBeam(): def __init__(self, arraybeam_map): self.__pyxb_beam=ltasip.ArrayBeam(**arraybeam_map.get_dict()) - def get_pyxb_beam(self): + def _get_pyxb_beam(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_beam @@ -747,7 +782,9 @@ class CoherentStokesBeam(): offset=offset, **arraybeam_map.get_dict()) - def get_pyxb_beam(self): + def _get_pyxb_beam(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_beam @@ -755,7 +792,9 @@ class IncoherentStokesBeam(): def __init__(self, arraybeam_map): self.__pyxb_beam=ltasip.IncoherentStokesBeam(**arraybeam_map.get_dict()) - def get_pyxb_beam(self): + def _get_pyxb_beam(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_beam @@ -764,10 +803,12 @@ class FlysEyeBeam(): arraybeam_map, station): self.__pyxb_beam=ltasip.FlysEyeBeam( - station=station.get_pyxb_station(), + station=station._get_pyxb_station(suppress_warning=True), **arraybeam_map.get_dict()) - def get_pyxb_beam(self): + def _get_pyxb_beam(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_beam @@ -798,7 +839,9 @@ class CorrelatorProcessing(): # Somehow this does not work in the constructor: self.__pyxb_rtprocessing.channelwidth=__channelwidth - def get_pyxb_rtprocessing(self): + def _get_pyxb_rtprocessing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_rtprocessing class CoherentStokesProcessing(): @@ -821,7 +864,7 @@ class CoherentStokesProcessing(): __stations = ltasip.Stations() for station in stations: - __stations.append(station.get_pyxb_station()) + __stations.append(station._get_pyxb_station(suppress_warning=True)) __channelwidth=None if channelwidth_frequency and channelwidth_frequencyunit: @@ -844,7 +887,9 @@ class CoherentStokesProcessing(): # Somehow this does not work in the constructor: self.__pyxb_rtprocessing.channelwidth=__channelwidth - def get_pyxb_rtprocessing(self): + def _get_pyxb_rtprocessing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_rtprocessing # This is identical to coherent stokes. Redundancy already in the SIP schema... @@ -867,7 +912,7 @@ class IncoherentStokesProcessing(): ): __stations = ltasip.Stations() for station in stations: - __stations.append(station.get_pyxb_station()) + __stations.append(station._get_pyxb_station(suppress_warning=True)) __channelwidth=None if channelwidth_frequency and channelwidth_frequencyunit: @@ -889,7 +934,9 @@ class IncoherentStokesProcessing(): # Somehow this does not work in the constructor: self.__pyxb_rtprocessing.channelwidth=__channelwidth - def get_pyxb_rtprocessing(self): + def _get_pyxb_rtprocessing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_rtprocessing class FlysEyeProcessing(): @@ -924,7 +971,9 @@ class FlysEyeProcessing(): self.__pyxb_rtprocessing.channelwidth=__channelwidth - def get_pyxb_rtprocessing(self): + def _get_pyxb_rtprocessing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_rtprocessing class NonStandardProcessing(): @@ -940,7 +989,9 @@ class NonStandardProcessing(): channelWidth=ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit) ) - def get_pyxb_rtprocessing(self): + def _get_pyxb_rtprocessing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_rtprocessing @@ -961,15 +1012,15 @@ class ProcessMap(): __relations=ltasip.ProcessRelations() for rel in relations: - __relations.append(rel.get_pyxb_processrelation()) - self.process_map = dict(processIdentifier=identifier.get_pyxb_identifier(), - observationId=observation_identifier.get_pyxb_identifier(), + __relations.append(rel._get_pyxb_processrelation(suppress_warning=True)) + self.process_map = dict(processIdentifier=identifier._get_pyxb_identifier(suppress_warning=True), + observationId=observation_identifier._get_pyxb_identifier(suppress_warning=True), relations=__relations, strategyName=strategyname, strategyDescription=strategydescription, startTime=starttime, duration=duration) if parset_identifier: - self.process_map["parset"]=parset_identifier.get_pyxb_identifier() + self.process_map["parset"]=parset_identifier._get_pyxb_identifier(suppress_warning=True) def get_dict(self): return self.process_map @@ -982,10 +1033,12 @@ class ProcessRelation(): self.__pyxb_processrelation=ltasip.ProcessRelation( relationType=ltasip.ProcessRelationType(type), - identifier=identifier.get_pyxb_identifier() + identifier=identifier._get_pyxb_identifier(suppress_warning=True) ) - def get_pyxb_processrelation(self): + def _get_pyxb_processrelation(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_processrelation @@ -993,7 +1046,9 @@ class SimpleProcess(): def __init__(self, process_map): self.pyxb_process = ltasip.Process(**process_map.get_dict()) - def get_pyxb_process(self): + def _get_pyxb_process(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.pyxb_process @@ -1015,7 +1070,9 @@ class PointingRaDec(): declination=ltasip.Angle(dec_angle, units=dec_angleunit), equinox=equinox) - def get_pyxb_pointing(self): + def _get_pyxb_pointing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pointing @@ -1033,7 +1090,9 @@ class PointingAltAz(): altitude=ltasip.Angle(alt_angle, units=alt_angleunit), equinox=equinox) - def get_pyxb_pointing(self): + def _get_pyxb_pointing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_pointing @@ -1056,7 +1115,9 @@ class LinearAxis(): referencePixel=referencepixel, referenceValue=referencevalue ) - def get_pyxb_axis(self): + def _get_pyxb_axis(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_axis @@ -1075,7 +1136,9 @@ class TabularAxis(): length=length, ) - def get_pyxb_axis(self): + def _get_pyxb_axis(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_axis @@ -1101,21 +1164,21 @@ class SubArrayPointing(): __relations=ltasip.ProcessRelations() for rel in relations: - __relations.append(rel.get_pyxb_processrelation()) + __relations.append(rel._get_pyxb_processrelation(suppress_warning=True)) __processing=None for processing in [correlatorprocessing, coherentstokesprocessing, incoherentstokesprocessing, flyseyeprocessing, nonstandardprocessing]: if processing: if __processing is None: __processing=ltasip.Processing() - __processing.append(processing.get_pyxb_rtprocessing() + __processing.append(processing._get_pyxb_rtprocessing(suppress_warning=True) ) self.__pyxb_subarraypointing = ltasip.SubArrayPointing( - pointing=pointing.get_pyxb_pointing(), + pointing=pointing._get_pyxb_pointing(suppress_warning=True), beamNumber=beamnumber, - subArrayPointingIdentifier=identifier.get_pyxb_identifier(), + subArrayPointingIdentifier=identifier._get_pyxb_identifier(suppress_warning=True), measurementType=measurementtype, targetName=targetname, startTime=starttime, @@ -1127,7 +1190,9 @@ class SubArrayPointing(): relations=__relations, measurementDescription=measurementdescription) - def get_pyxb_subarraypointing(self): + def _get_pyxb_subarraypointing(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_subarraypointing @@ -1158,7 +1223,7 @@ class Observation(): __stations = ltasip.Stations() for station in stations: - __stations.append(station.get_pyxb_station()) + __stations.append(station._get_pyxb_station(suppress_warning=True)) __tbbevents=None if(transientbufferboardevents): @@ -1170,7 +1235,7 @@ class Observation(): if(subarraypointings): __pointings = ltasip.SubArrayPointings() for point in subarraypointings: - __pointings.append(point.get_pyxb_subarraypointing()) + __pointings.append(point._get_pyxb_subarraypointing(suppress_warning=True)) self.__pyxb_observation = ltasip.Observation( observingMode=observingmode, @@ -1198,7 +1263,9 @@ class Observation(): if channelwidth_frequency and channelwidth_frequencyunit: self.__pyxb_observation.channelwidth=ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit), - def get_pyxb_observation(self): + def _get_pyxb_observation(self, suppress_warning=False): + if not suppress_warning: + print_user_warning() return self.__pyxb_observation @@ -1210,6 +1277,9 @@ class Observation(): # SIP document class Sip(object): + """ + The main Sip object. Instantiate this with the dataproduct you want to describe/ingest. + Then add all related items to it, like observation, pipeline runs, and intermediate dataproducts. """ print "\n################" print VERSION @@ -1242,18 +1312,25 @@ class Sip(object): coInvestigator=project_coinvestigators, ) - self.__sip.dataProduct=dataproduct.get_pyxb_dataproduct() + self.__sip.dataProduct=dataproduct._get_pyxb_dataproduct(suppress_warning=True) self.get_prettyxml() # for validation @classmethod def from_xml(cls, xml): + """ + If you are processing data that is already described by a SIP (e.g. LTA data), you can instantiate a Sip object + from that SIP XML string. When adding related dataproducts based on the SIP that describes it, this add all the + SIP content to your new SIP. This way, you only have to fill in the gaps, e.g. add your pipeline run. + + -> add_related_dataproduct_with_history() + """ newsip = Sip.__new__(Sip) newsip._set_pyxb_sip(ltasip.CreateFromDocument(xml), suppress_warning=True) newsip.get_prettyxml() # for validation return newsip - def get_pyxb_sip(self, suppress_warning=False): + def _get_pyxb_sip(self, suppress_warning=False): if not suppress_warning: print_user_warning() return self.__sip @@ -1269,16 +1346,16 @@ class Sip(object): #--- def add_related_dataproduct(self, dataproduct): - self.__sip.relatedDataProduct.append(dataproduct.get_pyxb_dataproduct()) + self.__sip.relatedDataProduct.append(dataproduct._get_pyxb_dataproduct(suppress_warning=True)) return self.get_prettyxml() def add_observation(self, observation): - self.__sip.observation.append(observation.get_pyxb_observation()) + self.__sip.observation.append(observation._get_pyxb_observation(suppress_warning=True)) return self.get_prettyxml() def add_pipelinerun(self, pipeline): - self.__sip.pipelineRun.append(pipeline.get_pyxb_pipeline()) + self.__sip.pipelineRun.append(pipeline._get_pyxb_pipeline(suppress_warning=True)) return self.get_prettyxml() @@ -1303,7 +1380,7 @@ class Sip(object): contents): self.__sip.parset.append(ltasip.Parset( - identifier=identifier.get_pyxb_identifier(), + identifier=identifier._get_pyxb_identifier(suppress_warning=True), contents=contents )) @@ -1342,6 +1419,15 @@ class Sip(object): self.__sip.parset.append(par) return self.get_prettyxml() + + def get_dataproduct_identifier(self): + """ + Get the identifier of the dataproduct that is described by this Sip, e.g. for reference by your pipeline run.. + """ + identifier = Identifier.__new__() + identifier._set_pyxb_identifier(self.__sip.dataproduct.identifier) + + # this will also validate the document so far def get_prettyxml(self): try: @@ -1356,7 +1442,6 @@ class Sip(object): print err.details() raise err - def prettyprint(self): print self.get_prettyxml() diff --git a/lib/validator.py b/lib/validator.py index 4443b513a31c951d7c0699d064a875704a600aef..095063e8ddc6708284649739f8b3e63c3e2fed24 100644 --- a/lib/validator.py +++ b/lib/validator.py @@ -1,11 +1,16 @@ from lxml import etree import os +import ltasip d = os.path.dirname(os.path.realpath(__file__)) +XSDPATH = d+"/LTA-SIP.xsd" -def validate(xmlpath, xsdpath): +def validate(xmlpath, xsdpath=XSDPATH): '''validates given xml file against given xsd file''' + + print "validating", xmlpath, "against", xsdpath + with open(xsdpath) as xsd: xmlschema_doc = etree.parse(xsd) xmlschema = etree.XMLSchema(xmlschema_doc) @@ -20,17 +25,88 @@ def validate(xmlpath, xsdpath): except Exception as err: print err + print "SIP is valid according to schema definition!" return valid -#expose a main method in this lib module which can be used by external programs -#or by the bin/validatesip 'program' + +def check_consistency(xmlpath): + """ + Checks the general structure of the provided SIP XML. E.g.: + Is/Are the processes/es present that created the described dataproduct / related dataproducts? + Are the input dataproducts for these processes present? + """ + + print "Checking", xmlpath, "for structural consistency" + + with open(xmlpath) as f: + xml = f.read() + sip = ltasip.CreateFromDocument(xml) + + + linkstodataproduct = {} + linkstoprocess = {} + + # the dataproduct that is described by the sip + data_out = sip.dataProduct + id_out = str(data_out.dataProductIdentifier.identifier) + id_process = str(data_out.processIdentifier.identifier) + linkstodataproduct.setdefault(id_out,[]).append(id_process) + + # the input / intermediate dataproducts + for data_in in sip.relatedDataProduct: + id_in = str(data_in.dataProductIdentifier.identifier) + id_process = str(data_in.processIdentifier.identifier) + linkstodataproduct.setdefault(id_in,[]).append(id_process) + + # the observations + for obs in sip.observation: + id_obs = str(obs.observationId.identifier) + id_process = str(obs.processIdentifier.identifier) + linkstoprocess.setdefault(id_process,[]) + + # the data processing steps + for pipe in sip.pipelineRun: + id_pipe = str(pipe.processIdentifier.identifier) + id_in = [] + for id in pipe.sourceData.content(): + id_in.append(str(id.identifier)) + linkstoprocess.setdefault(id_pipe,[]).append(id_in) + + # the data processing steps + for unspec in sip.unspecifiedProcess: + id_unspec = str(unspec.processIdentifier.identifier) + linkstoprocess.setdefault(id_unspec,[]) + + + # todo: online processing + # todo: parsets (?) + + for id in linkstodataproduct: + for id_from in linkstodataproduct.get(id): + if not id_from in linkstoprocess: + raise Exception("The pipeline or observation that created dataproduct '"+ id + "' seems to be missing! -> ", id_from) + + for id in linkstoprocess: + for ids_from in linkstoprocess.get(id): + for id_from in ids_from: + if not id_from in linkstodataproduct: + raise Exception("The input dataproduct for pipeline '"+ id +"' seems to be missing! -> ", id_from) + + print "General SIP structure seems ok!" + return True # already raised Exception if there was a problem... + + def main(xml): - #do the proper calls to the SIP API - # parse cmdline args etc + """ + validates given xml against the SIP XSD and does consistency check + """ + try: xml = xml - xsd = d+"/LTA-SIP.xsd" - return validate(xml, xsd) + xsd = XSDPATH + valid = validate(xml, xsd) + consistent = check_consistency(xml) + return valid and consistent except Exception as err: print "An error occurred:" print err diff --git a/lib/visualizer.py b/lib/visualizer.py index 0f288445aa69189d9102f22da2c4f73cf8d90392..28fc8be3f955b8e28d8c82e89b63cd161b016a25 100755 --- a/lib/visualizer.py +++ b/lib/visualizer.py @@ -9,7 +9,7 @@ ltasip.Namespace.setPrefix('sip') def visualize_sip(sip, path="sip.visualize", format="svg", view=False): if type(sip) == siplib.Sip: - sip = sip.sip + sip = sip._get_pyxb_sip(supress_warning=True) linkstodataproduct = {} linkstoprocess = {} @@ -161,12 +161,12 @@ def stylize(graph): -def main(argv): - print "Reading xml from file", argv[1] - with open(argv[1]) as f: +def main(xmlpath): + print "Reading xml from file", xmlpath + with open(xmlpath) as f: xml = f.read() sip = ltasip.CreateFromDocument(xml) - path = argv[1]+".visualize" + path = xmlpath+".visualize" format = 'svg' visualize_sip(sip, path, format) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ed910c6279a038e167d158c5e6c8b87a1214294f..a39907963bc2554996bf2a886b2ffa798eeebeb4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,11 +1,13 @@ # $Id: CMakeLists.txt 32909 2015-11-18 10:26:41Z schaap $ include(LofarCTest) +lofar_add_test(test_validator) lofar_add_test(test_siplib) lofar_add_test(test_feedback) lofar_add_test(test_visualizer) + set(test_resource_files sipfrommom.xml valid_sip.xml diff --git a/test/test_feedback.py b/test/test_feedback.py index faed74e79ece434154440767439af1eb5af1c12d..31aed41876cea35506b1e67297594dbf02f67c27 100755 --- a/test/test_feedback.py +++ b/test/test_feedback.py @@ -27,7 +27,6 @@ from lofar.lta.sip import feedback import uuid TMPFILE_PATH = "/tmp/test_siplib.xml"# todo: how to deploy in testdir? -VALIDFILE_PATH ="/tmp/valid_sip.xml" # todo: how to deploy in testdir? FEEDBACK_PATH="/tmp/testmetadata_file.Correlated.modified" # todo: how to deploy in testdir? def create_basicdoc(): @@ -41,11 +40,11 @@ def create_basicdoc(): dataproduct=siplib.SimpleDataProduct( siplib.DataProductMap( type="Unknown", - identifier=siplib.Identifier("source","1"), + identifier=siplib.Identifier("test"), size=1024, filename="/home/paulus/test.h5", fileformat="HDF5", - process_identifier=siplib.Identifier("source","1"), + process_identifier=siplib.Identifier("source"), checksum_md5="hash1", checksum_adler32= "hash2", storageticket="ticket" @@ -53,10 +52,6 @@ def create_basicdoc(): ) ) -class TestSIPvalidator(unittest.TestCase): - def test_validate(self): - self.assertTrue(validator.main(VALIDFILE_PATH)) - class TestSIPfeedback(unittest.TestCase): @@ -65,7 +60,7 @@ class TestSIPfeedback(unittest.TestCase): print "===\nCreating basic document:\n" mysip = create_basicdoc() mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) def test_dataproducts(self): mysip = create_basicdoc() @@ -74,14 +69,14 @@ class TestSIPfeedback(unittest.TestCase): text = f.readlines() fb = feedback.Feedback(text) dataproducts = fb.get_dataproducts(prefix="test.prefix") - product_identifier = siplib.Identifier(id=uuid.uuid1().int>>64, source='testquelle') + product_identifier = siplib.Identifier('test') dataproducts[0].set_identifier(product_identifier) for dp in dataproducts: print "...adding:",dp mysip.add_related_dataproduct(dp) mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) diff --git a/test/test_siplib.py b/test/test_siplib.py index 92b489c905a3b2a67e3837124c5e5827e82ee6d3..dbc121391f743bcb933f497442bf99ef9c679c24 100755 --- a/test/test_siplib.py +++ b/test/test_siplib.py @@ -27,10 +27,15 @@ import os #d = os.path.dirname(os.path.realpath(__file__)) TMPFILE_PATH = "/tmp/test_siplib.xml" -VALIDFILE_PATH = "/tmp/valid_sip.xml" # todo: how to deploy in testdir? RELATEDSIP = '/tmp/sipfrommom.xml' # todo: how to deploy in testdir? -print os.environ +dp_id = siplib.Identifier("test") +in_dpid1 = siplib.Identifier("test") +in_dpid2 = siplib.Identifier("test") +proc_id = siplib.Identifier("test") +pipe_id = siplib.Identifier("test") +obs_id = siplib.Identifier("test") +point_id = siplib.Identifier("test") def create_basicdoc(): return siplib.Sip( @@ -43,11 +48,11 @@ def create_basicdoc(): dataproduct=siplib.SimpleDataProduct( siplib.DataProductMap( type="Unknown", - identifier=siplib.Identifier("source",42), + identifier=dp_id, size=1024, filename="/home/paulus/test.h5", fileformat="HDF5", - process_identifier=siplib.Identifier("SIPlib", 1), + process_identifier=pipe_id, checksum_md5="hash1", checksum_adler32= "hash2", storageticket="ticket" @@ -62,15 +67,15 @@ def create_processmap(): strategydescription="awesome strategy", starttime="1980-03-23T10:20:15", duration= "P6Y3M10DT15H", - identifier=siplib.Identifier("MoM",1), - observation_identifier=siplib.Identifier("SAS",1), - parset_identifier=siplib.Identifier("source",42), + identifier=proc_id, + observation_identifier=obs_id, + parset_identifier=siplib.Identifier("test"), relations=[ siplib.ProcessRelation( - identifier=siplib.Identifier("source",45), + identifier=siplib.Identifier("test"), ), siplib.ProcessRelation( - identifier=siplib.Identifier("source",46), + identifier=siplib.Identifier("test"), ) ] ) @@ -80,26 +85,21 @@ def create_pipelinemap(): return siplib.PipelineMap( name="simple", version="version", - sourcedata_identifiers=[siplib.Identifier('source',1),siplib.Identifier('source',1)], + sourcedata_identifiers=[in_dpid1, in_dpid2], process_map=create_processmap(), ) def create_dataproductmap(): return siplib.DataProductMap( type="Unknown", - identifier=siplib.Identifier("source",1), + identifier=dp_id, size=2048, filename="/home/paulus/test.h5", fileformat="HDF5", - process_identifier=siplib.Identifier("SIPlib", 1), + process_identifier=proc_id, ) -class TestSIPvalidator(unittest.TestCase): - def test_validate(self): - self.assertTrue(validator.main(VALIDFILE_PATH)) - - class TestSIPlib(unittest.TestCase): def test_basic_doc(self): @@ -107,7 +107,7 @@ class TestSIPlib(unittest.TestCase): print "===\nCreating basic document:\n" mysip = create_basicdoc() mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) def test_dataproducts(self): mysip = create_basicdoc() @@ -139,7 +139,7 @@ class TestSIPlib(unittest.TestCase): filecontent=["content_a","content_b"], datatype="CoherentStokes", arraybeam=siplib.SimpleArrayBeam(siplib.ArrayBeamMap( - subarraypointing_identifier=siplib.Identifier("source",42), + subarraypointing_identifier=point_id, beamnumber=4, dispersionmeasure=16, numberofsubbands=3, @@ -163,7 +163,7 @@ class TestSIPlib(unittest.TestCase): create_dataproductmap(), beams=[siplib.FlysEyeBeam( arraybeam_map=siplib.ArrayBeamMap( - subarraypointing_identifier=siplib.Identifier("source",42), + subarraypointing_identifier=point_id, beamnumber=4, dispersionmeasure=16, numberofsubbands=3, @@ -281,7 +281,7 @@ class TestSIPlib(unittest.TestCase): print mysip.add_related_dataproduct( siplib.CorrelatedDataProduct( create_dataproductmap(), - subarraypointing_identifier=siplib.Identifier("source",42), + subarraypointing_identifier=siplib.Identifier("test"), subband="1", starttime="1980-03-23T10:20:15", duration= "P6Y3M10DT15H", @@ -392,8 +392,7 @@ class TestSIPlib(unittest.TestCase): ) ) mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) - + self.assertTrue(validator.validate(TMPFILE_PATH)) def test_observation(self): mysip = create_basicdoc() @@ -428,7 +427,7 @@ class TestSIPlib(unittest.TestCase): equinox="SUN" ), beamnumber=5, - identifier=siplib.Identifier("source",42), + identifier=point_id, measurementtype="All Sky", targetname="Sun", starttime="1980-03-23T10:20:15", @@ -437,7 +436,7 @@ class TestSIPlib(unittest.TestCase): numberofcorrelateddataproducts=2, numberofbeamformeddataproducts=1, relations=[siplib.ProcessRelation( - identifier=siplib.Identifier("source",42) + identifier=siplib.Identifier("test") )], correlatorprocessing=siplib.CorrelatorProcessing( integrationinterval=0.5, @@ -493,7 +492,7 @@ class TestSIPlib(unittest.TestCase): )) mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) @@ -501,11 +500,11 @@ class TestSIPlib(unittest.TestCase): mysip = create_basicdoc() print "===\nAdding parset:\n" print mysip.add_parset( - identifier=siplib.Identifier("source",42), + identifier=siplib.Identifier("test"), contents="blabla") mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) def test_unspecifiedprocess(self): mysip = create_basicdoc() @@ -516,8 +515,7 @@ class TestSIPlib(unittest.TestCase): process_map=create_processmap() ) mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) - + self.assertTrue(validator.validate(TMPFILE_PATH)) def test_pipelines(self): mysip = create_basicdoc() @@ -613,7 +611,7 @@ class TestSIPlib(unittest.TestCase): ) mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) @@ -625,7 +623,7 @@ class TestSIPlib(unittest.TestCase): sip = siplib.Sip.from_xml(xml) mysip.add_related_dataproduct_with_history(sip) mysip.save_to_file(TMPFILE_PATH) - self.assertTrue(validator.main(TMPFILE_PATH)) + self.assertTrue(validator.validate(TMPFILE_PATH)) # run tests if main if __name__ == '__main__': diff --git a/test/test_validator.py b/test/test_validator.py new file mode 100644 index 0000000000000000000000000000000000000000..33077d7e7acb6c088666794bcb8d6e1422557404 --- /dev/null +++ b/test/test_validator.py @@ -0,0 +1,36 @@ +#!/usr/bin/python + +# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy) +# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands +# +# This file is part of the LOFAR software suite. +# The LOFAR software suite is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# The LOFAR software suite is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>. + +# $Id: $ + +import unittest +from lofar.lta.sip import validator + + +VALIDFILE_PATH = "/tmp/valid_sip.xml" # todo: how to deploy in testdir? + +class TestSIPvalidator(unittest.TestCase): + def test_validate(self): + self.assertTrue(validator.validate(VALIDFILE_PATH)) + self.assertTrue(validator.check_consistency(VALIDFILE_PATH)) + self.assertTrue(validator.main(VALIDFILE_PATH)) + +# run tests if main +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test/test_validator.sh b/test/test_validator.sh new file mode 100755 index 0000000000000000000000000000000000000000..319b6406ed827270dec0e7353eb13dca6e9a492e --- /dev/null +++ b/test/test_validator.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +./runctest.sh test_validator \ No newline at end of file diff --git a/test/test_visualizer.py b/test/test_visualizer.py index 3bbe9adcc1547fc30385337df0f997dfd290c68b..352077c4eaa6c906164c4d623d007d2bda4603b1 100755 --- a/test/test_visualizer.py +++ b/test/test_visualizer.py @@ -27,7 +27,6 @@ from lofar.lta.sip import ltasip INPUTFILE_PATH = "/tmp/valid_sip.xml" - class TestSIPvisualizer(unittest.TestCase): def test_visualize(self): with open(INPUTFILE_PATH) as f: