TMSS-2173: update external siputils with changes made in LOFAR TMSS master over time

README.md

@@ -25,4 +25,4 @@ But there is some inline documentation in [lib/siplib.py](siplib.py) and you may
 
 The initial version of the siblib in this repo is version 0.4
 
-Because the siplib is strongly coupled to the SIP xsd, the version numbering of the siplib will match with the version of the xsd schema which is 2.8.0 now.
+Because the siplib is strongly coupled to the SIP xsd, the version numbering of the siplib will match with the version of the xsd schema which is 2.8.1 now.
\ No newline at end of file

lib/etc/LTA-SIP.xsd

 <?xml version="1.0" encoding="UTF-8"?>
-<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.8.0" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.8.1" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema">
 	<xs:annotation>
 		<xs:documentation>
 		XML Schema for data model Submission Information Package LOFAR Long Term Archive
@@ -616,7 +616,7 @@
 			<xs:element name="dataProductIdentifier" type="IdentifierType" maxOccurs="unbounded"/>
 		</xs:sequence>
 	</xs:complexType>
-	<xs:complexType name="PipelineRun">
+	<xs:complexType name="PipelineRun" abstract="true">
 		<xs:complexContent>
 			<xs:extension base="Process">
 				<xs:sequence>

lib/ltasip.py

@@ -5,6 +5,18 @@
 # Namespace http://www.astron.nl/SIP-Lofar
 from __future__ import unicode_literals
+
+try:
+    import pyxb.binding
+except AttributeError:
+    import sys
+    if sys.version_info.major == 3 and  sys.version_info.minor >= 10:
+        # MutableSequence, Iterable was moved in pyton 3.10
+        # make pyxb work with 3.10
+        import collections
+        collections.MutableSequence = collections.abc.MutableSequence
+        collections.Iterable = collections.abc.Iterable
+
 import pyxb
 import pyxb.binding
 import pyxb.binding.saxer
@@ -3610,7 +3622,7 @@ class PipelineRun(Process):
     """Complex type {http://www.astron.nl/SIP-Lofar}PipelineRun with content type ELEMENT_ONLY"""
     _TypeDefinition = None
     _ContentTypeTag = pyxb.binding.basis.complexTypeDefinition._CT_ELEMENT_ONLY
-    _Abstract = False
+    _Abstract = True
     _ExpandedName = pyxb.namespace.ExpandedName(Namespace, 'PipelineRun')
     _XSDLocation = pyxb.utils.utility.Location(DEFAULT_SIP_XSD_PATH, 619, 1)
     _ElementMap = Process._ElementMap.copy()

lib/siplib.py

@@ -39,7 +39,7 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-VERSION = "SIPlib 2.8.0"
+VERSION = "SIPlib 2.8.1"
 ltasip.Namespace.setPrefix('sip')
 
 
@@ -896,7 +896,7 @@ class CorrelatorProcessing():
 
         __channelwidth = None
         if channelwidth_frequency and channelwidth_frequencyunit:
-            __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit),
+            __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit)
 
         self.__pyxb_rtprocessing = ltasip.Correlator(
             integrationInterval=ltasip.Time(integrationinterval, units=integrationinterval_unit),
@@ -963,7 +963,7 @@ class CoherentStokesProcessing():
             processingType=processingtype)
 
         # Somehow this does not work in the constructor:
-        self.__pyxb_rtprocessing.channelwidth = __channelwidth
+        self.__pyxb_rtprocessing.channelWidth = __channelwidth
 
     def _get_pyxb_rtprocessing(self, suppress_warning=False):
         if not suppress_warning:
@@ -998,7 +998,7 @@ class IncoherentStokesProcessing():
 
         __channelwidth = None
         if channelwidth_frequency and channelwidth_frequencyunit:
-            __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit),
+            __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit)
 
         self.__pyxb_rtprocessing = ltasip.IncoherentStokes(
             rawSamplingTime=ltasip.Time(rawsamplingtime, units=rawsamplingtime_unit),
@@ -1016,7 +1016,7 @@ class IncoherentStokesProcessing():
             cobaltPipelineNumber=cobaltpipelinenumber,
             processingType = processingtype)
         # Somehow this does not work in the constructor:
-        self.__pyxb_rtprocessing.channelwidth = __channelwidth
+        self.__pyxb_rtprocessing.channelWidth = __channelwidth
 
     def _get_pyxb_rtprocessing(self, suppress_warning=False):
         if not suppress_warning:
@@ -1059,7 +1059,7 @@ class FlysEyeProcessing():
 
 
         # Somehow this does not work in the constructor:
-        self.__pyxb_rtprocessing.channelwidth = __channelwidth
+        self.__pyxb_rtprocessing.channelWidth = __channelwidth
 
     def _get_pyxb_rtprocessing(self, suppress_warning=False):
         if not suppress_warning:
@@ -1347,8 +1347,8 @@ class Observation():
 
         # Somehow this does not work in the constructor:
         if channelwidth_frequency and channelwidth_frequencyunit:
-            self.__pyxb_observation.channelwidth = ltasip.Frequency(channelwidth_frequency,
-                                                                    units=channelwidth_frequencyunit),
+            self.__pyxb_observation.channelWidth = ltasip.Frequency(channelwidth_frequency,
+                                                                    units=channelwidth_frequencyunit)
 
     def _get_pyxb_observation(self, suppress_warning=False):
         if not suppress_warning:
@@ -1446,7 +1446,7 @@ class Sip(object):
                                observingmode,
                                description,
                                process_map,
-                               ):
+                               return_xml=True):
 
         up = ltasip.UnspecifiedProcess(
             observingMode=observingmode,
@@ -1454,21 +1454,22 @@ class Sip(object):
             **process_map.get_dict()
         )
         self.__sip.unspecifiedProcess.append(up)
-
+        if return_xml:
             return self.get_prettyxml()
 
     def add_parset(self,
                    identifier,
-                   contents):
+                   contents,
+                   return_xml=True):
 
         self.__sip.parset.append(ltasip.Parset(
             identifier=identifier._get_pyxb_identifier(suppress_warning=True),
             contents=contents
         ))
-
+        if return_xml:
             return self.get_prettyxml()
 
-    def add_related_dataproduct_with_history(self, relateddataproduct_sip):
+    def add_related_dataproduct_with_history(self, relateddataproduct_sip, return_xml=True):
         # add the dataproduct described by the SIP (if not there)
         if not any(
                 x.dataProductIdentifier.identifier == relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier
@@ -1507,6 +1508,7 @@ class Sip(object):
             for par in relateddataproduct_sip.__sip.parset:
                 if not any(x.identifier.identifier == par.identifier.identifier for x in self.__sip.parset):
                     self.__sip.parset.append(par)
+        if return_xml:
             return self.get_prettyxml()
 
     def get_dataproduct_identifier(self):

lib/validator.py

@@ -95,7 +95,7 @@ def check_consistency(sip):
                 if not id_from in linkstodataproduct:
                     raise Exception("The input dataproduct for pipeline '" + id + "' seems to be missing! -> ", id_from)
 
-    logger.info("General SIP structure seems ok!")
+    logger.info("General SIP structure for dataproduct with sip_id=%s seems ok!", id_out)
     return True  # already raised Exception if there was a problem...
 
 

setup.py

@@ -11,7 +11,7 @@ scripts = glob.glob('bin/*')
 
 setuptools.setup(
     name="siputils",
-    version="2.8.0",
+    version="2.8.1",
     setuptools_git_versioning={
         "template": "{tag}",
         "dev_template": "{tag}.dev{ccount}+git.{sha}",

test/test_files/sipfrommom.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <sip:ltaSip xmlns:sip="http://www.astron.nl/SIP-Lofar"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.0.xsd ">
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.1.xsd ">
     <sipGeneratorVersion>Version v2.17.5 (29-08-2016) (modified by hand by RG, May 2022)</sipGeneratorVersion>
     <project>
         <projectCode>LC6_016</projectCode>

test/test_files/valid_sip.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <sip:ltaSip xmlns:sip="http://www.astron.nl/SIP-Lofar"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.0.xsd ">
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.1.xsd ">
     <sipGeneratorVersion>Version v2.17.5 (29-08-2016) (modified by hand by RG, May 2022)</sipGeneratorVersion>
     <project>
         <projectCode>LC6_016</projectCode>

test/test_siplib.py

@@ -594,12 +594,14 @@ class TestSIPlib(unittest.TestCase):
 
     def test_pipelines(self):
         mysip = create_basicdoc()
+        with self.assertRaises(pyxb.exceptions_.AbstractInstantiationError) as context:
             logger.info("===\nAdding simple pipelinerun:\n")
             logger.info(mysip.add_pipelinerun(
                 siplib.SimplePipeline(
                     create_pipelinemap()
                 )
             ))
+        self.assertIn('Cannot instantiate abstract type {http://www.astron.nl/SIP-Lofar}PipelineRun directly', str(context.exception))
 
         logger.info("===\nAdding generic pipelinerun:\n")
         logger.info(mysip.add_pipelinerun(

version

-2.8.0
+2.8.1