From 8ed768e0c8b8f34fcebb375a1a4ad26bee08f755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20K=C3=BCnsem=C3=B6ller?=
<jkuensem@physik.uni-bielefeld.de>
Date: Thu, 4 Jul 2024 14:57:01 +0200
Subject: [PATCH] TMSS-2173: update external siputils with changes made in
LOFAR TMSS master over time
---
README.md | 2 +-
lib/etc/LTA-SIP.xsd | 4 ++--
lib/ltasip.py | 14 +++++++++++++-
lib/siplib.py | 34 ++++++++++++++++++----------------
lib/validator.py | 2 +-
setup.py | 2 +-
test/test_files/sipfrommom.xml | 2 +-
test/test_files/valid_sip.xml | 2 +-
test/test_siplib.py | 14 ++++++++------
version | 2 +-
10 files changed, 47 insertions(+), 31 deletions(-)
diff --git a/README.md b/README.md
index 7729924..9b6d74f 100644
--- a/README.md
+++ b/README.md
@@ -25,4 +25,4 @@ But there is some inline documentation in [lib/siplib.py](siplib.py) and you may
The initial version of the siblib in this repo is version 0.4
-Because the siplib is strongly coupled to the SIP xsd, the version numbering of the siplib will match with the version of the xsd schema which is 2.8.0 now.
\ No newline at end of file
+Because the siplib is strongly coupled to the SIP xsd, the version numbering of the siplib will match with the version of the xsd schema which is 2.8.1 now.
\ No newline at end of file
diff --git a/lib/etc/LTA-SIP.xsd b/lib/etc/LTA-SIP.xsd
index f1c32c7..cd6618a 100644
--- a/lib/etc/LTA-SIP.xsd
+++ b/lib/etc/LTA-SIP.xsd
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.8.0" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.8.1" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:annotation>
<xs:documentation>
XML Schema for data model Submission Information Package LOFAR Long Term Archive
@@ -616,7 +616,7 @@
<xs:element name="dataProductIdentifier" type="IdentifierType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
- <xs:complexType name="PipelineRun">
+ <xs:complexType name="PipelineRun" abstract="true">
<xs:complexContent>
<xs:extension base="Process">
<xs:sequence>
diff --git a/lib/ltasip.py b/lib/ltasip.py
index 518fc49..c7ad2b7 100644
--- a/lib/ltasip.py
+++ b/lib/ltasip.py
@@ -5,6 +5,18 @@
# Namespace http://www.astron.nl/SIP-Lofar
from __future__ import unicode_literals
+
+try:
+ import pyxb.binding
+except AttributeError:
+ import sys
+ if sys.version_info.major == 3 and sys.version_info.minor >= 10:
+ # MutableSequence, Iterable was moved in pyton 3.10
+ # make pyxb work with 3.10
+ import collections
+ collections.MutableSequence = collections.abc.MutableSequence
+ collections.Iterable = collections.abc.Iterable
+
import pyxb
import pyxb.binding
import pyxb.binding.saxer
@@ -3610,7 +3622,7 @@ class PipelineRun(Process):
"""Complex type {http://www.astron.nl/SIP-Lofar}PipelineRun with content type ELEMENT_ONLY"""
_TypeDefinition = None
_ContentTypeTag = pyxb.binding.basis.complexTypeDefinition._CT_ELEMENT_ONLY
- _Abstract = False
+ _Abstract = True
_ExpandedName = pyxb.namespace.ExpandedName(Namespace, 'PipelineRun')
_XSDLocation = pyxb.utils.utility.Location(DEFAULT_SIP_XSD_PATH, 619, 1)
_ElementMap = Process._ElementMap.copy()
diff --git a/lib/siplib.py b/lib/siplib.py
index 98e64f5..af8b638 100644
--- a/lib/siplib.py
+++ b/lib/siplib.py
@@ -39,7 +39,7 @@ import logging
logger = logging.getLogger(__name__)
-VERSION = "SIPlib 2.8.0"
+VERSION = "SIPlib 2.8.1"
ltasip.Namespace.setPrefix('sip')
@@ -896,7 +896,7 @@ class CorrelatorProcessing():
__channelwidth = None
if channelwidth_frequency and channelwidth_frequencyunit:
- __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit),
+ __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit)
self.__pyxb_rtprocessing = ltasip.Correlator(
integrationInterval=ltasip.Time(integrationinterval, units=integrationinterval_unit),
@@ -963,7 +963,7 @@ class CoherentStokesProcessing():
processingType=processingtype)
# Somehow this does not work in the constructor:
- self.__pyxb_rtprocessing.channelwidth = __channelwidth
+ self.__pyxb_rtprocessing.channelWidth = __channelwidth
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
@@ -998,7 +998,7 @@ class IncoherentStokesProcessing():
__channelwidth = None
if channelwidth_frequency and channelwidth_frequencyunit:
- __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit),
+ __channelwidth = ltasip.Frequency(channelwidth_frequency, units=channelwidth_frequencyunit)
self.__pyxb_rtprocessing = ltasip.IncoherentStokes(
rawSamplingTime=ltasip.Time(rawsamplingtime, units=rawsamplingtime_unit),
@@ -1016,7 +1016,7 @@ class IncoherentStokesProcessing():
cobaltPipelineNumber=cobaltpipelinenumber,
processingType = processingtype)
# Somehow this does not work in the constructor:
- self.__pyxb_rtprocessing.channelwidth = __channelwidth
+ self.__pyxb_rtprocessing.channelWidth = __channelwidth
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
@@ -1059,7 +1059,7 @@ class FlysEyeProcessing():
# Somehow this does not work in the constructor:
- self.__pyxb_rtprocessing.channelwidth = __channelwidth
+ self.__pyxb_rtprocessing.channelWidth = __channelwidth
def _get_pyxb_rtprocessing(self, suppress_warning=False):
if not suppress_warning:
@@ -1347,8 +1347,8 @@ class Observation():
# Somehow this does not work in the constructor:
if channelwidth_frequency and channelwidth_frequencyunit:
- self.__pyxb_observation.channelwidth = ltasip.Frequency(channelwidth_frequency,
- units=channelwidth_frequencyunit),
+ self.__pyxb_observation.channelWidth = ltasip.Frequency(channelwidth_frequency,
+ units=channelwidth_frequencyunit)
def _get_pyxb_observation(self, suppress_warning=False):
if not suppress_warning:
@@ -1446,7 +1446,7 @@ class Sip(object):
observingmode,
description,
process_map,
- ):
+ return_xml=True):
up = ltasip.UnspecifiedProcess(
observingMode=observingmode,
@@ -1454,21 +1454,22 @@ class Sip(object):
**process_map.get_dict()
)
self.__sip.unspecifiedProcess.append(up)
-
- return self.get_prettyxml()
+ if return_xml:
+ return self.get_prettyxml()
def add_parset(self,
identifier,
- contents):
+ contents,
+ return_xml=True):
self.__sip.parset.append(ltasip.Parset(
identifier=identifier._get_pyxb_identifier(suppress_warning=True),
contents=contents
))
+ if return_xml:
+ return self.get_prettyxml()
- return self.get_prettyxml()
-
- def add_related_dataproduct_with_history(self, relateddataproduct_sip):
+ def add_related_dataproduct_with_history(self, relateddataproduct_sip, return_xml=True):
# add the dataproduct described by the SIP (if not there)
if not any(
x.dataProductIdentifier.identifier == relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier
@@ -1507,7 +1508,8 @@ class Sip(object):
for par in relateddataproduct_sip.__sip.parset:
if not any(x.identifier.identifier == par.identifier.identifier for x in self.__sip.parset):
self.__sip.parset.append(par)
- return self.get_prettyxml()
+ if return_xml:
+ return self.get_prettyxml()
def get_dataproduct_identifier(self):
"""
diff --git a/lib/validator.py b/lib/validator.py
index 64097a4..d926789 100644
--- a/lib/validator.py
+++ b/lib/validator.py
@@ -95,7 +95,7 @@ def check_consistency(sip):
if not id_from in linkstodataproduct:
raise Exception("The input dataproduct for pipeline '" + id + "' seems to be missing! -> ", id_from)
- logger.info("General SIP structure seems ok!")
+ logger.info("General SIP structure for dataproduct with sip_id=%s seems ok!", id_out)
return True # already raised Exception if there was a problem...
diff --git a/setup.py b/setup.py
index 6214ee1..6b7ab58 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ scripts = glob.glob('bin/*')
setuptools.setup(
name="siputils",
- version="2.8.0",
+ version="2.8.1",
setuptools_git_versioning={
"template": "{tag}",
"dev_template": "{tag}.dev{ccount}+git.{sha}",
diff --git a/test/test_files/sipfrommom.xml b/test/test_files/sipfrommom.xml
index 2de6a69..d53da39 100644
--- a/test/test_files/sipfrommom.xml
+++ b/test/test_files/sipfrommom.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<sip:ltaSip xmlns:sip="http://www.astron.nl/SIP-Lofar"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.0.xsd ">
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.1.xsd ">
<sipGeneratorVersion>Version v2.17.5 (29-08-2016) (modified by hand by RG, May 2022)</sipGeneratorVersion>
<project>
<projectCode>LC6_016</projectCode>
diff --git a/test/test_files/valid_sip.xml b/test/test_files/valid_sip.xml
index bd7815c..ce62f96 100644
--- a/test/test_files/valid_sip.xml
+++ b/test/test_files/valid_sip.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<sip:ltaSip xmlns:sip="http://www.astron.nl/SIP-Lofar"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.0.xsd ">
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.astron.nl/SIP-Lofar LTA-SIP-2.8.1.xsd ">
<sipGeneratorVersion>Version v2.17.5 (29-08-2016) (modified by hand by RG, May 2022)</sipGeneratorVersion>
<project>
<projectCode>LC6_016</projectCode>
diff --git a/test/test_siplib.py b/test/test_siplib.py
index acb514b..53a9e3f 100755
--- a/test/test_siplib.py
+++ b/test/test_siplib.py
@@ -594,12 +594,14 @@ class TestSIPlib(unittest.TestCase):
def test_pipelines(self):
mysip = create_basicdoc()
- logger.info("===\nAdding simple pipelinerun:\n")
- logger.info(mysip.add_pipelinerun(
- siplib.SimplePipeline(
- create_pipelinemap()
- )
- ))
+ with self.assertRaises(pyxb.exceptions_.AbstractInstantiationError) as context:
+ logger.info("===\nAdding simple pipelinerun:\n")
+ logger.info(mysip.add_pipelinerun(
+ siplib.SimplePipeline(
+ create_pipelinemap()
+ )
+ ))
+ self.assertIn('Cannot instantiate abstract type {http://www.astron.nl/SIP-Lofar}PipelineRun directly', str(context.exception))
logger.info("===\nAdding generic pipelinerun:\n")
logger.info(mysip.add_pipelinerun(
diff --git a/version b/version
index 834f262..dbe5900 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-2.8.0
+2.8.1
--
GitLab