From 5e5f41dfab493dec5125cb4590092c4e2e1fa76d Mon Sep 17 00:00:00 2001 From: Roy de Goei <goei@astron.nl> Date: Tue, 22 Apr 2025 13:53:38 +0200 Subject: [PATCH] MAM-56: Update to version 2.8.3. Use pyproject.toml, python version minimal 3.9 and pyxb-x as update of pyxb --- .gitlab-ci.yml | 6 +-- README.md | 43 +++++++++++++++----- lib/etc/LTA-SIP.xsd | 2 +- lib/ltasip.py | 2 +- lib/query.py | 9 ++++- lib/validator.py | 10 ++--- pyproject.toml | 43 ++++++++++++++++++++ setup.py | 99 +++++++++++++++++++++++---------------------- tox.ini | 13 +++++- 9 files changed, 155 insertions(+), 72 deletions(-) create mode 100644 pyproject.toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ba0fed..8ac5ba2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ # This file is a template, and might need editing before it works on your project. # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/python/tags/ -image: python:3.7 +image: python:3.12 stages: - test - build @@ -32,14 +32,14 @@ test: build: stage: build script: - - python setup.py bdist_wheel + - python -m build upload: stage: publish retry: 2 allow_failure: true script: - - python setup.py sdist bdist_wheel + - python -m build - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url https://git.astron.nl/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/* artifacts: diff --git a/README.md b/README.md index 9b6d74f..2a5a46c 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,49 @@ -**About** +# The SIP Library The SIP library is a Python library to assist in creating a valid LOFAR SIP (Submission Information Package) that is required to put data in the Astron LTA (Long Term Archive). It also contains some utilities to validate and visulaize existing SIPs. +The packacha name with python is **lofar_lta_sip** -**General Structure** +## General Structure This library essentially consists of an auto-generated part ([ltasip.py](lib/ltasip.py)), which generates the XML and was itself generated from the XSD schema using pyxb, and a written part ([siplib.py](lib/siplib.py)) that wraps the generated part, -mainly to provide nicer constructors since the pyxb-generated code is a little hard to work against. +mainly to provide nicer constructors since the **pyxb-generated code** is a little hard to work against. -The ltasip.py file can be generated from the LTA SIP schema. Make sure you have installed pyxb (pip install pyxb): +The [ltasip.py](lib/ltasip.py) file can be generated from the LTA SIP schema. Make sure you have installed pyxb (pip install pyxb): pyxbgen -u <path_to>/LTA-SIP.xsd -Copy/paste the generated binding.py in the ltasip.py but watch out for the path used in pyxb.utils.utility.Location -which should have DEFAULT_SIP_XSD_PATH as first argument and the 'generated path', so find-replace this as well. +Copy/paste the generated binding.py in the ltasip.py but watch out for the path used in pyxb.utils.utility. +Location which should have `DEFAULT_SIP_XSD_PATH` as first argument and the 'generated path', so find-replace this as well. -**Getting Started** + +## Getting Started This library is somewhat work in progress, so there is not yet a real end user documentation in that sense. -But there is some inline documentation in [lib/siplib.py](siplib.py) and you may want to check out [test/example.py](example.py) to see how this can be used. +But there is some inline documentation in [lib/siplib.py](siplib.py), and you may want to check out [test/example.py](example.py) to see how this can be used. + + +## Versioning + +The initial version of the siplib in this repo is version 0.4 + +Because the siplib is strongly coupled to the SIP xsd, the version numbering of the siplib will match with the version of the xsd schema. +The latest version of the SIP schema is **2.8.3**. -**Versioning** +### Note about version compatibility +Initially this code was part of the [RO LOFAR code repository](https://git.astron.nl/ro/lofar), +but was split off for the purpose of the LDV-project in 2021. +At that moment version 0.4 was used as library version, but the LTA-SIP xsd version was already 2.7.1. +For COBALT2 updates the LTA model was changed and so also the LTA-SIP xsd version to 2.8.0. +Upon that version the library version should match with the LTA-SIP xsd version using tags. +Unfortunately after tag 2.8.0 was created (10 May 2022), new SIP-lib packages were built and created +in the [repository](https://git.astron.nl/ro/siputils/-/packages) with version 2.7.1 in its name. +These packages contain actually the LTA-SIP xsd version 2.8 -The initial version of the siblib in this repo is version 0.4 +### Note about the pyxb python package +The SIB-lib make use of pyxb-style bindings. The original pxpy python libray is already obsolete since 2016! +An alternative is available. [PyXB-X](https://github.com/renalreg/PyXB-X) is a 'fork' of pyxb which is updated for latest python versions and were legacy code is removed. +Therefore, this library will be used **since siplib version 2.8.3**. +This means also that the required python version for the SIP-lib will also shift to at least python3.9 and in near future 3.12 (end 2025). -Because the siplib is strongly coupled to the SIP xsd, the version numbering of the siplib will match with the version of the xsd schema which is 2.8.1 now. \ No newline at end of file diff --git a/lib/etc/LTA-SIP.xsd b/lib/etc/LTA-SIP.xsd index cd6618a..67102c7 100644 --- a/lib/etc/LTA-SIP.xsd +++ b/lib/etc/LTA-SIP.xsd @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.8.1" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema"> +<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.8.3" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema"> <xs:annotation> <xs:documentation> XML Schema for data model Submission Information Package LOFAR Long Term Archive diff --git a/lib/ltasip.py b/lib/ltasip.py index c7ad2b7..eff2f4d 100644 --- a/lib/ltasip.py +++ b/lib/ltasip.py @@ -34,7 +34,7 @@ DEFAULT_SIP_XSD_PATH = os.path.join(os.path.dirname(__file__), "etc/LTA-SIP.xsd" _GenerationUID = pyxb.utils.utility.UniqueIdentifier('urn:uuid:df82b81a-cbb6-11ec-9c1b-9364d05a68d6') # Version of PyXB used to generate the bindings -_PyXBVersion = '1.2.5' +_PyXBVersion = '1.2.6' # Generated bindings are not compatible across PyXB versions if pyxb.__version__ != _PyXBVersion: raise pyxb.PyXBVersionError(_PyXBVersion) diff --git a/lib/query.py b/lib/query.py index 4d01f61..e779393 100644 --- a/lib/query.py +++ b/lib/query.py @@ -9,7 +9,6 @@ import xmlrpc.client from configparser import ConfigParser import re import logging -import pkg_resources def _hide_password(message): @@ -172,6 +171,10 @@ def get_unique_id(source, userlabel): @lta_on_error_print def get_dataproduct_sip(projectname, dataproductid): + """ + This function calls the LTA which then calls MOM. Since MOM is obsolete this function should not be used any more + If you specific need MOM-data, use explict the MOM-interface!! + """ response = client.GetSip(projectname, dataproductid) error = response.get('error') if error: @@ -181,6 +184,10 @@ def get_dataproduct_sip(projectname, dataproductid): @lta_on_error_print def get_dataproduct_ids(projectname, sasid): + """ + This function calls the LTA which then calls MOM. Since MOM is obsolete this function should not be used any more + If you specific need MOM-data, use explict the MOM-interface!! + """ response = client.GetDataProductIDS(projectname, sasid) error = response.get('error') if error: diff --git a/lib/validator.py b/lib/validator.py index d926789..2de7787 100644 --- a/lib/validator.py +++ b/lib/validator.py @@ -1,7 +1,6 @@ -import pkg_resources from lxml import etree import sys -from . import ltasip, siplib +from . import siplib import logging import os LOGLEVEL = logging.DEBUG @@ -9,12 +8,11 @@ LOGLEVEL = logging.DEBUG logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=LOGLEVEL, stream=sys.stderr) logger = logging.getLogger(__name__) -#DEFAULT_SIP_XSD_PATH = pkg_resources.resource_string(__name__, "etc/LTA-SIP.xsd") DEFAULT_SIP_XSD_PATH = os.path.join(os.path.dirname(__file__), "etc/LTA-SIP.xsd") def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH): - '''validates given xml file against given xsd file''' + """validates given xml file against given xsd file""" logger.info("validating %s against %s" % (xmlpath, xsdpath)) @@ -38,11 +36,11 @@ def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH): def check_consistency(sip): - ''' + """ since a SIP can be valid XML but make no sense structurally, this makes sure that SIP contents contain a tree structure that ends with a single described dataproduct, and all pipelines and intermediary dataproducts are properly chained together. - ''' + """ sip = sip._get_pyxb_sip(suppress_warning=True) linkstodataproduct = {} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5578cfb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "wheel", "setuptools-git-versioning<=1.8.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "siputils" +version = "2.8.3" +authors = [ + { name = "SDC Rainbow Team", email = "sdc-helpdesk@astron.nl" } +] +description = "LTA sip utils" +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "pyxb-x", + "lxml", + "requests", + "graphviz", +] +urls = { homepage = "https://git.astron.nl/ro/siputils" } + +[project.optional-dependencies] +tests = ["pytest"] + + +[tool.setuptools] +packages = ["lofar_lta_sip"] + +[tool.setuptools-git-versioning] +template = "{tag}" +dev-template = "{tag}.dev{ccount}+git.{sha}" +dirty-template = "{tag}.dev{ccount}+git.{sha}.dirty" +starting-version = "0.0.1" +version-file = "version" +count-commits-from-version-file = true + +[tool.setuptools.package-data] +lofar_lta_sip = ["etc/*.xsd", "etc/.siplibrc", "StationPositions.parset"] + +[tool.setuptools.package-dir] +lofar_lta_sip = "lib" + + diff --git a/setup.py b/setup.py index 72fff28..adcfd28 100644 --- a/setup.py +++ b/setup.py @@ -1,51 +1,54 @@ import setuptools -import os -import glob -if os.path.exists('README.md'): - with open("README.md", "r") as fh: - long_description = fh.read() -else: - long_description = '' -scripts = glob.glob('bin/*') +setuptools.setup() -setuptools.setup( - name="siputils", - version="2.8.1", - setuptools_git_versioning={ - "template": "{tag}", - "dev_template": "{tag}.dev{ccount}+git.{sha}", - "dirty_template": "{tag}.dev{ccount}+git.{sha}.dirty", - "starting_version": "0.0.1", - "version_file": 'version', - "count_commits_from_version_file": True - }, - author="Mattia Mancini", - author_email="mancini@astron.nl", - description="LTA sip utils", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://git.astron.nl/ro/siputils", - packages=['lofar_lta_sip'], - package_dir={'lofar_lta_sip': 'lib'}, - package_data = {'lofar_lta_sip': ['etc/*.xsd','etc/.siplibrc','StationPositions.parset']}, - include_package_data=False, - scripts=scripts, - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache2 License", - "Operating System :: OS Independent", - ], - python_requires='>=3.6', - install_requires = [ - 'pyxb==1.2.5', - 'lxml', - 'requests', - 'graphviz' - ], - tests_require=['pytest'], - setup_requires = [ - 'setuptools-git-versioning<=1.8.1' - ], - test_suite='test', -) +import os +import glob +# if os.path.exists('README.md'): +# with open("README.md", "r") as fh: +# long_description = fh.read() +# else: +# long_description = '' +# +# scripts = glob.glob('bin/*') +# +# setuptools.setup( +# name="siputils", +# version="2.8.1", +# setuptools_git_versioning={ +# "template": "{tag}", +# "dev_template": "{tag}.dev{ccount}+git.{sha}", +# "dirty_template": "{tag}.dev{ccount}+git.{sha}.dirty", +# "starting_version": "0.0.1", +# "version_file": 'version', +# "count_commits_from_version_file": True +# }, +# author="Mattia Mancini", +# author_email="mancini@astron.nl", +# description="LTA sip utils", +# long_description=long_description, +# long_description_content_type="text/markdown", +# url="https://git.astron.nl/ro/siputils", +# packages=['lofar_lta_sip'], +# package_dir={'lofar_lta_sip': 'lib'}, +# package_data = {'lofar_lta_sip': ['etc/*.xsd','etc/.siplibrc','StationPositions.parset']}, +# include_package_data=False, +# scripts=scripts, +# classifiers=[ +# "Programming Language :: Python :: 3", +# "License :: OSI Approved :: Apache2 License", +# "Operating System :: OS Independent", +# ], +# python_requires='>=3.6', +# install_requires = [ +# 'pyxb==1.2.5', +# 'lxml', +# 'requests', +# 'graphviz' +# ], +# tests_require=['pytest'], +# setup_requires = [ +# 'setuptools-git-versioning<=1.8.1' +# ], +# test_suite='test', +# ) diff --git a/tox.ini b/tox.ini index ecfc3a3..0c8018e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = py37 +envlist = py312 + [testenv] # install testing framework # ... or install anything else you might need here @@ -8,3 +9,13 @@ deps = pytest # ... or run any other command line tool you need to run here commands = pytest + +[testenv:coverage] +commands = + {envpython} --version + pytest -v --cov + # disregard tests and __init__ from result + coverage report --omit="tests/*,*/__init__.py" +deps = pytest + pytest-cov + coverage -- GitLab