From af4d48663d1dad1a2f8a6921a6e3a24e7bb19b9c Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Fri, 21 May 2021 14:29:20 +0200
Subject: [PATCH] Fix tests and setup.py
---
.gitignore | 2 ++
.gitlab-ci.yml | 48 ++++++++++++++++++++++++++++++++++++
bin/validatesip | 12 ++++++---
{etc => lib/etc}/LTA-SIP.xsd | 0
lib/validator.py | 31 +++++++++++------------
setup.py | 26 ++++++++++++++++---
test/__init__.py | 0
tox.ini | 9 +++++++
version | 1 +
9 files changed, 106 insertions(+), 23 deletions(-)
create mode 100644 .gitlab-ci.yml
rename {etc => lib/etc}/LTA-SIP.xsd (100%)
create mode 100644 test/__init__.py
create mode 100644 tox.ini
create mode 100644 version
diff --git a/.gitignore b/.gitignore
index ef643eb..11657fc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,5 @@ dist
*.egg*
env
venv
+**/__pycache__/
+.tox
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..ca9b098
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,48 @@
+# This file is a template, and might need editing before it works on your project.
+# Official language image. Look for the different tagged releases at:
+# https://hub.docker.com/r/library/python/tags/
+image: python:latest
+stages:
+ - test
+ - build
+ - publish
+
+# Change pip's cache directory to be inside the project directory since we can
+# only cache local items.
+variables:
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+cache:
+ paths:
+ - .cache/pip
+ - venv/
+
+before_script:
+ - python -V # Print out python version for debugging
+ - pip install virtualenv
+ - virtualenv venv
+ - source venv/bin/activate
+ - pip install twine
+ - pip install -r requirements.txt
+
+test:
+ stage: test
+ script:
+ - python setup.py test
+
+build:
+ stage: build
+ script:
+ - python setup.py bdist_wheel
+
+upload:
+ stage: publish
+ retry: 2
+ allow_failure: true
+ script:
+ - python setup.py sdist bdist_wheel
+ - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url https://git.astron.nl/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
+
+ artifacts:
+ paths:
+ - dist/*.whl
diff --git a/bin/validatesip b/bin/validatesip
index f0ab905..079c597 100644
--- a/bin/validatesip
+++ b/bin/validatesip
@@ -1,8 +1,14 @@
#!/usr/bin/env python3
-
+from argparse import ArgumentParser
from lofar.lta.sip.validator import main
-import sys
+
+
+def parse_args():
+ parser = ArgumentParser(description='validates sip over the xsd')
+ parser.add_argument('sip', help='xml sip to validate')
+ return parser.parse_args()
if __name__ == '__main__':
- main(sys.argv[1])
+ args = parse_args()
+ main(args.sip)
diff --git a/etc/LTA-SIP.xsd b/lib/etc/LTA-SIP.xsd
similarity index 100%
rename from etc/LTA-SIP.xsd
rename to lib/etc/LTA-SIP.xsd
diff --git a/lib/validator.py b/lib/validator.py
index 4d7f30f..6301ad0 100644
--- a/lib/validator.py
+++ b/lib/validator.py
@@ -1,12 +1,10 @@
-
+import pkg_resources
from lxml import etree
-import os
+
from . import ltasip
-d = os.path.dirname(os.path.realpath(__file__))
-XSDPATH = d+"/LTA-SIP.xsd"
+DEFAULT_SIP_XSD_PATH = pkg_resources.resource_string(__name__, "etc/LTA-SIP.xsd")
-DEFAULT_SIP_XSD_PATH = os.path.join(os.environ.get('LOFARROOT', '/opt/lofar'), 'etc', 'lta', 'LTA-SIP.xsd')
def validate(xmlpath, xsdpath=DEFAULT_SIP_XSD_PATH):
'''validates given xml file against given xsd file'''
@@ -44,27 +42,26 @@ def check_consistency(xmlpath):
xml = f.read()
sip = ltasip.CreateFromDocument(xml)
-
linkstodataproduct = {}
linkstoprocess = {}
# the dataproduct that is described by the sip
- data_out = sip.dataProduct
+ data_out = sip.dataProduct
id_out = str(data_out.dataProductIdentifier.identifier)
id_process = str(data_out.processIdentifier.identifier)
- linkstodataproduct.setdefault(id_out,[]).append(id_process)
+ linkstodataproduct.setdefault(id_out, []).append(id_process)
# the input / intermediate dataproducts
for data_in in sip.relatedDataProduct:
id_in = str(data_in.dataProductIdentifier.identifier)
id_process = str(data_in.processIdentifier.identifier)
- linkstodataproduct.setdefault(id_in,[]).append(id_process)
+ linkstodataproduct.setdefault(id_in, []).append(id_process)
# the observations
for obs in sip.observation:
id_obs = str(obs.observationId.identifier)
id_process = str(obs.processIdentifier.identifier)
- linkstoprocess.setdefault(id_process,[])
+ linkstoprocess.setdefault(id_process, [])
# the data processing steps
for pipe in sip.pipelineRun:
@@ -72,13 +69,12 @@ def check_consistency(xmlpath):
id_in = []
for id in pipe.sourceData.content():
id_in.append(str(id.identifier))
- linkstoprocess.setdefault(id_pipe,[]).append(id_in)
+ linkstoprocess.setdefault(id_pipe, []).append(id_in)
# the data processing steps
for unspec in sip.unspecifiedProcess:
id_unspec = str(unspec.processIdentifier.identifier)
- linkstoprocess.setdefault(id_unspec,[])
-
+ linkstoprocess.setdefault(id_unspec, [])
# todo: online processing
# todo: parsets (?)
@@ -86,16 +82,19 @@ def check_consistency(xmlpath):
for id in linkstodataproduct:
for id_from in linkstodataproduct.get(id):
if not id_from in linkstoprocess:
- raise Exception("The pipeline or observation that created dataproduct '"+ id + "' seems to be missing! -> ", id_from)
+ raise Exception(
+ "The pipeline or observation that created dataproduct '" + id + "' seems to be missing! -> ",
+ id_from)
for id in linkstoprocess:
for ids_from in linkstoprocess.get(id):
for id_from in ids_from:
if not id_from in linkstodataproduct:
- raise Exception("The input dataproduct for pipeline '"+ id +"' seems to be missing! -> ", id_from)
+ raise Exception("The input dataproduct for pipeline '" + id + "' seems to be missing! -> ",
+ id_from)
print("General SIP structure seems ok!")
- return True # already raised Exception if there was a problem...
+ return True # already raised Exception if there was a problem...
def main(xml):
diff --git a/setup.py b/setup.py
index 63960bd..f3cac69 100644
--- a/setup.py
+++ b/setup.py
@@ -11,15 +11,26 @@ scripts = glob.glob('bin/*')
setuptools.setup(
name="siputils",
- version="0.0.1",
+ version="0.4",
+ version_config={
+ "template": "{tag}",
+ "dev_template": "{tag}.dev{ccount}+git.{sha}",
+ "dirty_template": "{tag}.dev{ccount}+git.{sha}.dirty",
+ "starting_version": "0.0.1",
+ "version_callback": None,
+ "version_file": 'version',
+ "count_commits_from_version_file": True
+ },
author="Mattia Mancini",
author_email="mancini@astron.nl",
description="LTA sip utils",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://git.astron.nl/ro/siputils",
- packages=setuptools.find_packages(),
- data_files = [('etc/lta/', ['etc/LTA-SIP.xsd'])],
+ packages=['lofar.lta.sip'],
+ package_dir={'lofar.lta.sip': 'lib'},
+ package_data = {'lofar.lta.sip': ['etc/*.xsd']},
+ include_package_data=False,
scripts=scripts,
classifiers=[
"Programming Language :: Python :: 3",
@@ -28,7 +39,14 @@ setuptools.setup(
],
python_requires='>=3.6',
install_requires = [
- 'pyxb',
+ 'pyxb==1.2.5',
+ 'lxml',
+ 'requests',
+ 'graphviz'
+ ],
+ tests_require=['pytest'],
+ setup_requires = [
+ 'setuptools-git-versioning'
],
test_suite='test',
)
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..d0d7c91
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,9 @@
+[tox]
+envlist = py37
+[testenv]
+# install testing framework
+# ... or install anything else you might need here
+deps = pytest
+# run the tests
+# ... or run any other command line tool you need to run here
+commands = pytest
diff --git a/version b/version
new file mode 100644
index 0000000..bd73f47
--- /dev/null
+++ b/version
@@ -0,0 +1 @@
+0.4
--
GitLab