From d930e5872a4b8c3a83415d7d3e6e020e7bc5e612 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Corn=C3=A9=20Lukken?= <lukken@astron.nl>
Date: Wed, 18 Dec 2024 10:22:51 +0000
Subject: [PATCH] TMSS-3170: Move things around
---
.gitlab-ci.yml | 44 ++-------
README.md | 17 +---
docker/ci-runner/Dockerfile | 12 ++-
docker/pycommon/Dockerfile | 10 +-
docs/cleanup.py | 23 -----
docs/requirements.txt | 5 -
docs/source/conf.py | 93 -------------------
docs/source/index.rst | 16 ----
docs/source/readme.rst | 2 -
docs/source/static/css/custom.css | 14 ---
pycommon/CMakeLists.txt | 51 ----------
pycommon/__init__.py | 23 +----
pycommon/cache.py | 26 +-----
pycommon/cep4_utils.py | 41 ++------
pycommon/cobaltblocksize.py | 6 +-
pycommon/database.py | 29 ++----
pycommon/datetimeutils.py | 24 +----
pycommon/dbcredentials | 3 -
pycommon/dbcredentials.py | 72 +-------------
pycommon/defaultmailaddresses.py | 25 +----
.../entrypoints/__init__.py | 0
pycommon/entrypoints/cep4_info.py | 21 +++++
pycommon/entrypoints/cobaltblocksize.py | 53 +++++++++++
pycommon/entrypoints/dbcredentials.py | 63 +++++++++++++
pycommon/entrypoints/oracle.py | 21 +++++
pycommon/factory.py | 23 +----
pycommon/flask_utils.py | 23 +----
pycommon/h5_utils.py | 26 ++----
pycommon/json_utils.py | 26 ++----
pycommon/lcu_utils.py | 43 ++-------
pycommon/locking.py | 4 +-
pycommon/math.py | 3 +
pycommon/methodtrigger.py | 6 +-
pycommon/metrics.py | 21 +----
pycommon/oracle.py | 47 ++--------
pycommon/parameterset.py | 22 +----
pycommon/postgres.py | 39 +++-----
pycommon/ring_coordinates.py | 6 +-
pycommon/ssh_utils.py | 20 +---
pycommon/station_coordinates.py | 3 +
pycommon/subprocess_utils.py | 4 +
pycommon/test_utils.py | 25 +----
pycommon/threading_utils.py | 26 +-----
pycommon/toposort.py | 25 +----
pycommon/typing.py | 19 +---
pycommon/util.py | 26 +-----
pycommon/xmlparse.py | 10 +-
requirements.txt | 10 +-
setup.cfg | 16 +++-
tests/CMakeLists.txt | 45 ---------
tests/__init__.py | 4 +-
tests/dbcredentials.py | 29 ++----
tests/postgres.py | 44 +++------
tests/python-coverage.sh.in | 57 ------------
tests/requirements.txt | 1 +
tests/t_cache.sh | 2 -
tests/t_cep4_utils.run | 23 -----
tests/t_cep4_utils.sh | 20 ----
tests/t_cobaltblocksize.run | 4 -
tests/t_cobaltblocksize.sh | 3 -
tests/t_dbcredentials.run | 4 -
tests/t_dbcredentials.sh | 2 -
tests/t_defaultmailaddresses.run | 4 -
tests/t_defaultmailaddresses.sh | 2 -
tests/t_json_utils.run | 5 -
tests/t_json_utils.sh | 2 -
tests/t_methodtrigger.sh | 2 -
tests/t_parameterset.run | 4 -
tests/t_parameterset.sh | 2 -
tests/t_postgres.run | 4 -
tests/t_postgres.sh | 2 -
tests/t_test_utils.run | 4 -
tests/t_test_utils.sh | 2 -
tests/t_toposort.sh | 2 -
tests/t_typing.run | 5 -
tests/t_typing.sh | 2 -
tests/t_util.run | 4 -
tests/t_util.sh | 2 -
tests/{t_cache.py => test_cache.py} | 19 ++--
tests/{t_cep4_utils.py => test_cep4_utils.py} | 36 ++-----
...ltblocksize.py => test_cobaltblocksize.py} | 15 +--
tests/test_cool_module.py | 16 ----
...dbcredentials.py => test_dbcredentials.py} | 23 ++---
...resses.py => test_defaultmailaddresses.py} | 24 ++---
tests/{t_json_utils.py => test_json_utils.py} | 44 +++------
...methodtrigger.py => test_methodtrigger.py} | 25 ++---
...t_parameterset.py => test_parameterset.py} | 2 +-
tests/{t_postgres.py => test_postgres.py} | 29 +++---
tests/{t_test_utils.py => test_test_utils.py} | 19 +---
tests/{t_toposort.py => test_toposort.py} | 19 ++--
tests/{t_typing.py => test_typing.py} | 30 ++----
tests/{t_util.py => test_util.py} | 32 +++----
tox.ini | 19 +---
93 files changed, 489 insertions(+), 1316 deletions(-)
delete mode 100644 docs/cleanup.py
delete mode 100644 docs/requirements.txt
delete mode 100644 docs/source/conf.py
delete mode 100644 docs/source/index.rst
delete mode 100644 docs/source/readme.rst
delete mode 100644 docs/source/static/css/custom.css
delete mode 100644 pycommon/CMakeLists.txt
delete mode 100755 pycommon/dbcredentials
rename tests/.gitkeep => pycommon/entrypoints/__init__.py (100%)
create mode 100644 pycommon/entrypoints/cep4_info.py
create mode 100644 pycommon/entrypoints/cobaltblocksize.py
create mode 100644 pycommon/entrypoints/dbcredentials.py
create mode 100644 pycommon/entrypoints/oracle.py
delete mode 100644 tests/CMakeLists.txt
delete mode 100755 tests/python-coverage.sh.in
delete mode 100755 tests/t_cache.sh
delete mode 100755 tests/t_cep4_utils.run
delete mode 100755 tests/t_cep4_utils.sh
delete mode 100755 tests/t_cobaltblocksize.run
delete mode 100755 tests/t_cobaltblocksize.sh
delete mode 100755 tests/t_dbcredentials.run
delete mode 100755 tests/t_dbcredentials.sh
delete mode 100755 tests/t_defaultmailaddresses.run
delete mode 100755 tests/t_defaultmailaddresses.sh
delete mode 100755 tests/t_json_utils.run
delete mode 100755 tests/t_json_utils.sh
delete mode 100755 tests/t_methodtrigger.sh
delete mode 100755 tests/t_parameterset.run
delete mode 100755 tests/t_parameterset.sh
delete mode 100755 tests/t_postgres.run
delete mode 100755 tests/t_postgres.sh
delete mode 100755 tests/t_test_utils.run
delete mode 100755 tests/t_test_utils.sh
delete mode 100755 tests/t_toposort.sh
delete mode 100755 tests/t_typing.run
delete mode 100755 tests/t_typing.sh
delete mode 100755 tests/t_util.run
delete mode 100755 tests/t_util.sh
rename tests/{t_cache.py => test_cache.py} (90%)
rename tests/{t_cep4_utils.py => test_cep4_utils.py} (76%)
rename tests/{t_cobaltblocksize.py => test_cobaltblocksize.py} (93%)
delete mode 100644 tests/test_cool_module.py
rename tests/{t_dbcredentials.py => test_dbcredentials.py} (91%)
rename tests/{t_defaultmailaddresses.py => test_defaultmailaddresses.py} (81%)
rename tests/{t_json_utils.py => test_json_utils.py} (94%)
rename tests/{t_methodtrigger.py => test_methodtrigger.py} (90%)
rename tests/{t_parameterset.py => test_parameterset.py} (97%)
rename tests/{t_postgres.py => test_postgres.py} (93%)
rename tests/{t_test_utils.py => test_test_utils.py} (88%)
rename tests/{t_toposort.py => test_toposort.py} (96%)
rename tests/{t_typing.py => test_typing.py} (81%)
rename tests/{t_util.py => test_util.py} (76%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a54e00..d4d4a68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,7 @@
default:
- image: $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG
+ image:
+ name: $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG
+ pull_policy: always
before_script:
- python --version # For debugging
cache:
@@ -10,8 +12,6 @@ default:
stages:
- prepare
- lint
- # check if this needs to be a separate step
- # - build_extensions
- test
- package
- images
@@ -52,11 +52,6 @@ run_pylint:
- tox -e pylint
allow_failure: true
-# build_extensions:
-# stage: build_extensions
-# script:
-# - echo "build fortran/c/cpp extension source code"
-
sast:
variables:
SAST_EXCLUDED_ANALYZERS: brakeman, flawfinder, kubesec, nodejs-scan, phpcs-security-audit,
@@ -76,6 +71,10 @@ secret_detection:
# Basic setup for all Python versions for which we don't have a base image
.run_unit_test_version_base:
before_script:
+ - apt update
+ - apt install -y postgresql
+ - ln -sf /usr/lib/postgresql/15/bin/initdb /usr/bin/initdb
+ - useradd -m tests
- python --version # For debugging
- python -m pip install --upgrade pip
- python -m pip install --upgrade tox twine
@@ -86,10 +85,10 @@ run_unit_tests:
stage: test
image: python:3.${PY_VERSION}
script:
- - tox -e py3${PY_VERSION}
+ - su tests -c "tox -e py3${PY_VERSION}"
parallel:
matrix: # use the matrix for testing
- - PY_VERSION: [8, 9, 10, 11]
+ - PY_VERSION: [10, 11, 12]
# Run code coverage on the base image thus also performing unit tests
run_unit_tests_coverage:
@@ -114,15 +113,6 @@ package_files:
script:
- tox -e build
-package_docs:
- stage: package
- artifacts:
- expire_in: 1w
- paths:
- - docs/build/*
- script:
- - tox -e docs
-
docker_build:
stage: images
image: docker:latest
@@ -134,8 +124,7 @@ docker_build:
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build -f docker/pycommon/Dockerfile . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/pycommon:$CI_COMMIT_REF_SLUG
- # enable this push line once you have configured docker registry cleanup policy
- # - docker push $CI_REGISTRY_IMAGE/pycommon:$CI_COMMIT_REF_SLUG
+ - docker push $CI_REGISTRY_IMAGE/pycommon:$CI_COMMIT_REF_SLUG
run_integration_tests:
stage: integration
@@ -200,19 +189,6 @@ publish_on_pypi:
# --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/*
- exit 1
-publish_to_readthedocs:
- stage: publish
- allow_failure: true
- environment: readthedocs
- needs:
- - package_docs
- when: manual
- rules:
- - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"'
- script:
- - echo "scp docs/* ???"
- - exit 1
-
release_job:
stage: publish
image: registry.gitlab.com/gitlab-org/release-cli:latest
diff --git a/README.md b/README.md
index e2d390d..e975581 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,10 @@
# PyCommon
-
-
-<!--  -->
+
+
+
-An example repository of an CI/CD pipeline for building, testing and publishing a python package.
+Common Python library for TMSS backend
## Installation
```
@@ -22,13 +22,6 @@ the CI/CD pipelines. And you can consider creating protected version tags for so
Once the cleanup policy for docker registry is setup you can uncomment the `docker push` comment in the `.gitlab-ci.yml`
file from the `docker_build` job. This will allow to download minimal docker images with your Python package installed.
-## Usage
-```python
-from pycommon import cool_module
-
-cool_module.greeter() # prints "Hello World"
-```
-
## Contributing
To contribute, please create a feature branch and a "Draft" merge request.
@@ -50,4 +43,4 @@ To automatically apply most suggested linting changes execute:
```tox -e format```
## License
-This project is licensed under the Apache License Version 2.0
+This project is licensed under GPLv3
diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile
index 6cb4643..874c123 100644
--- a/docker/ci-runner/Dockerfile
+++ b/docker/ci-runner/Dockerfile
@@ -1,4 +1,14 @@
-FROM python:3.12
+FROM python:3.13
+
+RUN --mount=type=cache,target=/var/cache/apt \
+ apt-get update
+RUN --mount=type=cache,target=/var/cache/apt \
+ apt-get install -y postgresql
+RUN ln -sf /usr/lib/postgresql/15/bin/initdb /usr/bin/initdb
RUN python -m pip install --upgrade pip
RUN python -m pip install --upgrade tox twine
+
+RUN useradd -m tests
+
+USER tests
diff --git a/docker/pycommon/Dockerfile b/docker/pycommon/Dockerfile
index ceea7e7..4b7fe6d 100644
--- a/docker/pycommon/Dockerfile
+++ b/docker/pycommon/Dockerfile
@@ -1,6 +1,6 @@
ARG BUILD_ENV=no_copy
-FROM python:3.11 AS build_no_copy
+FROM python:3.13 AS build_no_copy
ADD ../../requirements.txt .
COPY ../.. /work
RUN rm -r /work/dist | true
@@ -8,11 +8,15 @@ RUN python -m pip install --user tox
WORKDIR /work
RUN python -m tox -e build
-FROM python:3.11 AS build_copy
+FROM python:3.13 AS build_copy
COPY dist /work/dist
FROM build_${BUILD_ENV} AS build
-FROM python:3.11-slim
+FROM python:3.13-slim
COPY --from=build /work/dist /dist
+RUN --mount=type=cache,target=/var/cache/apt \
+ apt-get update
+RUN --mount=type=cache,target=/var/cache/apt \
+ apt-get install -y postgresql build-essential libpq-dev
RUN python -m pip install /dist/*.whl
diff --git a/docs/cleanup.py b/docs/cleanup.py
deleted file mode 100644
index 3a4508d..0000000
--- a/docs/cleanup.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-
-file_dir = os.path.dirname(os.path.realpath(__file__))
-
-clean_dir = os.path.join(file_dir, "source", "source_documentation")
-print(f"Cleaning.. {clean_dir}/*")
-
-if not os.path.exists(clean_dir):
- exit()
-
-for file_name in os.listdir(clean_dir):
- file = os.path.join(clean_dir, file_name)
-
- if file_name == "index.rst":
- continue
-
- print(f"Removing.. {file}")
- os.remove(file)
diff --git a/docs/requirements.txt b/docs/requirements.txt
deleted file mode 100644
index 3c6e46c..0000000
--- a/docs/requirements.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-sphinx!=1.6.6,!=1.6.7,>=1.6.5 # BSD
-sphinx-rtd-theme>=0.4.3 #MIT
-sphinxcontrib-apidoc>=0.3.0 #BSD
-myst-parser>=2.0 # MIT
-docutils>=0.17 # BSD
diff --git a/docs/source/conf.py b/docs/source/conf.py
deleted file mode 100644
index a0e3d33..0000000
--- a/docs/source/conf.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-
-from pycommon import __version__
-
-# -- General configuration ----------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [
- "sphinx.ext.autodoc",
- "sphinx.ext.viewcode",
- "sphinxcontrib.apidoc",
- "sphinx_rtd_theme",
- "myst_parser"
-]
-
-# Assumes tox is used to call sphinx-build
-project_root_directory = os.getcwd()
-
-apidoc_module_dir = "../../pycommon"
-apidoc_output_dir = "source_documentation"
-apidoc_excluded_paths = []
-apidoc_separate_modules = True
-apidoc_toc_file = False
-# This should include private methods but does not work
-# https://github.com/sphinx-contrib/apidoc/issues/14
-apidoc_extra_args = ["--private"]
-
-# The suffix of source filenames.
-source_suffix = [".rst"]
-
-# The master toctree document.
-master_doc = "index"
-
-# General information about the project.
-project = "PyCommon"
-copyright = "2023, ASTRON"
-
-# openstackdocstheme options
-repository_name = "git.astron.nl/tmss/libraries/pycommon"
-bug_project = "none"
-bug_tag = ""
-html_last_updated_fmt = "%Y-%m-%d %H:%M"
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-add_function_parentheses = True
-
-version = __version__
-
-modindex_common_prefix = ["pycommon."]
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-add_module_names = True
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# -- Options for HTML output --------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. Major themes that come with
-# Sphinx are currently 'default' and 'sphinxdoc'.
-# html_theme_path = ["."]
-html_theme = "sphinx_rtd_theme"
-html_static_path = ["static"]
-html_css_files = [
- "css/custom.css",
-]
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = "%sdoc" % project
-
-# Conf.py variables exported to sphinx rst files access using |NAME|
-variables_to_export = [
- "project",
- "copyright",
- "version",
-]
-
-# Write to rst_epilog to export `variables_to_export` extract using `locals()`
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-rst_epilog
-frozen_locals = dict(locals())
-rst_epilog = "\n".join(
- map(
- lambda x: f".. |{x}| replace:: {frozen_locals[x]}", # noqa: F821
- variables_to_export,
- )
-)
-# Pep is not able to determine that frozen_locals always exists so noqa
-del frozen_locals
diff --git a/docs/source/index.rst b/docs/source/index.rst
deleted file mode 100644
index a1d9e87..0000000
--- a/docs/source/index.rst
+++ /dev/null
@@ -1,16 +0,0 @@
-====================================================
-Welcome to the documentation of PyCommon
-====================================================
-
-..
- To define more variables see rst_epilog generation in conf.py
-
-Documentation for version: |version|
-
-Contents:
-
-.. toctree::
- :maxdepth: 2
-
- readme
- source_documentation/index
diff --git a/docs/source/readme.rst b/docs/source/readme.rst
deleted file mode 100644
index 87c96de..0000000
--- a/docs/source/readme.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-.. include:: ../../README.md
- :parser: myst_parser.sphinx_
diff --git a/docs/source/static/css/custom.css b/docs/source/static/css/custom.css
deleted file mode 100644
index 3ea8a2f..0000000
--- a/docs/source/static/css/custom.css
+++ /dev/null
@@ -1,14 +0,0 @@
-.orange { color: #c65d09; }
-
-.green { color: #5dc609; }
-
-.yellow { color: #c6c609; }
-
-.bolditalic {
- font-weight: bold;
- font-style: italic;
-}
-
-.rst-content code, .rst-content tt, code {
- white-space: break-spaces;
-}
diff --git a/pycommon/CMakeLists.txt b/pycommon/CMakeLists.txt
deleted file mode 100644
index a680686..0000000
--- a/pycommon/CMakeLists.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-# $Id: CMakeLists.txt 720 2014-12-08 16:29:33Z loose $
-
-lofar_package(PyCommon 1.0)
-
-lofar_find_package(Python 3.4 REQUIRED)
-include(PythonInstall)
-
-include(FindPythonModule)
-find_python_module(jsonschema)
-find_python_module(psycopg2)
-find_python_module(cx_Oracle)
-find_python_module(lxml)
-find_python_module(prometheus_client)
-
-set(_py_files
- __init__.py
- ssh_utils.py
- cep4_utils.py
- cobaltblocksize.py
- threading_utils.py
- lcu_utils.py
- cache.py
- dbcredentials.py
- defaultmailaddresses.py
- factory.py
- math.py
- methodtrigger.py
- metrics.py
- util.py
- database.py
- oracle.py
- postgres.py
- datetimeutils.py
- flask_utils.py
- h5_utils.py
- subprocess_utils.py
- xmlparse.py
- json_utils.py
- locking.py
- test_utils.py
- typing.py
- toposort.py
- ring_coordinates.py
- station_coordinates.py
- parameterset.py)
-
-python_install(${_py_files} DESTINATION lofar/common)
-
-lofar_add_bin_scripts(dbcredentials)
-
-add_subdirectory(test)
diff --git a/pycommon/__init__.py b/pycommon/__init__.py
index 0453289..8fc7d9b 100644
--- a/pycommon/__init__.py
+++ b/pycommon/__init__.py
@@ -1,24 +1,5 @@
-# __init__.py: Module initialization file.
-#
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: __init__.py 1568 2015-09-18 15:21:11Z loose $
+# Copyright (C) 2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
"""
Module initialization file.
diff --git a/pycommon/cache.py b/pycommon/cache.py
index 514bf53..9f12a70 100644
--- a/pycommon/cache.py
+++ b/pycommon/cache.py
@@ -1,29 +1,11 @@
-# cache.py: function return value cache
-#
-# copyright (c) 2015
-# astron (netherlands institute for radio astronomy)
-# p.o.box 2, 7990 aa dwingeloo, the netherlands
-#
-# this file is part of the lofar software suite.
-# the lofar software suite is free software: you can redistribute it
-# and/or modify it under the terms of the gnu general public license as
-# published by the free software foundation, either version 3 of the
-# license, or (at your option) any later version.
-#
-# the lofar software suite is distributed in the hope that it will be
-# useful, but without any warranty; without even the implied warranty of
-# merchantability or fitness for a particular purpose. see the
-# gnu general public license for more details.
-#
-# you should have received a copy of the gnu general public license along
-# with the lofar software suite. if not, see <http://www.gnu.org/licenses/>.
-#
-# $id: __init__.py 1568 2015-09-18 15:21:11z loose $
+# Copyright (C) 2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
import functools
+
class cache(object):
- """ A simple cache for function call return values in Python 2.
+ """ A simple cache for function call return values in Python 2.
Use:
diff --git a/pycommon/cep4_utils.py b/pycommon/cep4_utils.py
index f2dfd20..8caf0c9 100755
--- a/pycommon/cep4_utils.py
+++ b/pycommon/cep4_utils.py
@@ -1,30 +1,16 @@
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-from lofar.common.ssh_utils import ssh_cmd_list
-from subprocess import Popen, PIPE
-from lofar.common.subprocess_utils import check_output_returning_strings, communicate_returning_strings, execute_in_parallel
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import logging
from random import randint
import math
import os
-from time import sleep
-from datetime import datetime, timedelta
-import logging
+from pycommon.ssh_utils import ssh_cmd_list
+from pycommon.subprocess_utils import (
+ check_output_returning_strings, execute_in_parallel
+)
+
logger = logging.getLogger(__name__)
# a selection of slurm states relevant for lofar usage
@@ -402,12 +388,3 @@ def parallelize_cmd_over_cep4_cpu_nodes(cmd, parallelizable_option, parallelizab
logger.error('%s/%s parallelized cmds finished with errors', len(failed_results), num_workers)
return success
-
-if __name__ == '__main__':
- logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
- logger.info(convert_slurm_nodes_string_to_node_number_list(' \t cpu[20-39,41,45-48] '))
- logger.info(convert_slurm_nodes_string_to_node_number_list(' \t cpu03 '))
- logger.info(get_cep4_available_nodes())
- logger.info(get_cep4_available_nodes_sorted_ascending_by_load(min_nr_of_nodes=3, partition=SLURM_CPU_PARTITION))
- logger.info(get_cep4_available_nodes_sorted_ascending_by_load(min_nr_of_nodes=3, partition=SLURM_GPU_PARTITION))
-
diff --git a/pycommon/cobaltblocksize.py b/pycommon/cobaltblocksize.py
index 20bf2a7..3ec0e89 100755
--- a/pycommon/cobaltblocksize.py
+++ b/pycommon/cobaltblocksize.py
@@ -1,3 +1,6 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
"""
Code to derive the following parset input parameters for Cobalt. These keys need to be tuned
specifically to make sure all Cobalt processing fits inside a block. Only two processing
@@ -26,7 +29,8 @@
"""
from math import ceil
-from lofar.common.math import lcm
+from pycommon.math import lcm
+
class CorrelatorSettings(object):
""" Settings for the Correlator. """
diff --git a/pycommon/database.py b/pycommon/database.py
index eceb46a..f8c8aa4 100644
--- a/pycommon/database.py
+++ b/pycommon/database.py
@@ -1,23 +1,5 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
'''
common abstract database connection class
@@ -25,11 +7,11 @@ common abstract database connection class
import logging
from datetime import datetime, timedelta
-import collections
import time
import re
-from lofar.common.util import single_line_with_single_spaces
-from lofar.common.dbcredentials import DBCredentials
+
+from pycommon.util import single_line_with_single_spaces
+from pycommon.dbcredentials import DBCredentials
logger = logging.getLogger(__name__)
@@ -37,6 +19,7 @@ FETCH_NONE=0
FETCH_ONE=1
FETCH_ALL=2
+
class DatabaseError(Exception):
pass
diff --git a/pycommon/datetimeutils.py b/pycommon/datetimeutils.py
index fb1df87..73a29df 100644
--- a/pycommon/datetimeutils.py
+++ b/pycommon/datetimeutils.py
@@ -1,27 +1,7 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from datetime import datetime, timedelta
-import sys
-import os
def monthRanges(min_date, max_date, month_step=1):
diff --git a/pycommon/dbcredentials b/pycommon/dbcredentials
deleted file mode 100755
index df39203..0000000
--- a/pycommon/dbcredentials
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-# Just forward to the __main__ of the module
-exec python3 -m lofar.common.dbcredentials "$@"
diff --git a/pycommon/dbcredentials.py b/pycommon/dbcredentials.py
index 172e578..3656e8e 100644
--- a/pycommon/dbcredentials.py
+++ b/pycommon/dbcredentials.py
@@ -1,30 +1,12 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from glob import glob
import os
import pwd
from configparser import ConfigParser, NoSectionError, DuplicateSectionError
from optparse import OptionGroup
-from os import stat, path, chmod
+from os import stat, chmod
import logging
logger = logging.getLogger(__name__)
@@ -323,51 +305,3 @@ def parse_options(options, filepatterns=None):
if options.dbName: creds.database = options.dbName
return creds
-
-
-if __name__ == "__main__":
- import sys
- from optparse import OptionParser
-
- parser = OptionParser("%prog [options]")
- parser.add_option("-D", "--database", dest="database", type="string", default="",
- help="Print credentials of a specific database")
- parser.add_option("-S", "--shell", dest="shell", action="store_true", default=False,
- help="Use machine-readable output for use in shell scripts")
- parser.add_option("-L", "--list", dest="list", action="store_true", default=False,
- help="List known databases")
- parser.add_option("-F", "--files", dest="files", action="store_true", default=False,
- help="List names of parsed configuration files")
- (options, args) = parser.parse_args()
-
- if not options.database and not options.list and not options.files:
- logger.error("Missing database name")
- parser.print_help()
- sys.exit(1)
-
- dbc = DBCredentials()
-
- if options.files:
- """ Print list of configuration files that we've read. """
- if dbc.files:
- logger.info("\n".join(dbc.files))
- sys.exit(0)
-
- if options.list:
- """ Print list of databases. """
- databases = dbc.list()
- if databases:
- logger.info("\n".join(databases))
- sys.exit(0)
-
- """ Print credentials of a specific database. """
- creds = dbc.get(options.database)
-
- if options.shell:
- print("DBUSER=%s" % (creds.user,))
- print("DBPASSWORD=%s" % (creds.password,))
- print("DBDATABASE=%s" % (creds.database,))
- print("DBHOST=%s" % (creds.host,))
- print("DBPORT=%s" % (creds.port,))
- else:
- logger.info(str(creds))
diff --git a/pycommon/defaultmailaddresses.py b/pycommon/defaultmailaddresses.py
index f23e6f6..8eb25f7 100644
--- a/pycommon/defaultmailaddresses.py
+++ b/pycommon/defaultmailaddresses.py
@@ -1,25 +1,6 @@
-# defaultmailaddresses.py: default mail addresses for the LOFAR software
-#
-# Copyright (C) 2017
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: $
-#
+# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
"""
This package contains the default mail addresses used in the LOFAR software
"""
diff --git a/tests/.gitkeep b/pycommon/entrypoints/__init__.py
similarity index 100%
rename from tests/.gitkeep
rename to pycommon/entrypoints/__init__.py
diff --git a/pycommon/entrypoints/cep4_info.py b/pycommon/entrypoints/cep4_info.py
new file mode 100644
index 0000000..e8c272d
--- /dev/null
+++ b/pycommon/entrypoints/cep4_info.py
@@ -0,0 +1,21 @@
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import logging
+
+from cep4_utils import (
+ convert_slurm_nodes_string_to_node_number_list,
+ get_cep4_available_nodes, get_cep4_available_nodes_sorted_ascending_by_load,
+ SLURM_GPU_PARTITION, SLURM_CPU_PARTITION
+)
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+ logger.info(convert_slurm_nodes_string_to_node_number_list(' \t cpu[20-39,41,45-48] '))
+ logger.info(convert_slurm_nodes_string_to_node_number_list(' \t cpu03 '))
+ logger.info(get_cep4_available_nodes())
+ logger.info(get_cep4_available_nodes_sorted_ascending_by_load(min_nr_of_nodes=3, partition=SLURM_CPU_PARTITION))
+ logger.info(get_cep4_available_nodes_sorted_ascending_by_load(min_nr_of_nodes=3, partition=SLURM_GPU_PARTITION))
diff --git a/pycommon/entrypoints/cobaltblocksize.py b/pycommon/entrypoints/cobaltblocksize.py
new file mode 100644
index 0000000..f416881
--- /dev/null
+++ b/pycommon/entrypoints/cobaltblocksize.py
@@ -0,0 +1,53 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import argparse
+import sys
+
+from pycommon.cobaltblocksize import (
+ CorrelatorSettings, StokesSettings, BlockConstraints, BlockSize
+)
+
+
+def main():
+ parser = argparse.ArgumentParser(description='Compute COBALT block sizes, based on the given constraints.')
+ parser.add_argument('-c', '--correlate', dest='correlate', action='store_true', default=False, help='enable the correlator')
+ parser.add_argument('--correlator_nrchannels', type=int, default=64, help='Correlator.nrChannelsPerSubband (default: %(default)s)')
+ parser.add_argument('--correlator_integration_time', type=float, default=1.0, help='Correlator.integrationTime (default: %(default)s)')
+
+ parser.add_argument('--coherent', dest='coherent', action='store_true', default=False, help='enable coherent beamforming')
+ parser.add_argument('--incoherent', dest='incoherent', action='store_true', default=False, help='enable incoherent beamforming')
+
+ parser.add_argument('--coherent_nrchannels', type=int, default=16, help='CoherentStokes.nrChannelsPerSubband (default: %(default)s)')
+ parser.add_argument('--coherent_time_integration_factor', type=int, default=1, help='CoherentStokes.timeIntegrationFactor (default: %(default)s)')
+
+ parser.add_argument('--incoherent_nrchannels', type=int, default=16, help='IncoherentStokes.nrChannelsPerSubband (default: %(default)s)')
+ parser.add_argument('--incoherent_time_integration_factor', type=int, default=1, help='IncoherentStokes.timeIntegrationFactor (default: %(default)s)')
+
+ args = parser.parse_args()
+
+ if not args.correlate and not args.coherent and not args.incoherent:
+ parser.print_help()
+ sys.exit(1)
+
+ corr = CorrelatorSettings()
+ corr.nrChannelsPerSubband = args.correlator_nrchannels
+ corr.integrationTime = args.correlator_integration_time
+
+ coh = StokesSettings()
+ coh.nrChannelsPerSubband = args.coherent_nrchannels
+ coh.timeIntegrationFactor = args.coherent_time_integration_factor
+
+ incoh = StokesSettings()
+ incoh.nrChannelsPerSubband = args.incoherent_nrchannels
+ incoh.timeIntegrationFactor = args.incoherent_time_integration_factor
+
+ constraints = BlockConstraints(correlatorSettings=corr if args.correlate else None,
+ coherentStokesSettings=[coh] if args.coherent else [],
+ incoherentStokesSettings=[incoh] if args.incoherent else [])
+
+ # will throw if the blocksize cannot be computed
+ blocksize = BlockSize(constraints)
+
+ # print optimal block size
+ print(blocksize.blockSize)
diff --git a/pycommon/entrypoints/dbcredentials.py b/pycommon/entrypoints/dbcredentials.py
new file mode 100644
index 0000000..20d0d0a
--- /dev/null
+++ b/pycommon/entrypoints/dbcredentials.py
@@ -0,0 +1,63 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import sys
+from optparse import OptionParser
+import logging
+
+from pycommon.dbcredentials import DBCredentials
+
+logger = logging.getLogger()
+
+
+def main():
+ parser = OptionParser("%prog [options]")
+ parser.add_option(
+ "-D", "--database", dest="database", type="string", default="",
+ help="Print credentials of a specific database"
+ )
+ parser.add_option(
+ "-S", "--shell", dest="shell", action="store_true", default=False,
+ help="Use machine-readable output for use in shell scripts"
+ )
+ parser.add_option(
+ "-L", "--list", dest="list", action="store_true", default=False,
+ help="List known databases"
+ )
+ parser.add_option(
+ "-F", "--files", dest="files", action="store_true", default=False,
+ help="List names of parsed configuration files"
+ )
+ (options, args) = parser.parse_args()
+
+ if not options.database and not options.list and not options.files:
+ logger.error("Missing database name")
+ parser.print_help()
+ sys.exit(1)
+
+ dbc = DBCredentials()
+
+ if options.files:
+ """ Print list of configuration files that we've read. """
+ if dbc.files:
+ logger.info("\n".join(dbc.files))
+ sys.exit(0)
+
+ if options.list:
+ """ Print list of databases. """
+ databases = dbc.list()
+ if databases:
+ logger.info("\n".join(databases))
+ sys.exit(0)
+
+ """ Print credentials of a specific database. """
+ creds = dbc.get(options.database)
+
+ if options.shell:
+ print("DBUSER=%s" % (creds.user,))
+ print("DBPASSWORD=%s" % (creds.password,))
+ print("DBDATABASE=%s" % (creds.database,))
+ print("DBHOST=%s" % (creds.host,))
+ print("DBPORT=%s" % (creds.port,))
+ else:
+ logger.info(str(creds))
diff --git a/pycommon/entrypoints/oracle.py b/pycommon/entrypoints/oracle.py
new file mode 100644
index 0000000..15525fe
--- /dev/null
+++ b/pycommon/entrypoints/oracle.py
@@ -0,0 +1,21 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import logging
+
+from pycommon.database import FETCH_ALL
+from pycommon.dbcredentials import DBCredentials
+from pycommon.oracle import OracleDatabaseConnection
+
+
+def main():
+ logging.basicConfig(format = '%(asctime)s %(levelname)s %(message)s', level = logging.INFO)
+
+ dbcreds = DBCredentials().get('LTA')
+ print(dbcreds.stringWithHiddenPassword())
+
+ with OracleDatabaseConnection(dbcreds=dbcreds) as db:
+ from pprint import pprint
+ pprint(db.executeQuery("SELECT table_name, owner, tablespace_name FROM all_tables", fetch=FETCH_ALL))
+ # pprint(db.executeQuery("SELECT * FROM awoper.aweprojects", fetch=FETCH_ALL))
+ #pprint(db.executeQuery("SELECT * FROM awoper.aweprojectusers", fetch=FETCH_ALL))
diff --git a/pycommon/factory.py b/pycommon/factory.py
index a80c613..448789f 100644
--- a/pycommon/factory.py
+++ b/pycommon/factory.py
@@ -1,24 +1,5 @@
-# factory.py: Generic object factory.
-#
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: factory.py 1584 2015-10-02 12:10:14Z loose $
+# Copyright (C) 2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
"""
Module for a generic object factory.
diff --git a/pycommon/flask_utils.py b/pycommon/flask_utils.py
index c254c6a..f31a0d6 100644
--- a/pycommon/flask_utils.py
+++ b/pycommon/flask_utils.py
@@ -1,29 +1,10 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from flask import after_this_request, request
from io import BytesIO as IO
import gzip
import functools
-from datetime import datetime
def gzipped(f):
@functools.wraps(f)
diff --git a/pycommon/h5_utils.py b/pycommon/h5_utils.py
index 7baa7cd..62d7daf 100644
--- a/pycommon/h5_utils.py
+++ b/pycommon/h5_utils.py
@@ -1,33 +1,21 @@
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+import logging
import os.path
from datetime import datetime, timedelta
from time import sleep
import errno
import os
-# prevent annoying h5py future/deprecation warnings
-os.environ["TF_CPP_MIN_LOG_LEVEL"]="3"
import h5py
-import logging
+# # prevent annoying h5py future/deprecation warnings
+# os.environ["TF_CPP_MIN_LOG_LEVEL"]="3"
+
logger = logging.getLogger(__name__)
+
class SharedH5File():
"""
Wrapper class aroung h5py.File to open an hdf5 file in read, write, or read/write mode safely,
diff --git a/pycommon/json_utils.py b/pycommon/json_utils.py
index 9e0d4b5..38a2849 100644
--- a/pycommon/json_utils.py
+++ b/pycommon/json_utils.py
@@ -1,30 +1,18 @@
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+from copy import deepcopy
+from datetime import datetime, timedelta
import json
import logging
import time
import typing
import jsonschema
-from copy import deepcopy
import requests
-from datetime import datetime, timedelta
-from .util import single_line_with_single_spaces
+
+from pycommon.util import single_line_with_single_spaces
+
class JSONError(Exception):
pass
diff --git a/pycommon/lcu_utils.py b/pycommon/lcu_utils.py
index c0a17b1..9c15b05 100755
--- a/pycommon/lcu_utils.py
+++ b/pycommon/lcu_utils.py
@@ -1,32 +1,20 @@
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-from lofar.common.ssh_utils import ssh_cmd_list
-from lofar.common.subprocess_utils import execute_in_parallel, wrap_composite_command, communicate_returning_strings
-from subprocess import Popen, PIPE
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+import logging
+from subprocess import Popen, PIPE
import os
import uuid
import struct
-from datetime import datetime, timedelta
-import logging
+from pycommon.ssh_utils import ssh_cmd_list
+from pycommon.subprocess_utils import (
+ execute_in_parallel, wrap_composite_command, communicate_returning_strings
+)
+
logger = logging.getLogger(__name__)
+
class LCURuntimeError(RuntimeError):
pass
@@ -449,14 +437,3 @@ def parse_station_calibration_file(filename):
# return tuple of header and complex table in result dict
return (header_dict, complexdata)
-
-
-
-if __name__ == '__main__':
- logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG)
- import pprint
- # pprint.pprint(get_station_cable_delays(['CS004', 'CS005']))
- # print get_station_calibration_tables(['CS001', 'RS407'], antenna_set_and_filter='LBA_INNER-10_90') #['CS001', 'DE601'])
- #pprint.pprint(execute_in_parallel_over_stations(cmd=wrap_composite_command('sleep 1; date'),
- # stations=['cs026c' for i in range(10)]))
- pprint.pprint(execute_in_parallel_over_station_group(cmd=wrap_composite_command('sleep 1 ; date ; sleep 1 ;'), station_group='today_core'))
diff --git a/pycommon/locking.py b/pycommon/locking.py
index a8ff6d7..4fef843 100644
--- a/pycommon/locking.py
+++ b/pycommon/locking.py
@@ -8,7 +8,6 @@
NamedAtomicLock - A Named atomic lock local to the machine
'''
-# vim: set ts=4 sw=4 expandtab :
import os
@@ -31,6 +30,7 @@ try:
except:
FileNotFoundError = OSError
+
class NamedAtomicLock(object):
def __init__(self, name, lockDir=None, maxLockAge=None):
@@ -241,7 +241,7 @@ class NamedAtomicLock(object):
# If we don't hold it currently, return False
if self.held is False:
return False
-
+
# Otherwise if we think we hold it, but it is not held, we have lost it.
if not self.isHeld:
self.acquiredAt = None
diff --git a/pycommon/math.py b/pycommon/math.py
index 3dd0b35..8558d43 100644
--- a/pycommon/math.py
+++ b/pycommon/math.py
@@ -1,3 +1,6 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
try:
from math import gcd
except ImportError:
diff --git a/pycommon/methodtrigger.py b/pycommon/methodtrigger.py
index 338d150..09cfd93 100644
--- a/pycommon/methodtrigger.py
+++ b/pycommon/methodtrigger.py
@@ -1,11 +1,15 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
from threading import Lock, Condition
__all__ = ["MethodTrigger"]
+
class MethodTrigger:
"""
Set a flag when a specific method is called, possibly asynchronously. Caller can wait on this flag.
-
+
Example:
class Foo(object):
diff --git a/pycommon/metrics.py b/pycommon/metrics.py
index d3498ce..b11e65a 100644
--- a/pycommon/metrics.py
+++ b/pycommon/metrics.py
@@ -1,19 +1,5 @@
-# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from functools import wraps
import time
@@ -24,9 +10,10 @@ from prometheus_client.utils import INF
# Buckets that are more useful for tracking longer durations
LONG_DURATION_BUCKETS = (0.1, 0.2, 0.5, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, 1800.0, 3600.0, INF)
+
def _add_default_metrics():
"""Create generic metrics that we expose by default."""
-
+
uptime = Gauge("uptime", "Uptime of the service")
start_time = time.time()
uptime.set_function(lambda: time.time() - start_time)
diff --git a/pycommon/oracle.py b/pycommon/oracle.py
index de63802..0fe8396 100644
--- a/pycommon/oracle.py
+++ b/pycommon/oracle.py
@@ -1,35 +1,21 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
'''
Module with nice postgres helper methods and classes.
'''
import logging
-from datetime import datetime, timedelta
+from datetime import datetime
import cx_Oracle
import re
-from lofar.common.dbcredentials import DBCredentials
-from lofar.common.database import AbstractDatabaseConnection, DatabaseError, DatabaseConnectionError, DatabaseExecutionError, FETCH_NONE, FETCH_ONE, FETCH_ALL
-from lofar.common.util import single_line_with_single_spaces
+
+from pycommon.dbcredentials import DBCredentials
+from pycommon.database import (
+ AbstractDatabaseConnection, DatabaseError, DatabaseConnectionError,
+ DatabaseExecutionError, FETCH_NONE, FETCH_ONE, FETCH_ALL
+)
+from pycommon.util import single_line_with_single_spaces
logger = logging.getLogger(__name__)
@@ -134,16 +120,3 @@ class OracleDatabaseConnection(AbstractDatabaseConnection):
else:
# wrap original error in OracleDBQueryExecutionError
raise OracleDBQueryExecutionError("Could not execute query '%s' error=%s" % (query_log_line, error_string))
-
-
-if __name__ == '__main__':
- logging.basicConfig(format = '%(asctime)s %(levelname)s %(message)s', level = logging.INFO)
-
- dbcreds = DBCredentials().get('LTA')
- print(dbcreds.stringWithHiddenPassword())
-
- with OracleDatabaseConnection(dbcreds=dbcreds) as db:
- from pprint import pprint
- pprint(db.executeQuery("SELECT table_name, owner, tablespace_name FROM all_tables", fetch=FETCH_ALL))
- # pprint(db.executeQuery("SELECT * FROM awoper.aweprojects", fetch=FETCH_ALL))
- #pprint(db.executeQuery("SELECT * FROM awoper.aweprojectusers", fetch=FETCH_ALL))
diff --git a/pycommon/parameterset.py b/pycommon/parameterset.py
index b8ddb81..8805eb0 100644
--- a/pycommon/parameterset.py
+++ b/pycommon/parameterset.py
@@ -1,23 +1,5 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
"""Drop-in replacement for LCS/pyparameterset, which in turn wraps LCS/Common/src/ParameterSet.cc.
diff --git a/pycommon/postgres.py b/pycommon/postgres.py
index 4e5b2a0..15c64a8 100644
--- a/pycommon/postgres.py
+++ b/pycommon/postgres.py
@@ -1,23 +1,5 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
'''
Module with nice postgres helper methods and classes.
@@ -26,19 +8,22 @@ Module with nice postgres helper methods and classes.
import logging
from threading import Thread, Lock
from queue import Queue, Empty
-from datetime import datetime, timedelta
+from datetime import datetime
import collections
-import time
-from typing import Union
import re
import select
+
import psycopg2
import psycopg2.extras
import psycopg2.extensions
-from lofar.common.util import single_line_with_single_spaces
-from lofar.common.datetimeutils import totalSeconds
-from lofar.common.dbcredentials import DBCredentials
-from lofar.common.database import AbstractDatabaseConnection, DatabaseError, DatabaseConnectionError, DatabaseExecutionError, FETCH_NONE, FETCH_ONE, FETCH_ALL
+
+from pycommon.util import single_line_with_single_spaces
+from pycommon.datetimeutils import totalSeconds
+from pycommon.dbcredentials import DBCredentials
+from pycommon.database import (
+ AbstractDatabaseConnection, DatabaseError, DatabaseConnectionError,
+ DatabaseExecutionError, FETCH_NONE, FETCH_ONE, FETCH_ALL
+)
logger = logging.getLogger(__name__)
diff --git a/pycommon/ring_coordinates.py b/pycommon/ring_coordinates.py
index 454dddb..986786e 100755
--- a/pycommon/ring_coordinates.py
+++ b/pycommon/ring_coordinates.py
@@ -1,9 +1,7 @@
-#!/usr/bin/env python3
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
-import sys
from math import sqrt, cos, sin, pi, asin, atan2
-import subprocess
-import itertools
class RingCoordinates:
diff --git a/pycommon/ssh_utils.py b/pycommon/ssh_utils.py
index 4d44a63..1cec3a7 100644
--- a/pycommon/ssh_utils.py
+++ b/pycommon/ssh_utils.py
@@ -1,23 +1,11 @@
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
import logging
+
logger = logging.getLogger(__name__)
+
def ssh_cmd_list(host, user='lofarsys', disable_strict_host_key_checking: bool=False, disable_remote_pseudo_terminal: bool=False):
'''
returns a subprocess compliant command list to do an ssh call to the given node
diff --git a/pycommon/station_coordinates.py b/pycommon/station_coordinates.py
index a9b8bb9..0ea0b6d 100644
--- a/pycommon/station_coordinates.py
+++ b/pycommon/station_coordinates.py
@@ -1,3 +1,6 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
from ast import literal_eval
import os
import functools
diff --git a/pycommon/subprocess_utils.py b/pycommon/subprocess_utils.py
index 26d6f2a..cd28ce3 100644
--- a/pycommon/subprocess_utils.py
+++ b/pycommon/subprocess_utils.py
@@ -1,3 +1,6 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
import logging
from datetime import datetime, timedelta
from time import sleep
@@ -11,6 +14,7 @@ except ImportError:
logger = logging.getLogger(__name__)
+
class SubprocessTimoutError(TimeoutError):
'''an error class indication that the running subprocess to longer than expected to complete'''
pass
diff --git a/pycommon/test_utils.py b/pycommon/test_utils.py
index 408b4b8..cf07128 100644
--- a/pycommon/test_utils.py
+++ b/pycommon/test_utils.py
@@ -1,25 +1,6 @@
-# test_utils.py: test utils for lofar software
-#
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: test_utils.py 1584 2015-10-02 12:10:14Z loose $
-#
+# Copyright (C) 2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
"""
This package contains different utilities that are common for LOFAR software testing
"""
diff --git a/pycommon/threading_utils.py b/pycommon/threading_utils.py
index aeccb4f..edb2fe2 100644
--- a/pycommon/threading_utils.py
+++ b/pycommon/threading_utils.py
@@ -1,25 +1,6 @@
-# threading_utils.py: test utils for lofar software
-#
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id$
-#
+# Copyright (C) 2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
"""
This module contains various utilities/methods that are common for LOFAR threading usage
"""
@@ -30,6 +11,7 @@ import logging
logger = logging.getLogger(__name__)
+
class TimeoutLock:
"""
A TimeoutLock is a threading.RLock which you can use in a 'with' context in conjunction with a timeout.
diff --git a/pycommon/toposort.py b/pycommon/toposort.py
index 1c07a42..dab68e6 100644
--- a/pycommon/toposort.py
+++ b/pycommon/toposort.py
@@ -1,22 +1,5 @@
-# toposort.py: Topological Sort, to sort nodes of DAGs
-#
-# copyright (c) 2021
-# astron (netherlands institute for radio astronomy)
-# p.o.box 2, 7990 aa dwingeloo, the netherlands
-#
-# this file is part of the lofar software suite.
-# the lofar software suite is free software: you can redistribute it
-# and/or modify it under the terms of the gnu general public license as
-# published by the free software foundation, either version 3 of the
-# license, or (at your option) any later version.
-#
-# the lofar software suite is distributed in the hope that it will be
-# useful, but without any warranty; without even the implied warranty of
-# merchantability or fitness for a particular purpose. see the
-# gnu general public license for more details.
-#
-# you should have received a copy of the gnu general public license along
-# with the lofar software suite. if not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2021 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
class Graph:
""" Graph definition to use in the topological sort. """
@@ -68,11 +51,11 @@ class Graph:
def toposorted(graph: Graph) -> list:
""" Return the list of vertices, where, for each pair of vertices (a,b),
a appears earlier in the list than b if there is a path a -> b.
-
+
Regular sorting routines (like list.sort) fundamentally cannot do this,
as they demand a total ordering, that is, the comparison function
must yield the correct result for every pair of elements (a,b).
-
+
NOTE: Since a graph provides only a partial ordering, the order of
the output list can depend on the order of the vertices and edges in
the graph. For many graphs multiple solutions are valid, after all. """
diff --git a/pycommon/typing.py b/pycommon/typing.py
index cd154ec..040fe3b 100644
--- a/pycommon/typing.py
+++ b/pycommon/typing.py
@@ -1,23 +1,10 @@
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from functools import wraps
import inspect
+
def check_type_hints(func):
""" Decorator that verifies the type hints of the decorated function.
diff --git a/pycommon/util.py b/pycommon/util.py
index b0f55ae..7af3cc9 100644
--- a/pycommon/util.py
+++ b/pycommon/util.py
@@ -1,25 +1,6 @@
-# util.py: utils for lofar software
-#
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: util.py 1584 2015-10-02 12:10:14Z loose $
-#
+# Copyright (C) 2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
"""
This package contains different utilities that are common for LOFAR software
"""
@@ -29,6 +10,7 @@ import os, os.path
import time
from copy import deepcopy
import logging
+
logger = logging.getLogger(__name__)
def check_bit(value, bit):
diff --git a/pycommon/xmlparse.py b/pycommon/xmlparse.py
index eef9466..1e53403 100644
--- a/pycommon/xmlparse.py
+++ b/pycommon/xmlparse.py
@@ -1,7 +1,11 @@
-from lxml import etree
-from io import BytesIO
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from typing import Union
+from io import BytesIO
+
+from lxml import etree
+
def parse_xml_string_or_bytestring(xml_content: Union[str, bytes], parser=None) -> etree:
"""
@@ -14,4 +18,4 @@ def parse_xml_string_or_bytestring(xml_content: Union[str, bytes], parser=None)
else:
bstr = xml_content
- return etree.parse(BytesIO(bstr), parser=parser)
\ No newline at end of file
+ return etree.parse(BytesIO(bstr), parser=parser)
diff --git a/requirements.txt b/requirements.txt
index 8f81bc2..fd7bffb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,8 @@
-importlib-metadata>=0.12, <5.0;python_version<"3.8"
-numpy
+psycopg2
+lxml
+jsonschema
+requests
+flask
+h5py
+prometheus_client
+cx_Oracle
diff --git a/setup.cfg b/setup.cfg
index 513b55d..f4d1501 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,9 +1,9 @@
[metadata]
name = pycommon
-description = An example package for CI/CD working group
+description = PyCommon package for TMSS backend
long_description = file: README.md
long_description_content_type = text/markdown
-url = https://git.astron.nl/templates/python-package
+url = https://git.astron.nl/tmss/libraries/pycommon
license = GNU General Public License v3.0
classifiers =
Development Status :: 5 - Production/Stable
@@ -15,20 +15,26 @@ classifiers =
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3 :: Only
- Programming Language :: Python :: 3.8
- Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
Programming Language :: Python :: 3.12
+ Programming Language :: Python :: 3.13
Topic :: Scientific/Engineering
Topic :: Scientific/Engineering :: Astronomy
[options]
include_package_data = true
packages = find:
-python_requires = >=3.8
+python_requires = >=3.10
install_requires = file: requirements.txt
+[options.entry_points]
+console_scripts =
+ cep4-info = pycommon.entrypoints.cep4_info:main
+ cobaltblocksize = pycommon.entrypoints.cobaltblocksize:main
+ dbcredentials = pycommon.entrypoints.dbcredentials:main
+ oracle = pycommon.entrypoints.oracle:main
+
[flake8]
max-line-length = 88
extend-ignore = E203
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
deleted file mode 100644
index fec40b3..0000000
--- a/tests/CMakeLists.txt
+++ /dev/null
@@ -1,45 +0,0 @@
-# $Id$
-
-IF(BUILD_TESTING)
- lofar_find_package(Python 3.4 REQUIRED)
-
- include(PythonInstall)
-
- set(_py_files
- __init__.py
- postgres.py
- dbcredentials.py)
-
- python_install(${_py_files} DESTINATION lofar/common/testing)
-
- include(FindPythonModule)
- find_python_module(testing.postgresql)
-
- include(LofarCTest)
-
- configure_file(
- "${CMAKE_CURRENT_SOURCE_DIR}/python-coverage.sh.in"
- "${CMAKE_BINARY_DIR}/bin/python-coverage.sh" # internal, no need to install
- )
-
- lofar_add_test(t_cache)
- lofar_add_test(t_cobaltblocksize)
- lofar_add_test(t_dbcredentials)
- lofar_add_test(t_defaultmailaddresses)
- lofar_add_test(t_methodtrigger)
- lofar_add_test(t_parameterset)
- lofar_add_test(t_util)
- lofar_add_test(t_test_utils)
- lofar_add_test(t_cep4_utils)
- lofar_add_test(t_toposort)
- lofar_add_test(t_typing)
-
- IF(PYTHON_JSONSCHEMA)
- lofar_add_test(t_json_utils)
- ENDIF()
-
- IF(PYTHON_PSYCOPG2 AND PYTHON_TESTING.POSTGRESQL)
- lofar_add_test(t_postgres)
- ENDIF()
-
-ENDIF()
diff --git a/tests/__init__.py b/tests/__init__.py
index 05ca219..7a842a4 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,5 +1,5 @@
-from lofar.common import isProductionEnvironment
+from pycommon import isProductionEnvironment
if isProductionEnvironment():
print("The test modules in %s should not be imported and used in a lofar production environment" % __package__)
- exit(1)
\ No newline at end of file
+ exit(1)
diff --git a/tests/dbcredentials.py b/tests/dbcredentials.py
index c2279f1..f3ba461 100755
--- a/tests/dbcredentials.py
+++ b/tests/dbcredentials.py
@@ -1,31 +1,14 @@
-#!/usr/bin/env python3
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
-import os, sys
+import os
import uuid
-
import logging
+from pycommon.dbcredentials import Credentials, DBCredentials
+
logger = logging.getLogger(__name__)
-from lofar.common.dbcredentials import Credentials, DBCredentials
class TemporaryCredentials():
''' A helper class which creates/destroys dbcredentials automatically.
@@ -98,4 +81,4 @@ class TemporaryCredentials():
except Exception as e:
logger.error("Could not remove temporary credentials file '%s': %s", self._dbcreds_path, e)
-__all__ = ['TemporaryCredentials']
\ No newline at end of file
+__all__ = ['TemporaryCredentials']
diff --git a/tests/postgres.py b/tests/postgres.py
index 627c1de..2589173 100755
--- a/tests/postgres.py
+++ b/tests/postgres.py
@@ -1,38 +1,22 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# $Id$
-import psycopg2
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import datetime, timedelta
import os, sys
import logging
-import uuid
-
-logger = logging.getLogger(__name__)
+import psycopg2
import testing.postgresql
-from lofar.common.dbcredentials import Credentials
-from lofar.common.postgres import PostgresDatabaseConnection
-from lofar.common.testing.dbcredentials import TemporaryCredentials
-from lofar.common.util import find_free_port
-from datetime import datetime, timedelta
-from lofar.common.locking import NamedAtomicLock
+from pycommon.dbcredentials import Credentials
+from pycommon.postgres import PostgresDatabaseConnection
+from pycommon.util import find_free_port
+from pycommon.locking import NamedAtomicLock
+
+from tests.dbcredentials import TemporaryCredentials
+
+logger = logging.getLogger(__name__)
+
class PostgresTestDatabaseInstance():
''' A helper class which instantiates a running postgres server (not interfering with any other test/production postgres servers)
diff --git a/tests/python-coverage.sh.in b/tests/python-coverage.sh.in
deleted file mode 100755
index 3c5e166..0000000
--- a/tests/python-coverage.sh.in
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-
-# Default lines to exclude in python-coverage
-COVERAGE_EXCLUDE_LINES="[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n"
-
-# Determine python-coverage executable
-if type "coverage3" >& /dev/null; then
- COVERAGE=coverage3
-elif type "python3-coverage" >& /dev/null; then
- COVERAGE=python3-coverage
-else
- COVERAGE=""
-fi
-
-#
-# Run a python test under python-coverage (if available).
-#
-# Usage:
-#
-# python_coverage_test module mytest.py [testarg1 testarg2 ...]
-#
-function python_coverage_test {
- PYTHON_MODULE=$1
- shift
-
- if [ "$SKIP_PYTHON_COVERAGE" ] ; then
- echo "Running python test without coverage because SKIP_PYTHON_COVERAGE=$SKIP_PYTHON_COVERAGE"
- #run plain test script
- exec @PYTHON_EXECUTABLE@ "$@"
- fi
-
- if [ -n "$COVERAGE" ]; then
- #run test using python python-coverage tool
-
- #erase previous results
- $COVERAGE erase
-
- #setup python-coverage config file
- RCFILE=`basename $0`.python-coveragerc
- printf "$COVERAGE_EXCLUDE_LINES" > $RCFILE
-
- $COVERAGE run --rcfile $RCFILE --branch "$@"
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code python-coverage results *** "
- $COVERAGE report -m
- echo " *** End python-coverage results *** "
- fi
- exit $RESULT
- else
- #python-coverage not available
- echo "Please run: 'pip3 install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- exec @PYTHON_EXECUTABLE@ "$@"
- fi
-}
-
diff --git a/tests/requirements.txt b/tests/requirements.txt
index f14d0b9..a32343a 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -5,3 +5,4 @@ flake8 >= 5.0.0 # MIT
pylint >= 2.15.0 # GPLv2
pytest >= 7.0.0 # MIT
pytest-cov >= 3.0.0 # MIT
+testing.postgresql >= 1.1 # Apache 2
diff --git a/tests/t_cache.sh b/tests/t_cache.sh
deleted file mode 100755
index 8b7a9ab..0000000
--- a/tests/t_cache.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_cache
diff --git a/tests/t_cep4_utils.run b/tests/t_cep4_utils.run
deleted file mode 100755
index dbbadd7..0000000
--- a/tests/t_cep4_utils.run
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-# Run the unit test
-source python-coverage.sh
-python_coverage_test "*cep4_utils*" t_cep4_utils.py
-
diff --git a/tests/t_cep4_utils.sh b/tests/t_cep4_utils.sh
deleted file mode 100755
index 9298df5..0000000
--- a/tests/t_cep4_utils.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-./runctest.sh t_cep4_utils
diff --git a/tests/t_cobaltblocksize.run b/tests/t_cobaltblocksize.run
deleted file mode 100755
index 4841812..0000000
--- a/tests/t_cobaltblocksize.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-source python-coverage.sh
-python_coverage_test "cobaltblocksize*" t_cobaltblocksize.py
diff --git a/tests/t_cobaltblocksize.sh b/tests/t_cobaltblocksize.sh
deleted file mode 100755
index de952a2..0000000
--- a/tests/t_cobaltblocksize.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-./runctest.sh t_cobaltblocksize
diff --git a/tests/t_dbcredentials.run b/tests/t_dbcredentials.run
deleted file mode 100755
index 9497e58..0000000
--- a/tests/t_dbcredentials.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-source python-coverage.sh
-
-python_coverage_test dbcredentials t_dbcredentials.py
diff --git a/tests/t_dbcredentials.sh b/tests/t_dbcredentials.sh
deleted file mode 100755
index 16dd2c8..0000000
--- a/tests/t_dbcredentials.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_dbcredentials
diff --git a/tests/t_defaultmailaddresses.run b/tests/t_defaultmailaddresses.run
deleted file mode 100755
index cad0a43..0000000
--- a/tests/t_defaultmailaddresses.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-source python-coverage.sh
-
-python_coverage_test defaultmailaddresses t_defaultmailaddresses.py
diff --git a/tests/t_defaultmailaddresses.sh b/tests/t_defaultmailaddresses.sh
deleted file mode 100755
index 498398c..0000000
--- a/tests/t_defaultmailaddresses.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_defaultmailaddresses
diff --git a/tests/t_json_utils.run b/tests/t_json_utils.run
deleted file mode 100755
index 4ef087e..0000000
--- a/tests/t_json_utils.run
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-source python-coverage.sh
-python_coverage_test "*json_utils*" t_json_utils.py
-
diff --git a/tests/t_json_utils.sh b/tests/t_json_utils.sh
deleted file mode 100755
index 3152fa0..0000000
--- a/tests/t_json_utils.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_json_utils
diff --git a/tests/t_methodtrigger.sh b/tests/t_methodtrigger.sh
deleted file mode 100755
index c786a2c..0000000
--- a/tests/t_methodtrigger.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_methodtrigger
diff --git a/tests/t_parameterset.run b/tests/t_parameterset.run
deleted file mode 100755
index b2be23d..0000000
--- a/tests/t_parameterset.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-source python-coverage.sh
-
-python_coverage_test parameterset t_parameterset.py
diff --git a/tests/t_parameterset.sh b/tests/t_parameterset.sh
deleted file mode 100755
index eb7763a..0000000
--- a/tests/t_parameterset.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_parameterset
diff --git a/tests/t_postgres.run b/tests/t_postgres.run
deleted file mode 100755
index 05c53f0..0000000
--- a/tests/t_postgres.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-source python-coverage.sh
-
-python_coverage_test postgres t_postgres.py
diff --git a/tests/t_postgres.sh b/tests/t_postgres.sh
deleted file mode 100755
index 34699f2..0000000
--- a/tests/t_postgres.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_postgres
diff --git a/tests/t_test_utils.run b/tests/t_test_utils.run
deleted file mode 100755
index f3e1ccf..0000000
--- a/tests/t_test_utils.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-source python-coverage.sh
-
-python_coverage_test test_utils t_test_utils.py
diff --git a/tests/t_test_utils.sh b/tests/t_test_utils.sh
deleted file mode 100755
index 70c850d..0000000
--- a/tests/t_test_utils.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_test_utils
diff --git a/tests/t_toposort.sh b/tests/t_toposort.sh
deleted file mode 100755
index 2730b24..0000000
--- a/tests/t_toposort.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_toposort
diff --git a/tests/t_typing.run b/tests/t_typing.run
deleted file mode 100755
index 6bc23fa..0000000
--- a/tests/t_typing.run
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-source python-coverage.sh
-python_coverage_test "*json_utils*" t_typing.py
-
diff --git a/tests/t_typing.sh b/tests/t_typing.sh
deleted file mode 100755
index d788f5a..0000000
--- a/tests/t_typing.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_typing
diff --git a/tests/t_util.run b/tests/t_util.run
deleted file mode 100755
index ba34891..0000000
--- a/tests/t_util.run
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-source python-coverage.sh
-
-python_coverage_test util t_util.py
diff --git a/tests/t_util.sh b/tests/t_util.sh
deleted file mode 100755
index 02ec676..0000000
--- a/tests/t_util.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh t_util
diff --git a/tests/t_cache.py b/tests/test_cache.py
similarity index 90%
rename from tests/t_cache.py
rename to tests/test_cache.py
index 2cf8ef0..6b5d619 100644
--- a/tests/t_cache.py
+++ b/tests/test_cache.py
@@ -1,8 +1,14 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
import unittest
-from lofar.common.cache import cache
-from lofar.common.test_utils import unit_test, exit_with_skipped_code_if_skip_unit_tests
+
+from pycommon.cache import cache
+from pycommon.test_utils import unit_test, exit_with_skipped_code_if_skip_unit_tests
+
exit_with_skipped_code_if_skip_unit_tests()
+
class TestCache(unittest.TestCase):
@cache
@@ -97,12 +103,3 @@ class TestCache(unittest.TestCase):
self.assertEqual(result, (1, 3))
self.assertEqual(self.invocations, 2)
-
-def main(argv):
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main(sys.argv[1:])
-
diff --git a/tests/t_cep4_utils.py b/tests/test_cep4_utils.py
similarity index 76%
rename from tests/t_cep4_utils.py
rename to tests/test_cep4_utils.py
index 5db7994..5d56202 100755
--- a/tests/t_cep4_utils.py
+++ b/tests/test_cep4_utils.py
@@ -1,34 +1,19 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
import unittest
from subprocess import call
-
import logging
-from lofar.common.cep4_utils import *
-from lofar.common.test_utils import integration_test
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_integration_tests
-exit_with_skipped_code_if_skip_integration_tests()
+from pycommon.cep4_utils import *
+from pycommon.test_utils import integration_test
+from pycommon.test_utils import exit_with_skipped_code_if_skip_integration_tests
+exit_with_skipped_code_if_skip_integration_tests()
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
+
@integration_test
class TestCep4Utils(unittest.TestCase):
@@ -104,8 +89,3 @@ class TestCep4Utils(unittest.TestCase):
#redirect stdout/stderr to /dev/null
with open('/dev/null', 'w') as dev_null:
self.assertEqual(0, call(cmd, stdout=dev_null, stderr=dev_null))
-
-if __name__ == '__main__':
- logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG)
-
- unittest.main()
diff --git a/tests/t_cobaltblocksize.py b/tests/test_cobaltblocksize.py
similarity index 93%
rename from tests/t_cobaltblocksize.py
rename to tests/test_cobaltblocksize.py
index 62931b9..33d9b89 100644
--- a/tests/t_cobaltblocksize.py
+++ b/tests/test_cobaltblocksize.py
@@ -1,11 +1,18 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
import unittest
-from lofar.common.cobaltblocksize import StokesSettings, CorrelatorSettings, BlockConstraints, BlockSize
import logging
-from lofar.common.test_utils import unit_test
+
+from pycommon.cobaltblocksize import (
+ StokesSettings, CorrelatorSettings, BlockConstraints, BlockSize
+)
+from pycommon.test_utils import unit_test
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)
+
class TestBlockConstraints(unittest.TestCase):
@unit_test
@@ -126,7 +133,3 @@ class TestBlockSize(unittest.TestCase):
# we're happy if a valid block size was found
self.assertNotEqual(bs.blockSize, None)
-
-
-if __name__ == "__main__":
- unittest.main(verbosity=2)
diff --git a/tests/test_cool_module.py b/tests/test_cool_module.py
deleted file mode 100644
index c347632..0000000
--- a/tests/test_cool_module.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-
-"""Testing of the Cool Module"""
-from unittest import TestCase
-
-from pycommon.cool_module import greeter
-
-
-class TestCoolModule(TestCase):
- """Test Case of the Cool Module"""
-
- def test_greeter(self):
- """Testing that the greeter does not crash"""
- greeter()
- self.assertEqual(2 + 2, 4)
diff --git a/tests/t_dbcredentials.py b/tests/test_dbcredentials.py
similarity index 91%
rename from tests/t_dbcredentials.py
rename to tests/test_dbcredentials.py
index d269365..52b3f84 100644
--- a/tests/t_dbcredentials.py
+++ b/tests/test_dbcredentials.py
@@ -1,15 +1,18 @@
-#!/usr/bin/env python3
-
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_unit_tests
-exit_with_skipped_code_if_skip_unit_tests()
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
import os
import unittest
import tempfile
from uuid import uuid4
-from lofar.common.dbcredentials import *
from configparser import NoSectionError
-from lofar.common.test_utils import unit_test
+
+from pycommon.test_utils import exit_with_skipped_code_if_skip_unit_tests
+from pycommon.test_utils import unit_test
+from pycommon.dbcredentials import *
+
+exit_with_skipped_code_if_skip_unit_tests()
+
def setUpModule():
pass
@@ -144,11 +147,3 @@ test = word word
# test if the free-form config options got through
self.assertEqual(c_out.config["foo"], "bar")
self.assertEqual(c_out.config["test"], "word word")
-
-def main(argv):
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main(sys.argv[1:])
diff --git a/tests/t_defaultmailaddresses.py b/tests/test_defaultmailaddresses.py
similarity index 81%
rename from tests/t_defaultmailaddresses.py
rename to tests/test_defaultmailaddresses.py
index 315cfb6..cc2b914 100644
--- a/tests/t_defaultmailaddresses.py
+++ b/tests/test_defaultmailaddresses.py
@@ -1,12 +1,15 @@
-#!/usr/bin/env python3
-
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_unit_tests
-exit_with_skipped_code_if_skip_unit_tests()
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
import unittest
import tempfile
-from lofar.common.defaultmailaddresses import PipelineEmailConfig
-from lofar.common.test_utils import unit_test
+
+from pycommon.test_utils import exit_with_skipped_code_if_skip_unit_tests
+from pycommon.test_utils import unit_test
+from pycommon.defaultmailaddresses import PipelineEmailConfig
+
+exit_with_skipped_code_if_skip_unit_tests()
+
def setUpModule():
pass
@@ -57,12 +60,3 @@ error-sender
pec = PipelineEmailConfig(filepatterns=[f.name])
with self.assertRaises(Exception):
print(pec["error-sender"])
-
-
-def main():
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main()
diff --git a/tests/t_json_utils.py b/tests/test_json_utils.py
similarity index 94%
rename from tests/t_json_utils.py
rename to tests/test_json_utils.py
index 4dbadee..72e257e 100755
--- a/tests/t_json_utils.py
+++ b/tests/test_json_utils.py
@@ -1,33 +1,21 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_unit_tests
-exit_with_skipped_code_if_skip_unit_tests()
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+import unittest
+import threading
+import json
import logging
+
+from pycommon.test_utils import exit_with_skipped_code_if_skip_unit_tests
+from pycommon.json_utils import (
+ get_default_json_object_for_schema, replace_host_in_urls, resolved_remote_refs,
+ resolved_local_refs, get_sub_schema, raise_on_self_refs
+)
+
+exit_with_skipped_code_if_skip_unit_tests()
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(process)s %(threadName)s %(levelname)s %(message)s', level=logging.DEBUG)
-import unittest
-import threading
-import json
-from lofar.common.json_utils import get_default_json_object_for_schema, replace_host_in_urls, resolved_remote_refs, resolved_local_refs, get_sub_schema, raise_on_self_refs, resolved_refs
class TestJSONUtils(unittest.TestCase):
def setUp(self) -> None:
@@ -241,7 +229,7 @@ class TestJSONUtils(unittest.TestCase):
'''test if $refs to URL's are properly resolved'''
import http.server
import socketserver
- from lofar.common.util import find_free_port
+ from pycommon.util import find_free_port
port = find_free_port(8000, allow_reuse_of_lingering_port=False)
host = "127.0.0.1"
@@ -407,7 +395,7 @@ class TestJSONUtils(unittest.TestCase):
'''test if ambiguous $refs raise'''
import http.server
import socketserver
- from lofar.common.util import find_free_port
+ from pycommon.util import find_free_port
port = find_free_port(8000, allow_reuse_of_lingering_port=False)
host = "127.0.0.1"
@@ -518,5 +506,3 @@ class TestJSONUtils(unittest.TestCase):
self.assertEqual("http://json-schema.org/draft-06/schema#", url_fixed_schema['$schema'])
self.assertEqual(json.dumps(schema, indent=2).replace(base_host, new_base_host), json.dumps(url_fixed_schema, indent=2))
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/t_methodtrigger.py b/tests/test_methodtrigger.py
similarity index 90%
rename from tests/t_methodtrigger.py
rename to tests/test_methodtrigger.py
index 064085e..ad085a8 100644
--- a/tests/t_methodtrigger.py
+++ b/tests/test_methodtrigger.py
@@ -1,12 +1,16 @@
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_unit_tests
-exit_with_skipped_code_if_skip_unit_tests()
-
-import unittest
-from lofar.common.methodtrigger import MethodTrigger
-from lofar.common.test_utils import unit_test
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
from threading import Thread
import time
+import unittest
+
+from pycommon.test_utils import exit_with_skipped_code_if_skip_unit_tests
+from pycommon.methodtrigger import MethodTrigger
+from pycommon.test_utils import unit_test
+
+exit_with_skipped_code_if_skip_unit_tests()
+
class TestMethodTrigger(unittest.TestCase):
def setUp(self):
@@ -124,12 +128,3 @@ class TestArgs(unittest.TestCase):
# Check stored arguments
self.assertEqual(self.trigger.args, (1, 2))
self.assertEqual(self.trigger.kwargs, {"c": 3, "d": 4})
-
-def main(argv):
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main(sys.argv[1:])
-
diff --git a/tests/t_parameterset.py b/tests/test_parameterset.py
similarity index 97%
rename from tests/t_parameterset.py
rename to tests/test_parameterset.py
index 40af91b..11cbd74 100644
--- a/tests/t_parameterset.py
+++ b/tests/test_parameterset.py
@@ -1,6 +1,6 @@
from unittest import TestCase
-from lofar.common.parameterset import parameterset
+from pycommon.parameterset import parameterset
class TestParameterSet(TestCase):
diff --git a/tests/t_postgres.py b/tests/test_postgres.py
similarity index 93%
rename from tests/t_postgres.py
rename to tests/test_postgres.py
index 91e298c..f67f0f0 100755
--- a/tests/t_postgres.py
+++ b/tests/test_postgres.py
@@ -1,20 +1,24 @@
-#!/usr/bin/env python3
-
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_integration_tests
-exit_with_skipped_code_if_skip_integration_tests()
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+import logging
+import signal
+from copy import deepcopy
import unittest
from unittest import mock
-from lofar.common.postgres import *
-from lofar.common.testing.postgres import PostgresTestDatabaseInstance, PostgresTestMixin
+
import psycopg2
-import signal
-from copy import deepcopy
-from lofar.common.test_utils import integration_test
-import logging
+from pycommon.test_utils import exit_with_skipped_code_if_skip_integration_tests
+from pycommon.postgres import *
+from pycommon.test_utils import integration_test
+from tests.postgres import PostgresTestDatabaseInstance, PostgresTestMixin
+
+exit_with_skipped_code_if_skip_integration_tests()
+
logger = logging.getLogger(__name__)
+
class MyPostgresTestDatabaseInstance(PostgresTestDatabaseInstance):
def apply_database_schema(self):
# use 'normal' psycopg2 API to connect and setup the database,
@@ -41,7 +45,7 @@ class TestPostgres(MyPostgresTestMixin, unittest.TestCase):
incorrect_dbcreds.port += 1
# test if connecting fails
- with mock.patch('lofar.common.database.logger') as mocked_logger:
+ with mock.patch('pycommon.database.logger') as mocked_logger:
with self.assertRaises(DatabaseConnectionError):
NUM_CONNECT_RETRIES = 2
with PostgresDatabaseConnection(dbcreds=incorrect_dbcreds, connect_retry_interval=0.1, num_connect_retries=NUM_CONNECT_RETRIES) as db:
@@ -119,6 +123,3 @@ class TestPostgres(MyPostgresTestMixin, unittest.TestCase):
logging.basicConfig(format='%(asctime)s %(process)s %(threadName)s %(levelname)s %(message)s', level=logging.DEBUG)
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/t_test_utils.py b/tests/test_test_utils.py
similarity index 88%
rename from tests/t_test_utils.py
rename to tests/test_test_utils.py
index eeabd72..5cf36a3 100644
--- a/tests/t_test_utils.py
+++ b/tests/test_test_utils.py
@@ -1,11 +1,9 @@
-#!/usr/bin/env python3
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_unit_tests
-exit_with_skipped_code_if_skip_unit_tests()
+from pycommon.test_utils import *
-import unittest
-import tempfile
-from lofar.common.test_utils import *
+exit_with_skipped_code_if_skip_unit_tests()
def setUpModule():
pass
@@ -67,12 +65,3 @@ class TestTestUtils(unittest.TestCase):
xml2 = '<xml><test att="2">test text</test></xml>'
with self.assertRaises(AssertionError):
assertEqualXML(xml1, xml2)
-
-
-def main(argv):
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main(sys.argv[1:])
diff --git a/tests/t_toposort.py b/tests/test_toposort.py
similarity index 96%
rename from tests/t_toposort.py
rename to tests/test_toposort.py
index d47841e..f31fd1d 100644
--- a/tests/t_toposort.py
+++ b/tests/test_toposort.py
@@ -1,6 +1,10 @@
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
import unittest
-from lofar.common.toposort import Graph, toposorted
-from lofar.common.test_utils import unit_test
+
+from pycommon.toposort import Graph, toposorted
+from pycommon.test_utils import unit_test
class TestGraph(unittest.TestCase):
@@ -153,7 +157,7 @@ class TestToposort(unittest.TestCase):
@unit_test
def test_disconnected_graph(self):
- # 5 -> 4 -> 3
+ # 5 -> 4 -> 3
# 2 -> 1 -> 0
g = Graph([0,1,2,3,4,5])
g.add_edge(5, 4)
@@ -203,12 +207,3 @@ class TestToposort(unittest.TestCase):
result = toposorted(g)
self.assertTopoSorted(g, result)
-
-def main(argv):
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main(sys.argv[1:])
-
diff --git a/tests/t_typing.py b/tests/test_typing.py
similarity index 81%
rename from tests/t_typing.py
rename to tests/test_typing.py
index b88641d..6323243 100755
--- a/tests/t_typing.py
+++ b/tests/test_typing.py
@@ -1,29 +1,14 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+# Copyright (C) 2012 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
+import unittest
import logging
+
+from pycommon.typing import check_type_hints
+
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(process)s %(threadName)s %(levelname)s %(message)s', level=logging.DEBUG)
-from lofar.common.typing import check_type_hints
-
-import unittest
class TestCheckTypeHints(unittest.TestCase):
def test_no_argument(self):
@@ -172,6 +157,3 @@ class TestCheckTypeHints(unittest.TestCase):
with self.assertRaises(TypeError):
myfunc(None)
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/t_util.py b/tests/test_util.py
similarity index 76%
rename from tests/t_util.py
rename to tests/test_util.py
index c54747c..41528e2 100644
--- a/tests/t_util.py
+++ b/tests/test_util.py
@@ -1,12 +1,14 @@
-#!/usr/bin/env python3
-
-from lofar.common.test_utils import exit_with_skipped_code_if_skip_unit_tests
-exit_with_skipped_code_if_skip_unit_tests()
+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: GPL-3.0-or-later
import unittest
-import tempfile
-from lofar.common.util import *
-from lofar.common.test_utils import unit_test
+
+from pycommon.test_utils import exit_with_skipped_code_if_skip_unit_tests
+from pycommon.test_utils import unit_test
+
+from pycommon.util import *
+
+exit_with_skipped_code_if_skip_unit_tests()
def setUpModule():
pass
@@ -38,11 +40,11 @@ class TestUtils(unittest.TestCase):
self.assertFalse(is_iterable(None))
def test_merge_nested_dicts(self):
- dict_a = {'a': 1,
+ dict_a = {'a': 1,
'b': {
- 'c': 3,
- 'd': [4, 5, 6]},
- 'e': [{'foo': 1},
+ 'c': 3,
+ 'd': [4, 5, 6]},
+ 'e': [{'foo': 1},
{'bar': 2}]}
dict_b = {'a': 2}
@@ -60,11 +62,3 @@ class TestUtils(unittest.TestCase):
dict_b = {'e': [{'foo': 2}, {'bar': 3}]}
merged = dict_with_overrides(dict_a, dict_b)
self.assertEqual({'a': 1, 'b': {'c': 3, 'd': [4, 5, 6]}, 'e': [{'foo': 2}, {'bar': 3}]}, merged)
-
-def main(argv):
- unittest.main()
-
-if __name__ == "__main__":
- # run all tests
- import sys
- main(sys.argv[1:])
diff --git a/tox.ini b/tox.ini
index f72b92a..23099c4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,9 @@
[tox]
# Generative environment list to test all supported Python versions
-envlist = py3{8,9,10,11,12},black,pep8,pylint
-minversion = 3.18.0
+envlist = py3{10,11,12,13}
+min_version = 4.3.3
+requires =
+ tox-ignore-env-name-mismatch ~= 0.2.0
[testenv]
usedevelop = True
@@ -9,8 +11,6 @@ package = wheel
wheel_build_env = .pkg
setenv =
- LANGUAGE=en_US
- LC_ALL=en_US.UTF-8
PYTHONWARNINGS=default::DeprecationWarning
deps =
-r{toxinidir}/requirements.txt
@@ -40,17 +40,6 @@ commands =
format: {envpython} -m autopep8 -v -aa --in-place --recursive tests
format: {envpython} -m black -v pycommon tests
-[testenv:docs]
-; unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise
-setenv =
-deps =
- -r{toxinidir}/requirements.txt
- -r{toxinidir}/docs/requirements.txt
-changedir = {toxinidir}
-commands =
- {envpython} docs/cleanup.py
- sphinx-build -b html docs/source docs/build/html
-
[testenv:build]
usedevelop = False
deps = build
--
GitLab