diff --git a/.editorconfig b/.editorconfig index 16beae29cc7f31303c709b4981421f395512d84b..a441dcac54e8933af5718db461f9c7d5d4be28bd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -146,3 +146,6 @@ ij_yaml_sequence_on_new_line = false ij_yaml_space_before_colon = false ij_yaml_spaces_within_braces = true ij_yaml_spaces_within_brackets = true + +[*.proto] +indent_size = 2 diff --git a/.gitignore b/.gitignore index 7dd0c18279c65434a8223147cd436487f8b979db..bb31fb866796bec93181de1066b2004656254e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,15 @@ dist/* *.egg-info *.pyc .tox +**/venv .coverage coverage.xml htmlcov/* build dist +proto/**/*.py +proto/**/*.pyi # Documentation docs/source/source_documentation diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9b381ec4420ebbf2856daa8f9777f6843de8e8c..df1319bb548a3a865bbd8cc046d302277dea8f49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,26 +35,10 @@ trigger_prepare: before_script: - python --version # For debugging -run_black: +run_lint: stage: lint - extends: .python_before - script: - - tox -e black - allow_failure: true - -run_flake8: - stage: lint - extends: .python_before script: - - tox -e pep8 - allow_failure: true - -run_pylint: - stage: lint - extends: .python_before - script: - - tox -e pylint - allow_failure: true + - tox -e lint sast: variables: @@ -78,7 +62,7 @@ run_unit_tests: - tox -e py3${PY_VERSION} parallel: matrix: # use the matrix for testing - - PY_VERSION: [12] + - PY_VERSION: [10, 11, 12, 13] # Run code coverage on the base image thus also performing unit tests run_unit_tests_coverage: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..018cfbb0da2e30e8267792cac1c6c618f71b695e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +default_stages: [pre-push] + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-toml + - id: detect-private-key + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.10.0 + hooks: + - id: shellcheck + - repo: local + hooks: + - id: tox-lint + name: tox-lint (local) + entry: tox + language: python + types: [file, python] + args: ["-e", "lint", "--"] diff --git a/bin/build_proto.py b/bin/build_proto.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1239487ab2cfd148589b0084cafda968749803 --- /dev/null +++ b/bin/build_proto.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 + +import os +import glob +from importlib import resources +from grpc_tools import protoc + +proto_dir = os.path.join(os.getcwd(), "proto") +print(f"Compiling proto files in dir {proto_dir}...") + +if not os.path.exists(proto_dir): + exit(-1) + +files = glob.glob('./**/*.proto', recursive=True) + +proto_file_name = (resources.files("grpc_tools") / "_proto").resolve() +for file in files: + print(f"Process {file}") + protoc.main([ + '-c', '-Ilofar_sid/interface=proto', f'-I{proto_file_name}', '--python_out=.', '--pyi_out=.', '--grpc_python_out=.', file + ]) + +print("Complete") diff --git a/bin/install-hooks/pre-commit.sh b/bin/install-hooks/pre-commit.sh new file mode 100755 index 0000000000000000000000000000000000000000..792a3aabef83dc4ebf0c94635ae1de0e7412c479 --- /dev/null +++ b/bin/install-hooks/pre-commit.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ ! -f "setup.sh" ]; then + echo "pre-commit.sh must be executed with repository root as working directory!" + exit 1 +fi + +pre-commit install --hook-type pre-push diff --git a/proto/stationcontrol/station.proto b/proto/stationcontrol/station.proto new file mode 100644 index 0000000000000000000000000000000000000000..52a6258f7e91e4e89464813ed34e1ab0dd266737 --- /dev/null +++ b/proto/stationcontrol/station.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package station_control.station; + +enum Station_State { + OFF = 0; + HIBERNATE = 1; + STANDBY = 2; + ON = 3; +} + +service Station { + rpc GetStationState(GetStationStateRequest) returns (StationStateReply) {} + rpc SetState(SetStationStateRequest) returns (stream StationStateReply) {} + rpc SoftReset(SoftResetRequest) returns (stream StationStateReply) {} + rpc HardReset(HardResetRequest) returns (stream StationStateReply) {} +} + +message SetStationStateRequest { + Station_State station_state = 1; +} + +message GetStationStateRequest { +} + +message SoftResetRequest { + +} + +message HardResetRequest { + +} + +message StationStateResult { + Station_State station_state = 1; +} + +message StationStateReply { + StationStateResult result = 1; +} diff --git a/pyproject.toml b/pyproject.toml index 9e8708c560070ca89ba382aeb10c1e0df6ce95c9..ecb9be0024a1fb0d1cf3eaa6543eb5ab1542765c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,83 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] version_file = "lofar_sid/_version.py" -[tool.pylint] -ignore = "_version.py" +[tool.ruff] +exclude = [ + ".venv", + ".git", + ".tox", + "dist", + "docs", + "*lib/python*", + "*egg", + "_version.py", + "*_pb2.py", + "*_pb2_grpc.py" +] + +[tool.ruff.lint] +ignore = ["E203"] +[tool.tox] +# Generative environment list to test all supported Python versions +requires = ["tox>=4.21"] +env_list = ["fix", "coverage", "lint", "format", "py{13, 12, 11, 10}"] + +[tool.tox.env_run_base] +package = "editable" +deps = [ + "-r{toxinidir}/requirements.txt", + "-r{toxinidir}/tests/requirements.txt"] +set_env = { LANGUAGE = "en_US", LC_ALL = "en_US.UTF-8", PYTHONWARNINGS = "default::DeprecationWarning" } +commands_pre = [ + ["python", "--version"], + ["python", "bin/build_proto.py"] +] +commands = [["python", "--version"], ["python", "-m", "pytest", "tests/{posargs}"]] + +[tool.tox.env.fix] +description = "format the code base to adhere to our styles, and complain about what we cannot do automatically" +skip_install = true +deps = ["pre-commit-uv>=4.1.1"] +commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure"]] + +[tool.tox.env.coverage] +commands = [ + ["python", "--version"], + ["python", "-m", "pytest", "--cov-report", "term", "--cov-report", "xml", "--cov-report", "html", "--cov=lofar_sid", "tests/{posargs}"] +] + +# Command prefixes to reuse the same virtualenv for all linting jobs. +[tool.tox.env.lint] +deps = [ + "ruff", + "-r{toxinidir}/tests/requirements.txt"] +commands = [ + ["python", "-m", "ruff", "--version"], + ["python", "-m", "ruff", "check", { replace = "posargs", default = ["lofar_sid", "tests"], extend = true }] +] + +[tool.tox.env.format] +deps = [ + "ruff", + "-r{toxinidir}/tests/requirements.txt"] +commands = [ + ["python", "-m", "ruff", "format", "-v", { replace = "posargs", default = ["lofar_sid", "tests"], extend = true }] +] + +[tool.tox.env.docs] +deps = [ + "-r{toxinidir}/requirements.txt", + "-r{toxinidir}/docs/requirements.txt"] +# unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise +set_env = "" +changedir = "{tox_root}" +commands = [ + ["python", "docs/cleanup.py"], + ["sphinx-build", "-b", "html", "docs/source", "docs/build/html"] +] -[tool.black] -exclude = '.*pb2.*\.py' \ No newline at end of file +[tool.tox.env.build] +package = "wheel" +deps = ["build>=0.8.0"] +commands = [["python", "-m", "build"]] diff --git a/requirements.txt b/requirements.txt index f05a2233eeebf821dfe9a372a79868b1bbe85bc5..724145affc11d853a08874b81514e2a05ec69776 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -importlib-metadata>=0.12, <5.0;python_version<"3.8" numpy -grpcio-tools # Apache 2 \ No newline at end of file +grpcio-tools # Apache 2 diff --git a/setup.cfg b/setup.cfg index f5336dc3c062a04b632a912fa17bff10281451f2..2de01b85f3f8d10761b4e12848ea179474338f88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,13 +10,10 @@ classifiers = Environment :: Plugins Intended Audience :: Developers Intended Audience :: Science/Research - License :: OSI Approved :: Apache Software License Operating System :: OS Independent Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 @@ -28,10 +25,5 @@ classifiers = [options] include_package_data = true packages = find: -python_requires = >=3.8 +python_requires = >=3.10 install_requires = file: requirements.txt - -[flake8] -max-line-length = 88 -extend-ignore = E203 -exclude=.venv,.git,.tox,dist,docs,*lib/python*,*egg,_version.py diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000000000000000000000000000000000000..66fa5d1de40bd4d52a1f61ebd97f140b48682b9b --- /dev/null +++ b/setup.sh @@ -0,0 +1,40 @@ +#! /usr/bin/env bash +# +# Copyright (C) 2025 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 +# + +# Compatibility with zsh +# shellcheck disable=SC2128 +if [ -z "${BASH_SOURCE}" ]; then + # shellcheck disable=SC2296 + BASH_SOURCE=${(%):-%x} +fi + +ABSOLUTE_PATH="$(realpath "$(dirname "${BASH_SOURCE}")")" +export PROJECT_DIR=${1:-${ABSOLUTE_PATH}} + +# Create a virtual environment directory if it doesn't exist +VENV_DIR="${PROJECT_DIR}/venv" +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment..." + python3 -m venv "$VENV_DIR" +fi + +# Activate the virtual environment +# shellcheck disable=SC1091 +source "$VENV_DIR/bin/activate" +python -m pip install pre-commit +python -m pip install "tox>=4.21.0" + +# Install git hooks +if [ ! -f "${PROJECT_DIR}/.git/hooks/post-checkout" ]; then + # shellcheck disable=SC1091 + source "${PROJECT_DIR}/bin/install-hooks/submodule-and-lfs.sh" +fi + +# Install git pre-commit pre-push hook +if [ ! -f "${PROJECT_DIR}/.git/hooks/pre-push.legacy" ]; then + # shellcheck disable=SC1091 + source "${PROJECT_DIR}/bin/install-hooks/pre-commit.sh" +fi diff --git a/tests/test_sid.py b/tests/test_sid.py index b748bd9afdfd48f7889cea6ced9826c4a399d143..c1e4120c84c3e05eed383fd7bb27154aebf3d2fb 100644 --- a/tests/test_sid.py +++ b/tests/test_sid.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Testing of the Cool Module""" + from unittest import TestCase from lofar_sid.contract import about diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 91e1eeea5deab4d18e4c1bf45db6a156fb6eb16b..0000000000000000000000000000000000000000 --- a/tox.ini +++ /dev/null @@ -1,63 +0,0 @@ -[tox] -# Generative environment list to test all supported Python versions -min_version = 4.3.3 -requires = - tox-ignore-env-name-mismatch >= 0.2.0 - -[testenv] -usedevelop = True -package = wheel -wheel_build_env = .pkg - -setenv = - PYTHONWARNINGS=default::DeprecationWarning -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/tests/requirements.txt - -commands_pre = - {envpython} --version - {envpython} -m grpc_tools.protoc -Ilofar_sid/interface=proto --python_out=. --pyi_out=. --grpc_python_out=. proto/stationcontrol/observation.proto proto/stationcontrol/antennafield.proto proto/opah/grafana-apiv3.proto proto/stationcontrol/statistics.proto proto/stationcontrol/subrack.proto - - -commands = - {envpython} --version - {envpython} -m pytest tests/{posargs} - -[testenv:coverage] -commands = - {envpython} --version - {envpython} -m pytest --cov-report term --cov-report xml --cov-report html --cov=lofar_sid tests/{posargs} - -# Use generative name and command prefixes to reuse the same virtualenv -# for all linting jobs. -[testenv:{pep8,black,pylint,format}] -usedevelop = False -package = editable -envdir = {toxworkdir}/linting -commands = - pep8: {envpython} -m flake8 --version - pep8: {envpython} -m flake8 --exclude=lofar_sid/interface lofar_sid tests - black: {envpython} -m black --version - black: {envpython} -m black --check --diff lofar_sid tests --exclude '.*pb2.*\.py|_version\.py' - pylint: {envpython} -m pylint --version - pylint: {envpython} -m pylint --prefer-stub=true lofar_sid tests - format: {envpython} -m autopep8 -v -aa --in-place --recursive lofar_sid - format: {envpython} -m autopep8 -v -aa --in-place --recursive tests - format: {envpython} -m black -v lofar_sid tests - -[testenv:docs] -; unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise -setenv = -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/docs/requirements.txt -changedir = {toxinidir} -commands = - {envpython} docs/cleanup.py - sphinx-build -b html docs/source docs/build/html - -[testenv:build] -usedevelop = False -deps = build -commands = {envpython} -m build