Skip to content
Snippets Groups Projects
Commit b4305f1b authored by Hannes Feldt's avatar Hannes Feldt
Browse files

Merge branch 'add-pre-commit-improve-tox-setup' into 'main'

Add pre-commit hook and improve with better linting

See merge request !33
parents dbf96a5f e2f321cf
No related branches found
No related tags found
1 merge request!33Add pre-commit hook and improve with better linting
Pipeline #109977 failed
Pipeline: Python Package

#109978

    Showing
    with 175 additions and 107 deletions
    ...@@ -19,6 +19,15 @@ build-template: ...@@ -19,6 +19,15 @@ build-template:
    - cookiecutter --no-input --overwrite-if-exists --output-dir . . - cookiecutter --no-input --overwrite-if-exists --output-dir . .
    - cd my_awesome_app - cd my_awesome_app
    - git init - git init
    - git config user.email "ci-runner@example.com"
    - git config user.name "CI Runner"
    - source ./setup.sh
    - ls -lah
    - tox --version
    - pip install --upgrade tox
    - tox -e fix
    - rm -r .venv
    - rm -r .tox
    # cannot use needs, for artifacts on child pipeline so must regenerate template! # cannot use needs, for artifacts on child pipeline so must regenerate template!
    artifacts: artifacts:
    paths: paths:
    ......
    { {
    "project_name": "My Awesome App", "project_name": "My Awesome App",
    "project_slug": "{{ cookiecutter.project_name.lower()|replace(' ', '_')|replace('-', '_')|replace('.', '_')|trim() }}", "project_slug": "{{ cookiecutter.project_name.lower()|replace(' ', '_')|replace('-', '_')|replace('.', '_')|trim() }}",
    "project_package": "{{ cookiecutter.project_name.lower()|replace(' ', '-')|replace('_', '-')|replace('.', '-')|trim() }}",
    "project_url": "https://git.astron.nl/{{ cookiecutter.project_slug }}", "project_url": "https://git.astron.nl/{{ cookiecutter.project_slug }}",
    "description": "An example package for CI/CD working group" "description": "An example package for CI/CD working group"
    } }
    ...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
    # The generated gitlab-ci.yml from this `build-template` job is used for the actual # The generated gitlab-ci.yml from this `build-template` job is used for the actual
    # trigger include to prevent including jobs that still contain template arguments # trigger include to prevent including jobs that still contain template arguments
    trigger_prepare: trigger_prepare:
    rules: rules:
    - if: $CI_PIPELINE_SOURCE == "parent_pipeline" - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
    ...@@ -15,17 +16,7 @@ default: ...@@ -15,17 +16,7 @@ default:
    before_script: before_script:
    - cd my_awesome_app - cd my_awesome_app
    run_black: run_lint:
    needs:
    - pipeline: $PARENT_PIPELINE_ID
    job: build-template
    run_flake8:
    needs:
    - pipeline: $PARENT_PIPELINE_ID
    job: build-template
    run_pylint:
    needs: needs:
    - pipeline: $PARENT_PIPELINE_ID - pipeline: $PARENT_PIPELINE_ID
    job: build-template job: build-template
    ......
    .tox
    build
    *.egg-info
    .venv
    ...@@ -2,6 +2,7 @@ dist/* ...@@ -2,6 +2,7 @@ dist/*
    *.egg-info *.egg-info
    *.pyc *.pyc
    .tox .tox
    .venv
    .coverage .coverage
    coverage.xml coverage.xml
    ......
    ...@@ -36,22 +36,10 @@ trigger_prepare: ...@@ -36,22 +36,10 @@ trigger_prepare:
    strategy: depend strategy: depend
    include: .prepare.gitlab-ci.yml include: .prepare.gitlab-ci.yml
    run_black: run_lint:
    stage: lint stage: lint
    script: script:
    - tox -e black - tox -e lint
    allow_failure: true
    run_flake8:
    stage: lint
    script:
    - tox -e pep8
    allow_failure: true
    run_pylint:
    stage: lint
    script:
    - tox -e pylint
    allow_failure: true allow_failure: true
    # build_extensions: # build_extensions:
    ...@@ -91,7 +79,7 @@ run_unit_tests: ...@@ -91,7 +79,7 @@ run_unit_tests:
    - tox -e py3${PY_VERSION} - tox -e py3${PY_VERSION}
    parallel: parallel:
    matrix: # use the matrix for testing matrix: # use the matrix for testing
    - PY_VERSION: [8, 9, 10, 11] - PY_VERSION: [9, 10, 11, 12]
    # Run code coverage on the base image thus also performing unit tests # Run code coverage on the base image thus also performing unit tests
    run_unit_tests_coverage: run_unit_tests_coverage:
    ......
    default_stages: [ pre-commit, pre-push ]
    default_language_version:
    python: python3
    exclude: '^docs/.*\.py$'
    repos:
    - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v4.5.0
    hooks:
    - id: end-of-file-fixer
    - id: trailing-whitespace
    - id: check-yaml
    - id: check-toml
    - id: detect-private-key
    - repo: local
    hooks:
    - id: tox-lint
    name: tox-lint (local)
    entry: tox
    language: python
    types: [file, python]
    args: ["-e", "lint", "--"]
    ...@@ -29,8 +29,16 @@ from {{cookiecutter.project_slug}} import cool_module ...@@ -29,8 +29,16 @@ from {{cookiecutter.project_slug}} import cool_module
    cool_module.greeter() # prints "Hello World" cool_module.greeter() # prints "Hello World"
    ``` ```
    ## Contributing ## Development
    ### Development environment
    To setup and activte the develop environment run ```source ./setup.sh``` from within the source directory.
    If PyCharm is used, this only needs to be done once.
    Afterward the Python virtual env can be setup within PyCharm.
    ### Contributing
    To contribute, please create a feature branch and a "Draft" merge request. To contribute, please create a feature branch and a "Draft" merge request.
    Upon completion, the merge request should be marked as ready and a reviewer Upon completion, the merge request should be marked as ready and a reviewer
    should be assigned. should be assigned.
    ......
    #!/bin/bash
    if [ ! -f "setup.sh" ]; then
    echo "pre-commit.sh must be executed with repository root as working directory!"
    exit 1
    fi
    pre-commit install --hook-type pre-push
    FROM python:3.12 FROM python:3.13
    RUN python -m pip install --upgrade pip RUN python -m pip install --upgrade pip
    RUN python -m pip install --upgrade tox twine RUN python -m pip install --upgrade tox twine
    ARG BUILD_ENV=no_copy ARG BUILD_ENV=no_copy
    FROM python:3.11 AS build_no_copy FROM python:3.13 AS build_no_copy
    ADD ../../requirements.txt . ADD ../../requirements.txt .
    COPY ../.. /work COPY ../.. /work
    RUN rm -r /work/dist | true RUN rm -r /work/dist | true
    ...@@ -8,11 +8,11 @@ RUN python -m pip install --user tox ...@@ -8,11 +8,11 @@ RUN python -m pip install --user tox
    WORKDIR /work WORKDIR /work
    RUN python -m tox -e build RUN python -m tox -e build
    FROM python:3.11 AS build_copy FROM python:3.13 AS build_copy
    COPY dist /work/dist COPY dist /work/dist
    FROM build_${BUILD_ENV} AS build FROM build_${BUILD_ENV} AS build
    FROM python:3.11-slim FROM python:3.13-slim
    COPY --from=build /work/dist /dist COPY --from=build /work/dist /dist
    RUN python -m pip install /dist/*.whl RUN python -m pip install /dist/*.whl
    [build-system] [build-system]
    requires = [ requires = [
    "setuptools>=62.6", "setuptools>=70.0",
    "setuptools_scm[toml]>=8.0", "setuptools_scm[toml]>=8.0",
    "wheel" "wheel"
    ] ]
    ...@@ -9,5 +9,76 @@ build-backend = "setuptools.build_meta" ...@@ -9,5 +9,76 @@ build-backend = "setuptools.build_meta"
    [tool.setuptools_scm] [tool.setuptools_scm]
    version_file = "{{cookiecutter.project_slug}}/_version.py" version_file = "{{cookiecutter.project_slug}}/_version.py"
    [tool.pylint] [tool.ruff]
    ignore = "_version.py" exclude = [
    ".venv",
    ".git",
    ".tox",
    "dist",
    "docs",
    "*lib/python*",
    "*egg",
    "_version.py"
    ]
    [tool.ruff.lint]
    ignore = ["E203"]
    [tool.tox]
    # Generative environment list to test all supported Python versions
    requires = ["tox>=4.21"]
    env_list = ["fix", "coverage", "lint", "format", "py{13, 12, 11, 10}"]
    [tool.tox.env_run_base]
    package = "editable"
    deps = [
    "-r{toxinidir}/requirements.txt",
    "-r{toxinidir}/tests/requirements.txt"]
    set_env = { LANGUAGE = "en_US", LC_ALL = "en_US.UTF-8", PYTHONWARNINGS = "default::DeprecationWarning" }
    commands = [["python", "--version"], ["python", "-m", "pytest", "tests/{posargs}"]]
    [tool.tox.env.fix]
    description = "format the code base to adhere to our styles, and complain about what we cannot do automatically"
    skip_install = true
    deps = ["pre-commit-uv>=4.1.1"]
    commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure"]]
    [tool.tox.env.coverage]
    commands = [
    ["python", "--version"],
    ["python", "-m", "pytest", "--cov-report", "term", "--cov-report", "xml", "--cov-report", "html", "--cov={{cookiecutter.project_slug}}", "tests/{posargs}"]]
    # Command prefixes to reuse the same virtualenv for all linting jobs.
    [tool.tox.env.lint]
    deps = [
    "ruff",
    "-r{toxinidir}/tests/requirements.txt"]
    commands = [
    ["python", "-m", "ruff", "--version"],
    ["python", "-m", "ruff", "check", { replace = "posargs", default = ["{{cookiecutter.project_slug}}", "tests"], extend = true }]
    ]
    [tool.tox.env.format]
    deps = [
    "ruff",
    "-r{toxinidir}/tests/requirements.txt"]
    commands = [
    ["python", "-m", "ruff", "format", "-v", { replace = "posargs", default = ["{{cookiecutter.project_slug}}", "tests"], extend = true }]
    ]
    [tool.tox.env.docs]
    deps = [
    "-r{toxinidir}/requirements.txt",
    "-r{toxinidir}/docs/requirements.txt"]
    # unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise
    set_env = ""
    changedir = "{tox_root}"
    commands = [
    ["python", "docs/cleanup.py"],
    ["sphinx-build", "-b", "html", "docs/source", "docs/build/html"]
    ]
    [tool.tox.env.build]
    package = "wheel"
    deps = ["build>=0.8.0"]
    commands = [["python", "-m", "build"]]
    importlib-metadata>=0.12, <5.0;python_version<"3.8"
    numpy numpy
    [metadata] [metadata]
    name = {{cookiecutter.project_slug}} name = {{cookiecutter.project_package}}
    description = {{cookiecutter.description}} description = {{cookiecutter.description}}
    long_description = file: README.md long_description = file: README.md
    long_description_content_type = text/markdown long_description_content_type = text/markdown
    ...@@ -15,11 +15,10 @@ classifiers = ...@@ -15,11 +15,10 @@ classifiers =
    Programming Language :: Python Programming Language :: Python
    Programming Language :: Python :: 3 Programming Language :: Python :: 3
    Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3 :: Only
    Programming Language :: Python :: 3.8
    Programming Language :: Python :: 3.9
    Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.10
    Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.11
    Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.12
    Programming Language :: Python :: 3.13
    Topic :: Internet :: WWW/HTTP Topic :: Internet :: WWW/HTTP
    Topic :: Internet :: WWW/HTTP :: Dynamic Content Topic :: Internet :: WWW/HTTP :: Dynamic Content
    Topic :: Scientific/Engineering Topic :: Scientific/Engineering
    ...@@ -28,10 +27,6 @@ classifiers = ...@@ -28,10 +27,6 @@ classifiers =
    [options] [options]
    include_package_data = true include_package_data = true
    packages = find: packages = find:
    python_requires = >=3.8 python_requires = >=3.10
    install_requires = file: requirements.txt install_requires = file: requirements.txt
    [flake8]
    max-line-length = 88
    extend-ignore = E203
    exclude=.venv,.git,.tox,dist,docs,*lib/python*,*egg,_version.py
    # Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
    # SPDX-License-Identifier: Apache-2.0
    """ Setuptools entry point """
    import setuptools import setuptools
    setuptools.setup() setuptools.setup()
    #! /usr/bin/env bash
    #
    # Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
    # SPDX-License-Identifier: Apache-2.0
    #
    # This file's directory is used to determine the station control directory
    # location.
    if [ -z ${BASH_SOURCE} ]; then
    BASH_SOURCE=${(%):-%x}
    fi
    ABSOLUTE_PATH=$(realpath $(dirname ${BASH_SOURCE}))
    # Create a virtual environment directory if it doesn't exist
    VENV_DIR="${ABSOLUTE_PATH}/.venv"
    if [ ! -d "$VENV_DIR" ]; then
    echo "Creating virtual environment..."
    python3 -m venv "$VENV_DIR"
    fi
    # Activate the virtual environment
    source "$VENV_DIR/bin/activate"
    python -m pip install pre-commit
    python -m pip install "tox>=4.21.0"
    # Install git pre-commit pre-push hook
    if [ ! -f "${ABSOLUTE_PATH}/.git/hooks/pre-push.legacy" ]; then
    source "${ABSOLUTE_PATH}/bin/install-hooks/pre-commit.sh"
    fi
    autopep8 >= 1.7.0 # MIT
    black >= 22.0.0 # MIT
    build >= 0.8.0 # MIT
    flake8 >= 5.0.0 # MIT
    pylint >= 2.15.0 # GPLv2
    pytest >= 7.0.0 # MIT pytest >= 7.0.0 # MIT
    pytest-cov >= 3.0.0 # MIT pytest-cov >= 3.0.0 # MIT
    [tox]
    # Generative environment list to test all supported Python versions
    envlist = py3{9,10,11,12,13},black,pep8,pylint
    min_version = 4.3.3
    requires =
    tox-ignore-env-name-mismatch >= 0.2.0
    [testenv]
    usedevelop = True
    package = wheel
    wheel_build_env = .pkg
    setenv =
    PYTHONWARNINGS=default::DeprecationWarning
    deps =
    -r{toxinidir}/requirements.txt
    -r{toxinidir}/tests/requirements.txt
    commands =
    {envpython} --version
    {envpython} -m pytest tests/{posargs}
    [testenv:coverage]
    commands =
    {envpython} --version
    {envpython} -m pytest --cov-report term --cov-report xml --cov-report html --cov={{cookiecutter.project_slug}} tests/{posargs}
    # Use generative name and command prefixes to reuse the same virtualenv
    # for all linting jobs.
    [testenv:{pep8,black,pylint,format}]
    usedevelop = False
    package = editable
    envdir = {toxworkdir}/linting
    commands =
    pep8: {envpython} -m flake8 --version
    pep8: {envpython} -m flake8 {{cookiecutter.project_slug}} tests
    black: {envpython} -m black --version
    black: {envpython} -m black --check --diff {{cookiecutter.project_slug}} tests
    pylint: {envpython} -m pylint --version
    pylint: {envpython} -m pylint {{cookiecutter.project_slug}} tests
    format: {envpython} -m autopep8 -v -aa --in-place --recursive {{cookiecutter.project_slug}}
    format: {envpython} -m autopep8 -v -aa --in-place --recursive tests
    format: {envpython} -m black -v {{cookiecutter.project_slug}} tests
    [testenv:docs]
    ; unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise
    setenv =
    deps =
    -r{toxinidir}/requirements.txt
    -r{toxinidir}/docs/requirements.txt
    changedir = {toxinidir}
    commands =
    {envpython} docs/cleanup.py
    sphinx-build -b html docs/source docs/build/html
    [testenv:build]
    usedevelop = False
    deps = build
    commands = {envpython} -m build
    0% Loading or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Please register or to comment