From 2fc2125ed6db503e12d18c83f09b43078ba4032a Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 13:42:56 +0100 Subject: [PATCH 01/20] Add pre-commit hook and improve with better linting --- cookiecutter.json | 3 +- docker/ci-runner/Dockerfile | 4 +-- {{cookiecutter.project_slug}}/.dockerignore | 4 +++ {{cookiecutter.project_slug}}/.gitignore | 10 +++++-- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 27 +++++------------ .../.pre-commit-config.yaml | 22 ++++++++++++++ .../bin/install-hooks/pre-commit.sh | 8 +++++ .../docker/ci-runner/Dockerfile | 2 +- .../{{cookiecutter.project_slug}}/Dockerfile | 18 ++++++++++++ {{cookiecutter.project_slug}}/docs/cleanup.py | 23 +++++++++++++++ {{cookiecutter.project_slug}}/docs/cleanup.sh | 14 --------- {{cookiecutter.project_slug}}/pyproject.toml | 29 +++++++++---------- .../tests/requirements.txt | 6 ---- .../tests/test_cool_module.py | 3 ++ 14 files changed, 112 insertions(+), 61 deletions(-) create mode 100644 {{cookiecutter.project_slug}}/.dockerignore create mode 100644 {{cookiecutter.project_slug}}/.pre-commit-config.yaml create mode 100755 {{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh create mode 100644 {{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile create mode 100644 {{cookiecutter.project_slug}}/docs/cleanup.py delete mode 100644 {{cookiecutter.project_slug}}/docs/cleanup.sh diff --git a/cookiecutter.json b/cookiecutter.json index 1f29d35..8165b97 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -1,6 +1,7 @@ { "project_name": "My Awesome App", "project_slug": "{{ cookiecutter.project_name.lower()|replace(' ', '_')|replace('-', '_')|replace('.', '_')|trim() }}", - "project_url": "https://git.astron.nl/{{cookiecutter.project_slug}}", + "project_package": "{{ cookiecutter.project_name.lower()|replace(' ', '-')|replace('_', '-')|replace('.', '-')|trim() }}", + "project_url": "https://git.astron.nl/{{ cookiecutter.project_slug }}", "description": "An example package for CI/CD working group" } diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index 8b96217..497b89a 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -1,5 +1,5 @@ -FROM python:3.11 +FROM python:3.12 RUN python -m pip install --upgrade pip -RUN pip install --upgrade cookiecutter tox twine cibuildwheel==2.13.1 cookiecutter +RUN python -m pip install --upgrade cookiecutter tox twine cibuildwheel==2.13.1 cookiecutter RUN curl -sSL https://get.docker.com/ | sh diff --git a/{{cookiecutter.project_slug}}/.dockerignore b/{{cookiecutter.project_slug}}/.dockerignore new file mode 100644 index 0000000..141f90d --- /dev/null +++ b/{{cookiecutter.project_slug}}/.dockerignore @@ -0,0 +1,4 @@ +.tox +build +*.egg-info +.venv diff --git a/{{cookiecutter.project_slug}}/.gitignore b/{{cookiecutter.project_slug}}/.gitignore index 5513c1b..885470b 100644 --- a/{{cookiecutter.project_slug}}/.gitignore +++ b/{{cookiecutter.project_slug}}/.gitignore @@ -1,13 +1,19 @@ dist/* *.egg-info *.pyc +.tox +.venv .coverage coverage.xml htmlcov/* +build +dist -# Tox -.tox +# Documentation +docs/source/source_documentation +!docs/source/source_documentation/index.rst +docs/build # setuptools-scm src/{{cookiecutter.project_slug}}/_version.py diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index 4294826..5071e25 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -14,6 +14,7 @@ stages: - lint - test - package + - images - integration - publish # publish instead of deploy @@ -35,22 +36,10 @@ trigger_prepare: strategy: depend include: .prepare.gitlab-ci.yml -run_black: +run_lint: stage: lint script: - - tox -e black - allow_failure: true - -run_flake8: - stage: lint - script: - - tox -e pep8 - allow_failure: true - -run_pylint: - stage: lint - script: - - tox -e pylint + - tox -e lint allow_failure: true sast: @@ -74,7 +63,7 @@ secret_detection: before_script: - python --version # For debugging - python -m pip install --upgrade pip - - pip install --upgrade tox twine + - python -m pip install --upgrade tox twine # Run all unit tests for Python versions except the base image run_unit_tests: @@ -156,8 +145,7 @@ publish_on_test_pypi: - package_files when: manual rules: - - if: $CI_COMMIT_TAG - allow_failure: true + - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"' script: - echo "run twine for test pypi" # - | @@ -175,8 +163,7 @@ publish_on_pypi: - package_files when: manual rules: - - if: $CI_COMMIT_TAG - allow_failure: true + - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"' script: - echo "run twine for pypi" # - | @@ -195,7 +182,7 @@ publish_to_readthedocs: - package_docs when: manual rules: - - if: $CI_COMMIT_TAG + - if: '$CI_COMMIT_TAG && $CI_COMMIT_REF_PROTECTED == "true"' script: - echo "scp docs/* ???" - exit 1 diff --git a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml new file mode 100644 index 0000000..ce46e64 --- /dev/null +++ b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +default_stages: [ pre-commit, pre-push ] +default_language_version: + python: python3 +exclude: '^docs/.*\.py$' + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-toml + - id: detect-private-key + - repo: local + hooks: + - id: tox-lint + name: tox-lint (local) + entry: tox + language: python + types: [file, python] + args: ["-e", "lint", "--"] diff --git a/{{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh b/{{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh new file mode 100755 index 0000000..792a3aa --- /dev/null +++ b/{{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ ! -f "setup.sh" ]; then + echo "pre-commit.sh must be executed with repository root as working directory!" + exit 1 +fi + +pre-commit install --hook-type pre-push diff --git a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile index 48d4586..2deb674 100644 --- a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile +++ b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.13 RUN python -m pip install --upgrade pip -RUN pip install --upgrade tox twine cibuildwheel==2.13.1 +RUN python -m pip install --upgrade tox twine cibuildwheel==2.13.1 RUN curl -sSL https://get.docker.com/ | sh diff --git a/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile b/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile new file mode 100644 index 0000000..1f7eb27 --- /dev/null +++ b/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile @@ -0,0 +1,18 @@ +ARG BUILD_ENV=no_copy + +FROM python:3.13 AS build_no_copy +ADD ../../requirements.txt . +COPY ../.. /work +RUN rm -r /work/dist | true +RUN python -m pip install --user tox +WORKDIR /work +RUN python -m tox -e build + +FROM python:3.13 AS build_copy +COPY dist /work/dist + +FROM build_${BUILD_ENV} AS build + +FROM python:3.13-slim +COPY --from=build /work/dist /dist +RUN python -m pip install /dist/*.whl diff --git a/{{cookiecutter.project_slug}}/docs/cleanup.py b/{{cookiecutter.project_slug}}/docs/cleanup.py new file mode 100644 index 0000000..3a4508d --- /dev/null +++ b/{{cookiecutter.project_slug}}/docs/cleanup.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 + +import os + +file_dir = os.path.dirname(os.path.realpath(__file__)) + +clean_dir = os.path.join(file_dir, "source", "source_documentation") +print(f"Cleaning.. {clean_dir}/*") + +if not os.path.exists(clean_dir): + exit() + +for file_name in os.listdir(clean_dir): + file = os.path.join(clean_dir, file_name) + + if file_name == "index.rst": + continue + + print(f"Removing.. {file}") + os.remove(file) diff --git a/{{cookiecutter.project_slug}}/docs/cleanup.sh b/{{cookiecutter.project_slug}}/docs/cleanup.sh deleted file mode 100644 index aac4cef..0000000 --- a/{{cookiecutter.project_slug}}/docs/cleanup.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -FILE_DIR=$(dirname -- "$(readlink -f -- "${0}")") - -echo "Cleaning.. ${FILE_DIR}/source/source_documentation/*" - -for f in "${FILE_DIR}"/source/source_documentation/* -do - - case $f in - */index.rst) true;; - *) echo "Removing.. ${f}"; rm "${f}";; - esac -done diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index db601f1..3eefe30 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -46,20 +46,19 @@ archs = ["x86_64", "i686"] [tool.cibuildwheel.windows] archs = ["AMD64", "x86"] -[tool.pylint] -ignore = "_version.py" -# This does nothing until the issue below is resolved: -# https://github.com/PyCQA/flake8/issues/234 -[tool.flake8] -max-line-length = 88 -extend-ignore = ["E203"] -exclude = ["_version.py"] +[tool.ruff] +exclude = [ + ".venv", + ".git", + ".tox", + "dist", + "docs", + "*lib/python*", + "*egg", + "_version.py" +] + +[tool.ruff.lint] +ignore = ["E203"] -[tool.black] -line-length = 88 -extend-exclude = ''' -( - _version.py -) -''' diff --git a/{{cookiecutter.project_slug}}/tests/requirements.txt b/{{cookiecutter.project_slug}}/tests/requirements.txt index 82f1ee8..b507faf 100644 --- a/{{cookiecutter.project_slug}}/tests/requirements.txt +++ b/{{cookiecutter.project_slug}}/tests/requirements.txt @@ -1,8 +1,2 @@ -autopep8 >= 1.7.0 # MIT -black >= 22.0.0 # MIT -build >= 0.8.0 # MIT -flake8 >= 5.0.0 # MIT -pyproject-flake8 >= 5.0.4 # Unlicense -pylint >= 2.15.0 # GPLv2 pytest >= 7.0.0 # MIT pytest-cov >= 3.0.0 # MIT diff --git a/{{cookiecutter.project_slug}}/tests/test_cool_module.py b/{{cookiecutter.project_slug}}/tests/test_cool_module.py index 19e5cd2..2cacc8e 100644 --- a/{{cookiecutter.project_slug}}/tests/test_cool_module.py +++ b/{{cookiecutter.project_slug}}/tests/test_cool_module.py @@ -1,3 +1,6 @@ +# Copyright (C) 2025 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 + """Testing of the Cool Module""" from unittest import TestCase -- GitLab From 9716b9ea8dabdfffa2fa64d5a7ae27028155b777 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 13:53:27 +0100 Subject: [PATCH 02/20] fixes --- {{cookiecutter.project_slug}}/pyproject.toml | 65 ++++++++++++++++++++ {{cookiecutter.project_slug}}/tox.ini | 65 -------------------- 2 files changed, 65 insertions(+), 65 deletions(-) delete mode 100644 {{cookiecutter.project_slug}}/tox.ini diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 3eefe30..8018bc0 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -62,3 +62,68 @@ exclude = [ [tool.ruff.lint] ignore = ["E203"] + +[tool.tox] +# Generative environment list to test all supported Python versions +requires = ["tox>=4.21"] +env_list = ["fix", "coverage", "lint", "format", "py{13, 12, 11, 10}"] +requires = ["tox-ignore-env-name-mismatch >= 0.2.0"] + +[tool.tox.env_run_base] +package = "editable" +deps = [ + "-r{toxinidir}/requirements.txt", + "-r{toxinidir}/tests/requirements.txt"] +set_env = { LANGUAGE = "en_US", LC_ALL = "en_US.UTF-8", PYTHONWARNINGS = "default::DeprecationWarning", DYNAMIC_VERSION_SOURCE= "{toxinidir}" } +commands = [["python", "--version"], ["python", "-m", "pytest", "tests/{posargs}"]] + +[tool.tox.env.fix] +description = "format the code base to adhere to our styles, and complain about what we cannot do automatically" +skip_install = true +deps = ["pre-commit-uv>=4.1.1"] +commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure"]] + +[tool.tox.env.coverage] +commands = [ + ["python", "--version"], + ["python", "-m", "pytest", "--cov-report", "term", "--cov-report", "xml", "--cov-report", "html", "--cov={{cookiecutter.project_slug}}", "tests/{posargs}"]] + +# Command prefixes to reuse the same virtualenv for all linting jobs. +[tool.tox.env.lint] +deps = [ + "ruff", + "-r{toxinidir}/tests/requirements.txt"] +commands = [ + ["python", "-m", "ruff", "--version"], + ["python", "-m", "ruff", "check", { replace = "posargs", default = ["{{cookiecutter.project_slug}}", "tests"], extend = true }] +] + +[tool.tox.env.format] +deps = [ + "ruff", + "-r{toxinidir}/tests/requirements.txt"] +commands = [ + ["python", "-m", "ruff", "format", "-v", { replace = "posargs", default = ["{{cookiecutter.project_slug}}", "tests"], extend = true }] +] + +[tool.tox.env.docs] +deps = [ + "-r{toxinidir}/requirements.txt", + "-r{toxinidir}/docs/requirements.txt"] +# unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise +set_env = "" +changedir = "{tox_root}" +commands = [ + ["python", "docs/cleanup.py"], + ["sphinx-build", "-b", "html", "docs/source", "docs/build/html"] +] + +[tool.tox.env.build-local] +package = "wheel" +deps = ["-r{toxinidir}/build-requirements.txt"] +commands = [["python", "-m", "build"]] + +[tool.tox.env.build-ci-linux] +package = "wheel" +deps = ["-r{toxinidir}/build-requirements.txt"] +commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] diff --git a/{{cookiecutter.project_slug}}/tox.ini b/{{cookiecutter.project_slug}}/tox.ini deleted file mode 100644 index 37d15ba..0000000 --- a/{{cookiecutter.project_slug}}/tox.ini +++ /dev/null @@ -1,65 +0,0 @@ -[tox] -# Generative environment list to test all supported Python versions -envlist = py3{9,10,11,12,13},black,pep8,pylint -min_version = 4.3.3 -requires = - tox-ignore-env-name-mismatch >= 0.2.0 - -[testenv] -package = sdist # 'Source' package required for binary extension -use_develop = False # use_develop implies 'editable' package, not possible - -setenv = - PYTHONWARNINGS=default::DeprecationWarning - DYNAMIC_VERSION_SOURCE={toxinidir} -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/tests/requirements.txt -commands = - {envpython} --version - {envpython} -m pytest tests/{posargs} - -[testenv:coverage] -commands = - {envpython} --version - {envpython} -m pytest --cov-report term --cov-report xml --cov-report html --cov={{cookiecutter.project_slug}} tests/{posargs} - -# Use generative name and command prefixes to reuse the same virtualenv -# for all linting jobs. -[testenv:{pep8,black,pylint,format}] -usedevelop = False -envdir = {toxworkdir}/linting -commands = - pep8: {envpython} -m flake8 --version - pep8: {envpython} -m flake8 --exclude=_version.py --max-line-length 88 --extend-ignore=E203 src tests - black: {envpython} -m black --version - black: {envpython} -m black --check --diff src tests - pylint: {envpython} -m pylint --version - pylint: {envpython} -m pylint src tests - format: {envpython} -m autopep8 -v --max-line-length 88 -aa --in-place --recursive src - format: {envpython} -m autopep8 -v --max-line-length 88 -aa --in-place --recursive tests - format: {envpython} -m black -v src tests - -[testenv:docs] -allowlist_externals = - sh -; unset LC_ALL / LANGUAGE from testenv, would fail sphinx otherwise -setenv = - PYTHONWARNINGS=default::DeprecationWarning - DYNAMIC_VERSION_SOURCE={toxinidir} -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/docs/requirements.txt -changedir = {toxinidir} -commands = - sh docs/cleanup.sh - sphinx-build -b html docs/source docs/build/html - -[testenv:{build-local,build-ci-linux}] -deps = - -r{toxinidir}/tests/requirements.txt - -r{toxinidir}/build-requirements.txt -skip_install = True -commands = - build-local: {envpython} -m build - build-ci-linux: {envpython} -m cibuildwheel --platform linux --output-dir dist -- GitLab From 6e7b87d63e1399afa510d8fb8cd41f74fcfaae6e Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 13:59:31 +0100 Subject: [PATCH 03/20] fixes --- {{cookiecutter.project_slug}}/pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 8018bc0..8191c8b 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -65,9 +65,8 @@ ignore = ["E203"] [tool.tox] # Generative environment list to test all supported Python versions -requires = ["tox>=4.21"] +requires = ["tox>=4.21", "tox-ignore-env-name-mismatch >= 0.2.0"] env_list = ["fix", "coverage", "lint", "format", "py{13, 12, 11, 10}"] -requires = ["tox-ignore-env-name-mismatch >= 0.2.0"] [tool.tox.env_run_base] package = "editable" -- GitLab From ca769d04e8967dbf5ec30eecb36b6bb4a93035e0 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 14:03:19 +0100 Subject: [PATCH 04/20] fix --- {{cookiecutter.project_slug}}/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 8191c8b..3fc4fdf 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -94,7 +94,7 @@ deps = [ "-r{toxinidir}/tests/requirements.txt"] commands = [ ["python", "-m", "ruff", "--version"], - ["python", "-m", "ruff", "check", { replace = "posargs", default = ["{{cookiecutter.project_slug}}", "tests"], extend = true }] + ["python", "-m", "ruff", "check", { replace = "posargs", default = ["src", "tests"], extend = true }] ] [tool.tox.env.format] @@ -102,7 +102,7 @@ deps = [ "ruff", "-r{toxinidir}/tests/requirements.txt"] commands = [ - ["python", "-m", "ruff", "format", "-v", { replace = "posargs", default = ["{{cookiecutter.project_slug}}", "tests"], extend = true }] + ["python", "-m", "ruff", "format", "-v", { replace = "posargs", default = ["src", "tests"], extend = true }] ] [tool.tox.env.docs] -- GitLab From cd467f0bc21db4bc03dd2f394478f27e79a311b9 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 14:33:41 +0100 Subject: [PATCH 05/20] fix --- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 2 +- {{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index 5071e25..a08ac42 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -100,7 +100,7 @@ package_files: - dist/* script: - source scripts/setup-docker-host.sh - - cibuildwheel --platform linux --output-dir dist + - tox -e build-ci-linux package_docs: stage: package diff --git a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile index 2deb674..5af9fd4 100644 --- a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile +++ b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile @@ -1,5 +1,4 @@ FROM python:3.13 -RUN python -m pip install --upgrade pip -RUN python -m pip install --upgrade tox twine cibuildwheel==2.13.1 +RUN python -m pip install --upgrade cibuildwheel pip tox twine RUN curl -sSL https://get.docker.com/ | sh -- GitLab From 46787a73b81861611845f8fa585526caf6bcd63a Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 14:44:59 +0100 Subject: [PATCH 06/20] update --- .gitlab-ci.yml | 86 +++++++++++++---------------- project.gitlab-ci.yml | 122 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 49 deletions(-) create mode 100644 project.gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1650b82..20272ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,7 @@ -include: -- local: "{{cookiecutter.project_slug}}/.gitlab-ci.yml" - -variables: - GIT_SUBMODULE_STRATEGY: recursive +stages: + - prepare + - build + - test trigger_prepare: stage: prepare @@ -10,53 +9,42 @@ trigger_prepare: strategy: depend include: "{{cookiecutter.project_slug}}/.prepare.gitlab-ci.yml" -default: +# Generate template instance in my_awesome_app directory +build-template: + stage: build + image: $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG # Bootstrap Cookiecutter template to test provided ci pipeline template - before_script: + script: - python --version # For debugging - - git config --global user.name "unit test" - - git config --global user.email "info@astron.nl" - cookiecutter --no-input --overwrite-if-exists --output-dir . . - cd my_awesome_app - -# Override semgrep-sast before script -sast: - before_script: - - python --version # For debugging - -# Override unit test before script -.run_unit_test_version_base: - before_script: - - pip install cookiecutter - - !reference [default, before_script] - - python --version # For debugging - - python -m pip install --upgrade pip - - pip install --upgrade tox twine - -# Override artifact directories -package_docs: - stage: package - artifacts: - expire_in: 1w - paths: - - my_awesome_app/docs/build/* - -# override package files before script -package_files: - before_script: - - pip install cookiecutter - - !reference [default, before_script] + - git init + - git config user.email "ci-runner@example.com" + - git config user.name "CI Runner" + - source ./setup.sh + - ls -lah + - tox --version + - pip install --upgrade tox + - tox -e fix + - rm -r .venv + - rm -r .tox + # cannot use needs, for artifacts on child pipeline so must regenerate template! artifacts: - expire_in: 1w paths: - - my_awesome_app/dist/* - -# Override artifact directories -run_unit_tests_coverage: - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: my_awesome_app/coverage.xml - paths: - - my_awesome_app/htmlcov/* + - my_awesome_app/* + - project.gitlab-ci.yml + +# Spawn pipeline using the gitlab-ci.yml from generated template instance +# use project.gitlab.ci.yml for necessary job overrides from this template instance +# (due to changes in directories and paths etc) +project-pipeline: + stage: test + trigger: + strategy: depend + include: + - artifact: my_awesome_app/.gitlab-ci.yml + job: build-template + - artifact: project.gitlab-ci.yml + job: build-template + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID diff --git a/project.gitlab-ci.yml b/project.gitlab-ci.yml new file mode 100644 index 0000000..663b7b5 --- /dev/null +++ b/project.gitlab-ci.yml @@ -0,0 +1,122 @@ + +# This file overrides all the jobs defined in {{cookiecutter.project_slug}}/.gitlab.ci-yml +# this is to ensure they depend on the template installation artifact of the root +# .gitlab-ci.yml from job `build-template` +# The generated gitlab-ci.yml from this `build-template` job is used for the actual +# trigger include to prevent including jobs that still contain template arguments + + +trigger_prepare: + rules: + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" + when: never + +default: + # Bootstrap Cookiecutter template to test provided ci pipeline template + before_script: + - cd my_awesome_app + +run_lint: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + +sast: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + +dependency_scanning: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + +secret_detection: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + +.run_unit_test_version_base: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + +# Run all unit tests for Python versions except the base image +run_unit_tests: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + +# Run code coverage on the base image thus also performing unit tests +run_unit_tests_coverage: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + artifacts: + reports: + coverage_report: + coverage_format: cobertura + path: my_awesome_app/coverage.xml + paths: + - my_awesome_app/htmlcov/* + +package_files: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + artifacts: + expire_in: 1w + paths: + - my_awesome_app/dist/* + +package_docs: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + artifacts: + expire_in: 1w + paths: + - my_awesome_app/docs/build/* + +#docker_build: +# needs: +# - pipeline: $PARENT_PIPELINE_ID +# job: build-template +# - package_files +# before_script: +# - cd my_awesome_app + +run_integration_tests: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + - package_files + +publish_on_gitlab: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + - package_files + +publish_on_test_pypi: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + - package_files + +publish_on_pypi: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + - package_files + +publish_to_readthedocs: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template + - package_docs + +release_job: + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build-template -- GitLab From 45571645920f9cfb16312d7b6042d25b548b67d8 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 14:58:16 +0100 Subject: [PATCH 07/20] test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20272ac..003be8f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,9 +18,9 @@ build-template: - python --version # For debugging - cookiecutter --no-input --overwrite-if-exists --output-dir . . - cd my_awesome_app - - git init - git config user.email "ci-runner@example.com" - git config user.name "CI Runner" + - git init - source ./setup.sh - ls -lah - tox --version -- GitLab From c6da061ef8433ff37ff1f7bdb3fba252644bb91b Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 15:04:52 +0100 Subject: [PATCH 08/20] update --- .gitlab-ci.yml | 5 +++-- docker/ci-runner/Dockerfile | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 003be8f..0388ead 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,10 +16,11 @@ build-template: # Bootstrap Cookiecutter template to test provided ci pipeline template script: - python --version # For debugging + - git config --global user.email "ci-runner@example.com" + - git config --global user.name "CI Runner" + - git config --global init.defaultBranch main - cookiecutter --no-input --overwrite-if-exists --output-dir . . - cd my_awesome_app - - git config user.email "ci-runner@example.com" - - git config user.name "CI Runner" - git init - source ./setup.sh - ls -lah diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index 497b89a..4d06f79 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -1,5 +1,4 @@ -FROM python:3.12 +FROM python:3.13 RUN python -m pip install --upgrade pip -RUN python -m pip install --upgrade cookiecutter tox twine cibuildwheel==2.13.1 cookiecutter -RUN curl -sSL https://get.docker.com/ | sh +RUN python -m pip install --upgrade cookiecutter tox twine -- GitLab From 840430c484ede42a4344c313d5836d0da7336d39 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 15:08:45 +0100 Subject: [PATCH 09/20] add setup.sh --- {{cookiecutter.project_slug}}/setup.sh | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 {{cookiecutter.project_slug}}/setup.sh diff --git a/{{cookiecutter.project_slug}}/setup.sh b/{{cookiecutter.project_slug}}/setup.sh new file mode 100755 index 0000000..4548910 --- /dev/null +++ b/{{cookiecutter.project_slug}}/setup.sh @@ -0,0 +1,30 @@ +#! /usr/bin/env bash +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 +# + +# This file's directory is used to determine the station control directory +# location. +if [ -z ${BASH_SOURCE} ]; then + BASH_SOURCE=${(%):-%x} +fi + +ABSOLUTE_PATH=$(realpath $(dirname ${BASH_SOURCE})) + +# Create a virtual environment directory if it doesn't exist +VENV_DIR="${ABSOLUTE_PATH}/.venv" +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment..." + python3 -m venv "$VENV_DIR" +fi + +# Activate the virtual environment +source "$VENV_DIR/bin/activate" +python -m pip install pre-commit +python -m pip install "tox>=4.21.0" + +# Install git pre-commit pre-push hook +if [ ! -f "${ABSOLUTE_PATH}/.git/hooks/pre-push.legacy" ]; then + source "${ABSOLUTE_PATH}/bin/install-hooks/pre-commit.sh" +fi -- GitLab From 7993b014134e0e2cc6749309258638b9faefdc02 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 15:25:54 +0100 Subject: [PATCH 10/20] fix --- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 1 - {{cookiecutter.project_slug}}/pyproject.toml | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index a08ac42..41e7261 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -99,7 +99,6 @@ package_files: paths: - dist/* script: - - source scripts/setup-docker-host.sh - tox -e build-ci-linux package_docs: diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 3fc4fdf..2c4fa1b 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -125,4 +125,7 @@ commands = [["python", "-m", "build"]] [tool.tox.env.build-ci-linux] package = "wheel" deps = ["-r{toxinidir}/build-requirements.txt"] -commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] +commands = [ + ["source", "scripts/setup-docker-host.sh"], + ["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"] +] -- GitLab From 48a16050c2e428a0b7c581b2537bb255562e0817 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Tue, 11 Mar 2025 15:59:05 +0100 Subject: [PATCH 11/20] fix --- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 1 + {{cookiecutter.project_slug}}/pyproject.toml | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index 41e7261..a08ac42 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -99,6 +99,7 @@ package_files: paths: - dist/* script: + - source scripts/setup-docker-host.sh - tox -e build-ci-linux package_docs: diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 2c4fa1b..257b56d 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -125,7 +125,5 @@ commands = [["python", "-m", "build"]] [tool.tox.env.build-ci-linux] package = "wheel" deps = ["-r{toxinidir}/build-requirements.txt"] -commands = [ - ["source", "scripts/setup-docker-host.sh"], - ["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"] -] +allowlist_externals = ["docker"] +commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] -- GitLab From 48dc72ac250eebfc3f92818e1022b1d7894ef3ff Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Thu, 13 Mar 2025 10:02:45 +0100 Subject: [PATCH 12/20] fix --- docker/ci-runner/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index 4d06f79..ded7c6d 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -1,4 +1,5 @@ FROM python:3.13 -RUN python -m pip install --upgrade pip -RUN python -m pip install --upgrade cookiecutter tox twine +RUN python -m pip install --upgrade cibuildwheel pip tox twine +RUN curl -sSL https://get.docker.com/ | sh + -- GitLab From 76eed8bc5ba4f314760e1ba5a65e8174c9d4e915 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Thu, 13 Mar 2025 10:06:08 +0100 Subject: [PATCH 13/20] fix --- docker/ci-runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index ded7c6d..70893df 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.13 -RUN python -m pip install --upgrade cibuildwheel pip tox twine +RUN python -m pip install --upgrade cibuildwheel cookiecutter pip tox twine RUN curl -sSL https://get.docker.com/ | sh -- GitLab From 6860bd7ab8730e1c1a77a08a8caff7ad5593132e Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Thu, 13 Mar 2025 10:23:17 +0100 Subject: [PATCH 14/20] fix --- {{cookiecutter.project_slug}}/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 257b56d..5eb59e8 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -126,4 +126,5 @@ commands = [["python", "-m", "build"]] package = "wheel" deps = ["-r{toxinidir}/build-requirements.txt"] allowlist_externals = ["docker"] +pass_ent = ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"] commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] -- GitLab From 64dfb80af7c928e60e49f377633a91af41e0b487 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Thu, 13 Mar 2025 10:36:22 +0100 Subject: [PATCH 15/20] fix --- {{cookiecutter.project_slug}}/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 5eb59e8..85db592 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -126,5 +126,5 @@ commands = [["python", "-m", "build"]] package = "wheel" deps = ["-r{toxinidir}/build-requirements.txt"] allowlist_externals = ["docker"] -pass_ent = ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"] +pass_env = ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"] commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] -- GitLab From 1a5e491c6c2faff8f037ab324998f6be8b9e22d9 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Thu, 13 Mar 2025 10:45:52 +0100 Subject: [PATCH 16/20] fix --- {{cookiecutter.project_slug}}/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 85db592..c72b34b 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -125,6 +125,6 @@ commands = [["python", "-m", "build"]] [tool.tox.env.build-ci-linux] package = "wheel" deps = ["-r{toxinidir}/build-requirements.txt"] -allowlist_externals = ["docker"] +allowlist_externals = ["docker", "which"] pass_env = ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"] commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] -- GitLab From aba3915e2d61e77423ce6609507a26d6e70d1f0b Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Thu, 13 Mar 2025 11:00:24 +0100 Subject: [PATCH 17/20] remove python 3.9 --- {{cookiecutter.project_slug}}/pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index c72b34b..bdfbae8 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -11,13 +11,12 @@ name = "{{cookiecutter.project_slug}}" dynamic = ["version"] description="{{cookiecutter.description}}" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", -- GitLab From e8d9c673da07223be900658f395d97528cbb8c2a Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Fri, 14 Mar 2025 11:42:04 +0100 Subject: [PATCH 18/20] fix --- {{cookiecutter.project_slug}}/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index bdfbae8..9387c01 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -124,6 +124,6 @@ commands = [["python", "-m", "build"]] [tool.tox.env.build-ci-linux] package = "wheel" deps = ["-r{toxinidir}/build-requirements.txt"] -allowlist_externals = ["docker", "which"] +allowlist_externals = ["docker", "which", "{work_dir}/.tox/bin/python"] pass_env = ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"] commands = [["python", "-m", "cibuildwheel", "--platform", "linux", "--output-dir", "dist"]] -- GitLab From 1f3eeaac5ca305d93ebcf5b145e08b84f6ce331a Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Fri, 14 Mar 2025 11:52:45 +0100 Subject: [PATCH 19/20] fix --- {{cookiecutter.project_slug}}/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/{{cookiecutter.project_slug}}/CMakeLists.txt b/{{cookiecutter.project_slug}}/CMakeLists.txt index ce8b8df..3b8b18d 100644 --- a/{{cookiecutter.project_slug}}/CMakeLists.txt +++ b/{{cookiecutter.project_slug}}/CMakeLists.txt @@ -6,6 +6,9 @@ set( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/cmake-extra-utils/cmake" ) +set(LARGEFILE_SOURCE) +set(FILE_OFFSET_BITS 64) + # Include script include(DynamicVersion) dynamic_version( -- GitLab From 007d754de4bd10e80fe7db280bb36750165fe5c7 Mon Sep 17 00:00:00 2001 From: Hannes Feldt <feldt@astron.nl> Date: Fri, 14 Mar 2025 12:18:24 +0100 Subject: [PATCH 20/20] test --- docker/ci-runner/Dockerfile | 2 +- {{cookiecutter.project_slug}}/build-requirements.txt | 2 +- {{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index 70893df..2076807 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.13 -RUN python -m pip install --upgrade cibuildwheel cookiecutter pip tox twine +RUN python -m pip install --upgrade cibuildwheel==2.23.0 cookiecutter pip tox twine RUN curl -sSL https://get.docker.com/ | sh diff --git a/{{cookiecutter.project_slug}}/build-requirements.txt b/{{cookiecutter.project_slug}}/build-requirements.txt index 357a5ca..d2d6bde 100644 --- a/{{cookiecutter.project_slug}}/build-requirements.txt +++ b/{{cookiecutter.project_slug}}/build-requirements.txt @@ -1,3 +1,3 @@ -cibuildwheel >= 2.12.3 # BSD +cibuildwheel==2.23.0 # BSD tox >= 4.0 # ? build >= 0.8.0 # MIT diff --git a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile index 5af9fd4..c984a99 100644 --- a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile +++ b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.13 -RUN python -m pip install --upgrade cibuildwheel pip tox twine +RUN python -m pip install --upgrade cibuildwheel==2.23.0 pip tox twine RUN curl -sSL https://get.docker.com/ | sh -- GitLab