diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index a734451a11533307c67551452655bf6951d7235d..b35811a7d841a445e034ae812a2ee16bd55eae56 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.12 +# This dockerfile is used throughout steps of the CI/CD pipeline +# It contains dependencies to execute the template and run tests +FROM python:3.13 RUN python -m pip install --upgrade pip RUN python -m pip install --upgrade cookiecutter tox twine diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index b4e3610be0cd53f508c1915a5d45989564a1cbdb..e7d1fe0677279eea6b46645d39d40325292dbf45 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -2,8 +2,6 @@ default: image: name: $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG pull_policy: always - before_script: - - python --version # For debugging cache: paths: - .cache/pip @@ -24,12 +22,13 @@ stages: variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" +# Job templates from Gitlab that scan for security issues or leaked secrets include: - template: Security/SAST.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/Secret-Detection.gitlab-ci.yml -# Prepare image to run ci on +# Prepare image to run ci on, image will be used in most jobs trigger_prepare: stage: prepare trigger: @@ -53,16 +52,6 @@ sast: pmd-apex, security-code-scan, sobelow, spotbugs stage: test -dependency_scanning: - # override default before_script, job won't have Python available - before_script: - - uname - -secret_detection: - # override default before_script, job won't have Python available - before_script: - - uname - # Basic setup for all Python versions for which we don't have a base image .run_unit_test_version_base: before_script: @@ -71,6 +60,7 @@ secret_detection: - python -m pip install --upgrade tox twine # Run all unit tests for Python versions except the base image +# The base image is used for the highest Python version run_unit_tests: extends: .run_unit_test_version_base stage: test @@ -95,6 +85,7 @@ run_unit_tests_coverage: paths: - htmlcov/* +# Build and packge the project package_files: stage: package artifacts: @@ -104,6 +95,7 @@ package_files: script: - tox -e build +# Build the sphinx documentation package_docs: stage: package artifacts: @@ -113,6 +105,7 @@ package_docs: script: - tox -e docs +# Build a tiny docker image that contains the minimal dependencies to run this project docker_build: stage: images image: docker:latest @@ -203,6 +196,7 @@ publish_to_readthedocs: - echo "scp docs/* ???" - exit 1 +# automatically create a release on Gitlab for every tagged commit release_job: stage: publish image: registry.gitlab.com/gitlab-org/release-cli:latest diff --git a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml index ce46e6463265d2ad9704a24ae7f467cb7b50d8d9..c7c1b720cfc8d8d631b4c7c2640aebd0499b3a08 100644 --- a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml +++ b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml @@ -1,3 +1,7 @@ +# run several checks prior to creating a commit or pushing to a remote +# These can be skipped (but you shouldn't generally as the pipeline will fail) +# using `git push / commit --no-check` +# for more information see: https://pre-commit.com/ default_stages: [ pre-commit, pre-push ] default_language_version: python: python3 diff --git a/{{cookiecutter.project_slug}}/README.md b/{{cookiecutter.project_slug}}/README.md index 25f215207307771b9fefe5658653fe4816828721..612d22ac310c3e272dcfca9efed19c5a45be3c7a 100644 --- a/{{cookiecutter.project_slug}}/README.md +++ b/{{cookiecutter.project_slug}}/README.md @@ -57,5 +57,7 @@ To automatically apply most suggested linting changes execute: ```tox -e format``` +The configuration for linting and tox can be found in `pyproject.toml` + ## License This project is licensed under the Apache License Version 2.0 diff --git a/{{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh b/{{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh deleted file mode 100755 index 792a3aabef83dc4ebf0c94635ae1de0e7412c479..0000000000000000000000000000000000000000 --- a/{{cookiecutter.project_slug}}/bin/install-hooks/pre-commit.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if [ ! -f "setup.sh" ]; then - echo "pre-commit.sh must be executed with repository root as working directory!" - exit 1 -fi - -pre-commit install --hook-type pre-push diff --git a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile index 6063354805d6e2e35534ee5d09f80b51f856495b..5b45f3a8d8701901238be5c0b5509656b4b3c41f 100644 --- a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile +++ b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile @@ -1,3 +1,5 @@ +# This dockerfile is used throughout steps of the CI/CD pipeline +# It contains dependencies to run tests FROM python:3.13 RUN python -m pip install --upgrade pip diff --git a/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile b/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile index 1f7eb275020d13ed45cb7b0983949c12161ff953..e4f7edbedddadf9b39ff8192eda9c372fbe91e48 100644 --- a/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile +++ b/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile @@ -1,18 +1,28 @@ +# This dockerfile creates a minimal docker image that can run the project. +# It can copy the result of `tox -e build` from the host to the container +# or build from source inside the container. +# Set `docker build --build-arg BUILD_ENV=copy` to copy from the host, rebuild otherwise +# contrarily, to the ci image it does not contain any dependencies used for testing. ARG BUILD_ENV=no_copy +# Build without copying from host inside docker FROM python:3.13 AS build_no_copy ADD ../../requirements.txt . COPY ../.. /work -RUN rm -r /work/dist | true +RUN rm -rf /work/dist RUN python -m pip install --user tox WORKDIR /work RUN python -m tox -e build +# Copy build package from host `dist` directory FROM python:3.13 AS build_copy COPY dist /work/dist +# Dynamically set `build` from `build_no_copy` or `build_copy` depending on +# `BUILD_ENV` argument FROM build_${BUILD_ENV} AS build +# Copy build stage files into final python slim image and install FROM python:3.13-slim COPY --from=build /work/dist /dist RUN python -m pip install /dist/*.whl diff --git a/{{cookiecutter.project_slug}}/docs/cleanup.py b/{{cookiecutter.project_slug}}/docs/cleanup.py index 3a4508d859234544bea35b1008e3c8e4f73d7cc0..c88889067f1dc736a881d565ee02460f8cd05148 100644 --- a/{{cookiecutter.project_slug}}/docs/cleanup.py +++ b/{{cookiecutter.project_slug}}/docs/cleanup.py @@ -3,6 +3,16 @@ # Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +""" +Remove generated source documentation files except for index.rst + +If a source file is created, documentation is generated and the source file is later +removed. The documentation for this source file will persists. + +This file ensures generated source documentation files, which are automatically +generated from source, are removed between every build. +""" + import os file_dir = os.path.dirname(os.path.realpath(__file__)) @@ -15,7 +25,7 @@ if not os.path.exists(clean_dir): for file_name in os.listdir(clean_dir): file = os.path.join(clean_dir, file_name) - + if file_name == "index.rst": continue diff --git a/{{cookiecutter.project_slug}}/setup.sh b/{{cookiecutter.project_slug}}/setup.sh index 4548910b6ba00d9f564aea2ac5ddfab50c1d77db..6939c0502f2f95e42566748ce507829eb2190785 100755 --- a/{{cookiecutter.project_slug}}/setup.sh +++ b/{{cookiecutter.project_slug}}/setup.sh @@ -4,8 +4,10 @@ # SPDX-License-Identifier: Apache-2.0 # -# This file's directory is used to determine the station control directory -# location. +# Substitute BASH_SOURCE if unset this is required for simple shells +# such as the one found in docker or alpine docker images. +# `#! /usr/bin/env bash` does not actually ensure the sourcing is executed +# using BASH if [ -z ${BASH_SOURCE} ]; then BASH_SOURCE=${(%):-%x} fi @@ -24,7 +26,7 @@ source "$VENV_DIR/bin/activate" python -m pip install pre-commit python -m pip install "tox>=4.21.0" -# Install git pre-commit pre-push hook -if [ ! -f "${ABSOLUTE_PATH}/.git/hooks/pre-push.legacy" ]; then - source "${ABSOLUTE_PATH}/bin/install-hooks/pre-commit.sh" +# Install git pre-commit pre-push hook if not already installed +if [ ! -f "${ABSOLUTE_PATH}/.git/hooks/pre-push" ]; then + pre-commit install --hook-type pre-push fi