diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..67114cc7b7ca0d1b907c02cdee71a1876deb470c
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,4 @@
+.tox
+build
+*.egg-info
+venv
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..485dee64bcfb48793379b200a1afd14e85a8aaf4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+.idea
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 80cc459ceea7f1a10da18ade34b39d52cb29f81e..241614390b38a3321f8fcf818592e76f9590ef3f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,7 @@
-include:
-- local: "{{cookiecutter.project_slug}}/.gitlab-ci.yml"
+stages:
+ - prepare
+ - build
+ - test
trigger_prepare:
stage: prepare
@@ -7,41 +9,33 @@ trigger_prepare:
strategy: depend
include: "{{cookiecutter.project_slug}}/.prepare.gitlab-ci.yml"
-default:
+# Generate template instance in my_awesome_app directory
+build-template:
+ stage: build
+ image: $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG
# Bootstrap Cookiecutter template to test provided ci pipeline template
- before_script:
+ script:
- python --version # For debugging
- cookiecutter --no-input --overwrite-if-exists --output-dir . .
- cd my_awesome_app
- git init
-
-# Override semgrep-sast before script
-sast:
- before_script:
- - python --version # For debugging
-
- # Override unit test before script
-.run_unit_test_version_base:
- before_script:
- - pip install cookiecutter
- - !reference [default, before_script]
- - python -m pip install --upgrade pip
- - pip install --upgrade tox twine
-
-# Override artifact directories
-run_unit_tests_coverage:
+ # cannot use needs, for artifacts on child pipeline so must regenerate template!
artifacts:
- reports:
- coverage_report:
- coverage_format: cobertura
- path: my_awesome_app/coverage.xml
paths:
- - my_awesome_app/htmlcov/*
+ - my_awesome_app/*
+ - project.gitlab-ci.yml
-# Override artifact directories
-package_docs:
- stage: package
- artifacts:
- expire_in: 1w
- paths:
- - my_awesome_app/docs/build/*
\ No newline at end of file
+# Spawn pipeline using the gitlab-ci.yml from generated template instance
+# use project.gitlab.ci.yml for necessary job overrides from this template instance
+# (due to changes in directories and paths etc)
+project-pipeline:
+ stage: test
+ trigger:
+ strategy: depend
+ include:
+ - artifact: my_awesome_app/.gitlab-ci.yml
+ job: build-template
+ - artifact: project.gitlab-ci.yml
+ job: build-template
+ variables:
+ PARENT_PIPELINE_ID: $CI_PIPELINE_ID
diff --git a/README.md b/README.md
index 66d754d4362563e3e7fb345c60dda411928984f8..2b2be85db8be99f004c7fec6589e8873d62d6bac 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,9 @@
-# Example Python Package
+# Python Package Template
-An example repository of an CI/CD pipeline for building, testing and publishing a python package.
+Template to create Python repositories with CI/CD pipeline for building, testing and publishing a python package.
If you find some missing functionality with regards to CI/CD, testing, linting or something else, feel free to make a merge request with the proposed changes.
@@ -30,5 +30,13 @@ pages to configure Gitlab appropriately:
1. [Gitlab Repository Configuration](https://git.astron.nl/groups/templates/-/wikis/Gitlab-Repository-Configuration)
2. [Continuous delivery guideline](https://git.astron.nl/groups/templates/-/wikis/Continuous%20Delivery%20Guideline)
+## Setup
+
+Once you have used the template there are some additional steps to fully use this
+repository on Gitlab.
+
+1. [Cleanup Docker Registry Images](https://git.astron.nl/groups/templates/-/wikis/Cleanup-Docker-Registry-Images)
+2. [Setup Protected Verson Tags](https://git.astron.nl/groups/templates/-/wikis/Setting-up-Protected-Version-Tags)
+
## License
This project is licensed under the Apache License Version 2.0
diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile
index c9d9fcd37c84155b081ff8c5a1ddcb6a93154a28..a734451a11533307c67551452655bf6951d7235d 100644
--- a/docker/ci-runner/Dockerfile
+++ b/docker/ci-runner/Dockerfile
@@ -1,4 +1,4 @@
FROM python:3.12
RUN python -m pip install --upgrade pip
-RUN pip install --upgrade cookiecutter tox twine
+RUN python -m pip install --upgrade cookiecutter tox twine
diff --git a/project.gitlab-ci.yml b/project.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2d6eb4238cb4eed070a86b57386656f0806b1f4b
--- /dev/null
+++ b/project.gitlab-ci.yml
@@ -0,0 +1,131 @@
+
+# This file overrides all the jobs defined in {{cookiecutter.project_slug}}/.gitlab.ci-yml
+# this is to ensure they depend on the template installation artifact of the root
+# .gitlab-ci.yml from job `build-template`
+# The generated gitlab-ci.yml from this `build-template` job is used for the actual
+# trigger include to prevent including jobs that still contain template arguments
+
+trigger_prepare:
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+ when: never
+
+default:
+ # Bootstrap Cookiecutter template to test provided ci pipeline template
+ before_script:
+ - cd my_awesome_app
+
+run_black:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+run_flake8:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+run_pylint:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+sast:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+dependency_scanning:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+secret_detection:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+.run_unit_test_version_base:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+# Run all unit tests for Python versions except the base image
+run_unit_tests:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+
+# Run code coverage on the base image thus also performing unit tests
+run_unit_tests_coverage:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ artifacts:
+ reports:
+ coverage_report:
+ coverage_format: cobertura
+ path: my_awesome_app/coverage.xml
+ paths:
+ - my_awesome_app/htmlcov/*
+
+package_files:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ artifacts:
+ expire_in: 1w
+ paths:
+ - my_awesome_app/dist/*
+
+package_docs:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ artifacts:
+ expire_in: 1w
+ paths:
+ - my_awesome_app/docs/build/*
+
+docker_build:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ - package_files
+ before_script:
+ - cd my_awesome_app
+
+run_integration_tests:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ - package_files
+
+publish_on_gitlab:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ - package_files
+
+publish_on_test_pypi:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ - package_files
+
+publish_on_pypi:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ - package_files
+
+publish_to_readthedocs:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
+ - package_docs
+
+release_job:
+ needs:
+ - pipeline: $PARENT_PIPELINE_ID
+ job: build-template
\ No newline at end of file
diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml
index 9db35b850e19fdb2099e3f4f08e2981c92ce4790..93fffaca4e6fa5cee5b6ad53e29496bcaba216b1 100644
--- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml
+++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml
@@ -14,6 +14,7 @@ stages:
# - build_extensions
- test
- package
+ - images
- integration
- publish # publish instead of deploy
@@ -77,7 +78,7 @@ secret_detection:
before_script:
- python --version # For debugging
- python -m pip install --upgrade pip
- - pip install --upgrade tox twine
+ - python -m pip install --upgrade tox twine
# Run all unit tests for Python versions except the base image
run_unit_tests:
@@ -122,6 +123,20 @@ package_docs:
script:
- tox -e docs
+docker_build:
+ stage: images
+ image: docker:latest
+ needs:
+ - package_files
+ tags:
+ - dind
+ before_script: []
+ script:
+ - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - docker build -f docker/{{cookiecutter.project_slug}}/Dockerfile . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/{{cookiecutter.project_slug}}:$CI_COMMIT_REF_SLUG
+ # enable this push line once you have configured docker registry cleanup policy
+ # - docker push $CI_REGISTRY_IMAGE/{{cookiecutter.project_slug}}:$CI_COMMIT_REF_SLUG
+
run_integration_tests:
stage: integration
allow_failure: true
diff --git a/{{cookiecutter.project_slug}}/README.md b/{{cookiecutter.project_slug}}/README.md
index d542721824fc80436eb41df7896ef9e372741fa7..12a816d59c4e128a37a07c3ab27c26a11d44e94c 100644
--- a/{{cookiecutter.project_slug}}/README.md
+++ b/{{cookiecutter.project_slug}}/README.md
@@ -11,6 +11,17 @@ An example repository of an CI/CD pipeline for building, testing and publishing
pip install .
```
+## Setup
+
+One time template setup should include configuring the docker registry to regularly cleanup old images of
+the CI/CD pipelines. And you can consider creating protected version tags for software releases:
+
+1. [Cleanup Docker Registry Images](https://git.astron.nl/groups/templates/-/wikis/Cleanup-Docker-Registry-Images)
+2. [Setup Protected Verson Tags](https://git.astron.nl/groups/templates/-/wikis/Setting-up-Protected-Version-Tags)
+
+Once the cleanup policy for docker registry is setup you can uncomment the `docker push` comment in the `.gitlab-ci.yml`
+file from the `docker_build` job. This will allow to download minimal docker images with your Python package installed.
+
## Usage
```python
from {{cookiecutter.project_slug}} import cool_module
diff --git a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile
index 6268a1aa5f08de11246abd0fabbb456e2862af84..6cb46437656aad64d8292c0a10f6edaffffba8e3 100644
--- a/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile
+++ b/{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile
@@ -1,4 +1,4 @@
FROM python:3.12
RUN python -m pip install --upgrade pip
-RUN pip install --upgrade tox twine
+RUN python -m pip install --upgrade tox twine
diff --git a/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile b/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..ceea7e721fc3acaa124184e709986e04dbd9a3dd
--- /dev/null
+++ b/{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile
@@ -0,0 +1,18 @@
+ARG BUILD_ENV=no_copy
+
+FROM python:3.11 AS build_no_copy
+ADD ../../requirements.txt .
+COPY ../.. /work
+RUN rm -r /work/dist | true
+RUN python -m pip install --user tox
+WORKDIR /work
+RUN python -m tox -e build
+
+FROM python:3.11 AS build_copy
+COPY dist /work/dist
+
+FROM build_${BUILD_ENV} AS build
+
+FROM python:3.11-slim
+COPY --from=build /work/dist /dist
+RUN python -m pip install /dist/*.whl
diff --git a/{{cookiecutter.project_slug}}/setup.cfg b/{{cookiecutter.project_slug}}/setup.cfg
index 812f09cde700832f430ae7e3795420d614fc146e..ac0ae10fd9566ad1f8cb48ca1c780dc3b5be040e 100644
--- a/{{cookiecutter.project_slug}}/setup.cfg
+++ b/{{cookiecutter.project_slug}}/setup.cfg
@@ -28,7 +28,7 @@ classifiers =
[options]
include_package_data = true
packages = find:
-python_requires = >=3.7
+python_requires = >=3.8
install_requires = file: requirements.txt
[flake8]
diff --git a/{{cookiecutter.project_slug}}/tox.ini b/{{cookiecutter.project_slug}}/tox.ini
index dcb00e1b4ae1b4ce51301cf3d456a7d90cb8c5e1..871b5debda40fc76c1dc1cf36cac110e05c926eb 100644
--- a/{{cookiecutter.project_slug}}/tox.ini
+++ b/{{cookiecutter.project_slug}}/tox.ini
@@ -53,4 +53,5 @@ commands =
[testenv:build]
usedevelop = False
+deps = build
commands = {envpython} -m build