Add docker image build and refactor template pipeline inclusion

.dockerignore

+.tox
+build
+*.egg-info
+venv

.gitignore

+.idea

.gitlab-ci.yml

-include:
-- local: "{{cookiecutter.project_slug}}/.gitlab-ci.yml"
+stages:
+  - prepare
+  - build
+  - test
 
 trigger_prepare:
   stage: prepare
@@ -7,41 +9,33 @@ trigger_prepare:
     strategy: depend
     include: "{{cookiecutter.project_slug}}/.prepare.gitlab-ci.yml"
 
-default:
+# Generate template instance in my_awesome_app directory
+build-template:
+  stage: build
+  image: $CI_REGISTRY_IMAGE/ci-build-runner:$CI_COMMIT_REF_SLUG
   # Bootstrap Cookiecutter template to test provided ci pipeline template
-  before_script:
+  script:
     - python --version # For debugging
     - cookiecutter --no-input --overwrite-if-exists --output-dir . .
     - cd my_awesome_app
     - git init
-
-# Override semgrep-sast before script
-sast:
-  before_script:
-    - python --version # For debugging
-
-    # Override unit test before script
-.run_unit_test_version_base:
-  before_script:
-    - pip install cookiecutter
-    - !reference [default, before_script]
-    - python -m pip install --upgrade pip
-    - pip install --upgrade tox twine
-
-# Override artifact directories
-run_unit_tests_coverage:
+  # cannot use needs, for artifacts on child pipeline so must regenerate template!
   artifacts:
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: my_awesome_app/coverage.xml
     paths:
-      - my_awesome_app/htmlcov/*
+      - my_awesome_app/*
+      - project.gitlab-ci.yml
 
-# Override artifact directories
-package_docs:
-  stage: package
-  artifacts:
-    expire_in: 1w
-    paths:
-      - my_awesome_app/docs/build/*
+# Spawn pipeline using the gitlab-ci.yml from generated template instance
+# use project.gitlab.ci.yml for necessary job overrides from this template instance
+# (due to changes in directories and paths etc)
+project-pipeline:
+  stage: test
+  trigger:
+    strategy: depend
\ No newline at end of file
+    include:
+      - artifact: my_awesome_app/.gitlab-ci.yml
+        job: build-template
+      - artifact: project.gitlab-ci.yml
+        job: build-template
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID

README.md

-# Example Python Package
+# Python Package Template
 
 ![Build status](https://git.astron.nl/templates/python-package/badges/main/pipeline.svg)
 ![Test coverage](https://git.astron.nl/templates/python-package/badges/main/coverage.svg)
 
-An example repository of an CI/CD pipeline for building, testing and publishing a python package.
+Template to create Python repositories with CI/CD pipeline for building, testing and publishing a python package.
 
 If you find some missing functionality with regards to CI/CD, testing, linting or something else, feel free to make a merge request with the proposed changes.
 
@@ -30,5 +30,13 @@ pages to configure Gitlab appropriately:
 1. [Gitlab Repository Configuration](https://git.astron.nl/groups/templates/-/wikis/Gitlab-Repository-Configuration)
 2. [Continuous delivery guideline](https://git.astron.nl/groups/templates/-/wikis/Continuous%20Delivery%20Guideline)
 
+## Setup
+
+Once you have used the template there are some additional steps to fully use this
+repository on Gitlab.
+
+1. [Cleanup Docker Registry Images](https://git.astron.nl/groups/templates/-/wikis/Cleanup-Docker-Registry-Images)
+2. [Setup Protected Verson Tags](https://git.astron.nl/groups/templates/-/wikis/Setting-up-Protected-Version-Tags)
+
 ## License
 This project is licensed under the Apache License Version 2.0

docker/ci-runner/Dockerfile

 FROM python:3.12
 
 RUN python -m pip install --upgrade pip
-RUN pip install --upgrade cookiecutter tox twine
+RUN python -m pip install --upgrade cookiecutter tox twine

project.gitlab-ci.yml

+
+# This file overrides all the jobs defined in {{cookiecutter.project_slug}}/.gitlab.ci-yml
+# this is to ensure they depend on the template installation artifact of the root
+# .gitlab-ci.yml from job `build-template`
+# The generated gitlab-ci.yml from this `build-template` job is used for the actual
+# trigger include to prevent including jobs that still contain template arguments
+
+trigger_prepare:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+      when: never
+
+default:
+  # Bootstrap Cookiecutter template to test provided ci pipeline template
+  before_script:
+    - cd my_awesome_app
+
+run_black:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+run_flake8:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+run_pylint:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+sast:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+dependency_scanning:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+secret_detection:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+.run_unit_test_version_base:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+# Run all unit tests for Python versions except the base image
+run_unit_tests:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+
+# Run code coverage on the base image thus also performing unit tests
+run_unit_tests_coverage:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: my_awesome_app/coverage.xml
+    paths:
+      - my_awesome_app/htmlcov/*
+
+package_files:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+  artifacts:
+    expire_in: 1w
+    paths:
+      - my_awesome_app/dist/*
+
+package_docs:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+  artifacts:
+    expire_in: 1w
+    paths:
+      - my_awesome_app/docs/build/*
+
+docker_build:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+    - package_files
+  before_script:
+    - cd my_awesome_app
+
+run_integration_tests:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+    - package_files
+
+publish_on_gitlab:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+    - package_files
+
+publish_on_test_pypi:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+    - package_files
+
+publish_on_pypi:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+    - package_files
+
+publish_to_readthedocs:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
+    - package_docs
+
+release_job:
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build-template
\ No newline at end of file

{{cookiecutter.project_slug}}/.gitlab-ci.yml

@@ -14,6 +14,7 @@ stages:
   # - build_extensions
   - test
   - package
+  - images
   - integration
   - publish # publish instead of deploy
 
@@ -77,7 +78,7 @@ secret_detection:
   before_script:
     - python --version # For debugging
     - python -m pip install --upgrade pip
-    - pip install --upgrade tox twine
+    - python -m pip install --upgrade tox twine
 
 # Run all unit tests for Python versions except the base image
 run_unit_tests:
@@ -122,6 +123,20 @@ package_docs:
   script:
     - tox -e docs
 
+docker_build:
+  stage: images
+  image: docker:latest
+  needs:
+    - package_files
+  tags:
+    - dind
+  before_script: []
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -f docker/{{cookiecutter.project_slug}}/Dockerfile . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/{{cookiecutter.project_slug}}:$CI_COMMIT_REF_SLUG
+    # enable this push line once you have configured docker registry cleanup policy
+    # - docker push $CI_REGISTRY_IMAGE/{{cookiecutter.project_slug}}:$CI_COMMIT_REF_SLUG
+
 run_integration_tests:
   stage: integration
   allow_failure: true

{{cookiecutter.project_slug}}/README.md

@@ -11,6 +11,17 @@ An example repository of an CI/CD pipeline for building, testing and publishing
 pip install .
 ```
 
+## Setup
+
+One time template setup should include configuring the docker registry to regularly cleanup old images of
+the CI/CD pipelines. And you can consider creating protected version tags for software releases:
+
+1. [Cleanup Docker Registry Images](https://git.astron.nl/groups/templates/-/wikis/Cleanup-Docker-Registry-Images)
+2. [Setup Protected Verson Tags](https://git.astron.nl/groups/templates/-/wikis/Setting-up-Protected-Version-Tags)
+
+Once the cleanup policy for docker registry is setup you can uncomment the `docker push` comment in the `.gitlab-ci.yml`
+file from the `docker_build` job. This will allow to download minimal docker images with your Python package installed.
+
 ## Usage
 ```python
 from {{cookiecutter.project_slug}} import cool_module

{{cookiecutter.project_slug}}/docker/ci-runner/Dockerfile

 FROM python:3.12
 
 RUN python -m pip install --upgrade pip
-RUN pip install --upgrade tox twine
+RUN python -m pip install --upgrade tox twine

{{cookiecutter.project_slug}}/docker/{{cookiecutter.project_slug}}/Dockerfile

+ARG BUILD_ENV=no_copy
+
+FROM python:3.11 AS build_no_copy
+ADD ../../requirements.txt .
+COPY ../.. /work
+RUN rm -r /work/dist | true
+RUN python -m pip install --user tox
+WORKDIR /work
+RUN python -m tox -e build
+
+FROM python:3.11 AS build_copy
+COPY dist /work/dist
+
+FROM build_${BUILD_ENV} AS build
+
+FROM python:3.11-slim
+COPY --from=build /work/dist /dist
+RUN python -m pip install /dist/*.whl

{{cookiecutter.project_slug}}/setup.cfg

@@ -28,7 +28,7 @@ classifiers =
 [options]
 include_package_data = true
 packages = find:
-python_requires = >=3.7
+python_requires = >=3.8
 install_requires = file: requirements.txt
 
 [flake8]

{{cookiecutter.project_slug}}/tox.ini

@@ -53,4 +53,5 @@ commands =
 
 [testenv:build]
 usedevelop = False
+deps = build
 commands = {envpython} -m build