Merge branch 'initial-binary-wheel-setup' into 'main'

Initial binary wheel setup

See merge request !1

.gitlab-ci.yml

 include:
 - local: "{{cookiecutter.project_slug}}/.gitlab-ci.yml"
 
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+
 trigger_prepare:
   stage: prepare
   trigger:
@@ -11,9 +14,29 @@ default:
   # Bootstrap Cookiecutter template to test provided ci pipeline template
   before_script:
     - python --version # For debugging
+    - git config --global user.name "unit test"
+    - git config --global user.email "info@astron.nl"
     - cookiecutter --no-input --overwrite-if-exists --output-dir . .
     - cd my_awesome_app
-    - git init
+
+# Override unit test before script
+.run_unit_test_version_base:
+  before_script:
+    - pip install cookiecutter
+    - !reference [default, before_script]
+    - python --version # For debugging
+    - python -m pip install --upgrade pip
+    - pip install --upgrade tox twine
+
+# override package files before script
+package_files:
+  before_script:
+    - pip install cookiecutter
+    - !reference [default, before_script]
+  artifacts:
+    expire_in: 1w
+    paths:
+      - my_awesome_app/dist/*
 
 # Override artifact directories
 run_unit_tests_coverage:

README.md

-# Example Python Package
-
-![Build status](https://git.astron.nl/templates/python-package/badges/main/pipeline.svg)
-![Test coverage](https://git.astron.nl/templates/python-package/badges/main/coverage.svg)
-
-An example repository of an CI/CD pipeline for building, testing and publishing a python package.
-
-If you find some missing functionality with regards to CI/CD, testing, linting or something else, feel free to make a merge request with the proposed changes.
+# Example Python Binary Extension Wheel Package
+
+![Build status](https://git.astron.nl/templates/python-binary-wheel-package/badges/main/pipeline.svg)
+![Test coverage](https://git.astron.nl/templates/python-binary-wheel-package/badges/main/coverage.svg)
+
+An example repository of an CI/CD pipeline for building, testing and publishing
+a python package  that uses binary extensions compiled as wheels.
+
+If you find some missing functionality with regards to CI/CD, testing, linting
+or something else, feel free to make a merge request with the proposed changes.
+
+### Features:
+
+- Compile C / C++11 and bind with Python
+  - Easily define any C / C++ dependency to be included using `before-all = "apt / yum xxx"`
+  https://cibuildwheel.readthedocs.io/en/stable/options/#examples
+- Automatic versioning using semantic version tags (`git tag vX.X.X`)
+  - Versions from `setuptools-scm` in Python are bound and forwarded to the
+  C / C++ extension using the `DynamicVersion` git submodule.
+- Run any task performed by CI/CD locally such as:
+  - linting (`tox -e black`, `tox -e pylint`, `tox -e pep8`)
+    - automatically format code (`tox -e format`)
+  - unit tests (`tox -e py37`)
+  - code coverage (`tox -e coverage`)
+  - building & packaging (`tox -e build-local`, `tox -e build-ci-linux`)
+- Automatically publish new releases to the gitlab package repository
+- CI/CD uses [docker base image](docker/ci-runner/Dockerfile) to speed up pipeline
 
 ## How to apply this template
 
-This templates uses `cookiecutter` which can be
+This templates uses `cookiecutter` which can be installed easily:
 
 ```bash
 pip install --user cookiecutter
@@ -18,9 +37,10 @@ pip install --user cookiecutter
 Then you can create an instance of your template with:
 
 ```bash
-cookiecutter https://git.astron.nl/templates/python-package.git
+cookiecutter https://git.astron.nl/templates/python-binary-wheel-package.git
 # Next follow a set of prompts (such as the name and description of the package)
 ```
 
 ## License
+
 This project is licensed under the Apache License Version 2.0
\ No newline at end of file

cookiecutter.json

 {
     "project_name": "My Awesome App",
     "project_slug": "{{ cookiecutter.project_name.lower()|replace(' ', '_')|replace('-', '_')|replace('.', '_')|trim() }}",
+    "project_url": "git.astron.nl/{{cookiecutter.project_slug}}",
     "description": "An example package for CI/CD working group"
 }

hooks/post_gen_project.sh

+#!/bin/bash
+
+git init
+git submodule add https://github.com/LecrisUT/CMakeExtraUtils.git cmake/cmake-extra-utils
+git add --all
+git commit -m "initial commit" --no-edit
+git tag v0.0.1
\ No newline at end of file

{{cookiecutter.project_slug}}/.git_archival.txt

+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
+ref-names: $Format:%D$

{{cookiecutter.project_slug}}/.gitattributes

+.git_archival.txt  export-subst

{{cookiecutter.project_slug}}/.gitignore

@@ -6,8 +6,11 @@ dist/*
 coverage.xml
 htmlcov/*
 
-# Setuptools SCM
-{{cookiecutter.project_slug}}/_version.py
+# Tox
+.tox
+
+# setuptools-scm
+src/{{cookiecutter.project_slug}}/_version.py
 
 # IDE configuration
 .vscode

{{cookiecutter.project_slug}}/.gitlab-ci.yml

@@ -10,8 +10,6 @@ default:
 stages:
   - prepare
   - lint
-  # check if this needs to be a separate step
-  # - build_extensions
   - test
   - package
   - integration
@@ -47,11 +45,6 @@ run_pylint:
     - tox -e pylint
   allow_failure: true
 
-# build_extensions:
-#   stage: build_extensions
-#   script:
-#     - echo "build fortran/c/cpp extension source code"
-
 # Basic setup for all Python versions for which we don't have a base image
 .run_unit_test_version_base:
   before_script:
@@ -68,7 +61,7 @@ run_unit_tests:
     - tox -e py3${PY_VERSION}
   parallel:
     matrix: # use the matrix for testing
-      - PY_VERSION: [7, 8, 9, 10]
+      - PY_VERSION: [8, 9, 10, 11]
 
 # Run code coverage on the base image thus also performing unit tests
 run_unit_tests_coverage:
@@ -86,30 +79,37 @@ run_unit_tests_coverage:
 
 package_files:
   stage: package
+  image: python:3.8
   artifacts:
     expire_in: 1w
     paths:
       - dist/*
   script:
-    - tox -e build
+    - curl -sSL https://get.docker.com/ | sh
+    - python -m pip install cibuildwheel==2.13.1 cookiecutter
+    - cibuildwheel --platform linux --output-dir dist
 
 package_docs:
   stage: package
+  allow_failure: true
   artifacts:
     expire_in: 1w
     paths:
       - docs/* # update path to match the dest dir for documentation
   script:
     - echo "build and collect docs"
+    - exit 1
 
 run_integration_tests:
   stage: integration
+  allow_failure: true
   needs:
     - package_files
   script:
     - echo "make sure to move out of source dir"
     - echo "install package from filesystem (or use the artefact)"
     - echo "run against foreign systems (e.g. databases, cwl etc.)"
+    - exit 1
 
 publish_on_gitlab:
   stage: publish
@@ -163,6 +163,7 @@ publish_on_pypi:
 
 publish_to_readthedocs:
   stage: publish
+  allow_failure: true
   environment: readthedocs
   needs:
     - package_docs
@@ -171,3 +172,4 @@ publish_to_readthedocs:
     - if: $CI_COMMIT_TAG
   script:
     - echo "scp docs/* ???"
+    - exit 1

{{cookiecutter.project_slug}}/CMakeLists.txt

+cmake_minimum_required(VERSION 3.15...3.26)
+
+# Include submodule to dynamically determine version
+set(
+    CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake/cmake-extra-utils/cmake"
+)
+
+# Include script
+include(DynamicVersion)
+dynamic_version(
+    ${SKBUILD_PROJECT_NAME}
+    PROJECT_PREFIX PROJECT_VERSION_
+    PROJECT_SOURCE $ENV{DYNAMIC_VERSION_SOURCE}
+)
+
+# Get project name from scikit-build-core and version DynamicVersion
+project(${SKBUILD_PROJECT_NAME} VERSION ${PROJECT_VERSION})
+
+# Get required packages
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+find_package(pybind11 CONFIG REQUIRED)
+
+# Build our Pythonextension
+python_add_library(_core MODULE src/main.cpp WITH_SOABI)
+target_link_libraries(_core PRIVATE pybind11::headers)
+
+# Emit Python determined version as definition
+target_compile_definitions(_core PRIVATE VERSION_INFO=${PROJECT_VERSION})
+
+# Install the library into the build directory
+install(TARGETS _core DESTINATION python_binary_wheel)

{{cookiecutter.project_slug}}/MANIFEST.in

 include LICENSE
 include README.md
 
+include CMakeLists.txt
+
 recursive-include docs *
+recursive-include docker *
 recursive-exclude tests *

{{cookiecutter.project_slug}}/README.md

 # {{cookiecutter.project_name}}
 
-<!-- TODO: replace  -->
-![Build status](https://git.astron.nl/templates/python-package/badges/main/pipeline.svg)
-![Test coverage](https://git.astron.nl/templates/python-package/badges/main/coverage.svg)
-<!-- ![Latest release](https://git.astron.nl/templates/python-package/badges/main/release.svg) -->
+![Build status]({{cookiecutter.project_url}}/badges/main/pipeline.svg)
+![Test coverage]({{cookiecutter.project_url}}/badges/main/coverage.svg)
+![Latest release]({{cookiecutter.project_url}}/badges/main/release.svg)
 
-An example repository of an CI/CD pipeline for building, testing and publishing a python package.
+{{cookiecutter.description}}
 
 ## Installation
 ```

{{cookiecutter.project_slug}}/build-requirements.txt

+cibuildwheel >= 2.12.3 # BSD
+tox >= 4.0 # ?
+build >= 0.8.0 # MIT

{{cookiecutter.project_slug}}/pyproject.toml

 [build-system]
 requires = [
-    "setuptools>=45",
-    "setuptools_scm[toml]>=6.2",
-    "wheel"
+    "scikit-build-core>=0.3.3",
+    "pybind11",
+    "setuptools_scm[toml]>=6.2"
 ]
-build-backend = "setuptools.build_meta"
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "{{cookiecutter.project_slug}}"
+dynamic = ["version"]
+description="{{cookiecutter.description}}"
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+]
+
+[tool.scikit-build]
+experimental=true
+wheel.expand-macos-universal-tags = true
+metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
+sdist.include = ["src/{{cookiecutter.project_slug}}/_version.py"]
 
 [tool.setuptools_scm]
-write_to = "{{cookiecutter.project_slug}}/_version.py"
+write_to = "src/{{cookiecutter.project_slug}}/_version.py"
+
+[tool.cibuildwheel]
+skip = "pp*"
+build-verbosity = 1
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "universal2", "arm64"]
+
+[tool.cibuildwheel.linux]
+archs = ["x86_64", "i686"]
+
+[tool.cibuildwheel.windows]
+archs = ["AMD64", "x86"]
 
 [tool.pylint]
 ignore = "_version.py"
+
+[tool.flake8]
+max-line-length = 88
+extend-ignore = ["E203"]
+exclude = ["_version.py"]
+
+[tool.black]
+line-length = 88
+extend-exclude = '''
+(
+    _version.py
+)
+'''

{{cookiecutter.project_slug}}/setup.cfg

-[metadata]
-name = {{cookiecutter.project_slug}}
-description = An example package for CI/CD working group
-long_description = file: README.md
-long_description_content_type = text/markdown
-url = https://git.astron.nl/templates/python-package
-license = Apache License 2.0
-classifiers =
-    Development Status :: 3 - Alpha
-    Environment :: Web Environment
-    Intended Audience :: Developers
-    Intended Audience :: Science/Research
-    License :: OSI Approved :: Apache Software License
-    Operating System :: OS Independent
-    Programming Language :: Python
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.7
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-    Topic :: Internet :: WWW/HTTP
-    Topic :: Internet :: WWW/HTTP :: Dynamic Content
-    Topic :: Scientific/Engineering
-    Topic :: Scientific/Engineering :: Astronomy
-
-[options]
-include_package_data = true
-packages = find:
-python_requires = >=3.7
-install_requires =
-    importlib-metadata>=0.12, <5.0;python_version<"3.8"
-    numpy
-
-[flake8]
-max-line-length = 88
-extend-ignore = E203

{{cookiecutter.project_slug}}/setup.py

-import setuptools
-
-setuptools.setup()

{{cookiecutter.project_slug}}/src/main.cpp

+#include <pybind11/pybind11.h>
+
+#define STRINGIFY(x) #x
+#define MACRO_STRINGIFY(x) STRINGIFY(x)
+
+int add(int i, int j) {
+    return i + j;
+}
+
+namespace py = pybind11;
+
+PYBIND11_MODULE(_core, m) {
+    m.doc() = R"pbdoc(
+        Pybind11 example plugin
+        -----------------------
+
+        .. currentmodule:: python_binary_wheel
+
+        .. autosummary::
+           :toctree: _generate
+
+           add
+           subtract
+    )pbdoc";
+
+    m.def("add", &add, R"pbdoc(
+        Add two numbers
+
+        Some other explanation about the add function.
+    )pbdoc");
+
+    m.def("subtract", [](int i, int j) { return i - j; }, R"pbdoc(
+        Subtract two numbers
+
+        Some other explanation about the subtract function.
+    )pbdoc");
+}

{{cookiecutter.project_slug}}/tests/requirements.txt

@@ -2,6 +2,7 @@ autopep8 >= 1.7.0 # MIT
 black >= 22.0.0 # MIT
 build >= 0.8.0 # MIT
 flake8 >= 5.0.0 # MIT
+pyproject-flake8 >= 5.0.4 # Unlicense
 pylint >= 2.15.0 # GPLv2
 pytest >= 7.0.0 # MIT
 pytest-cov >= 3.0.0 # MIT

{{cookiecutter.project_slug}}/tox.ini

 [tox]
 # Generative environment list to test all supported Python versions
-envlist = py3{7,8,9,10},black,pep8,pylint
+envlist = py3{8,9,10,11},black,pep8,pylint
 minversion = 3.18.0
 
 [testenv]
-usedevelop = True
-package = wheel
-wheel_build_env = .pkg
+package = sdist  # 'Source' package required for binary extension
+use_develop = False  # use_develop implies 'editable' package, not possible
 
 setenv =
     LANGUAGE=en_US
     LC_ALL=en_US.UTF-8
     PYTHONWARNINGS=default::DeprecationWarning
+    DYNAMIC_VERSION_SOURCE={toxinidir}
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/tests/requirements.txt
@@ -31,16 +31,20 @@ usedevelop = False
 envdir = {toxworkdir}/linting
 commands =
     pep8: {envpython} -m flake8 --version
-    pep8: {envpython} -m flake8 --extend-exclude './.venv/','./venv/'
+    pep8: {envpython} -m flake8 src tests
     black: {envpython} -m black --version
-    black: {envpython} -m black --check --diff .
+    black: {envpython} -m black --check --diff src tests
     pylint: {envpython} -m pylint --version
-    pylint: {envpython} -m pylint {{cookiecutter.project_slug}} tests
-    format: {envpython} -m autopep8 -v -aa --in-place --recursive {{cookiecutter.project_slug}}
+    pylint: {envpython} -m pylint src tests
+    format: {envpython} -m autopep8 -v -aa --in-place --recursive src
     format: {envpython} -m autopep8 -v -aa --in-place --recursive tests
-    format: {envpython} -m black -v .
+    format: {envpython} -m black -v src tests
 
-
-[testenv:build]
-usedevelop = False
-commands = {envpython} -m build
+[testenv:{build-local,build-ci-linux}]
+deps =
+    -r{toxinidir}/tests/requirements.txt
+    -r{toxinidir}/build-requirements.txt
+skip_install = True
+commands =
+    build-local: {envpython} -m build
+    build-ci-linux: {envpython} -m cibuildwheel --platform linux --output-dir dist