diff --git a/.gitlab-ci.astron.yml b/.gitlab-ci.astron.yml index 2cd6a43a2424c685cac80d52b1e0236a1b031e8c..93209d60aaa76b9ba75dde24d7e7ca6fafe6fbc7 100644 --- a/.gitlab-ci.astron.yml +++ b/.gitlab-ci.astron.yml @@ -1,81 +1,9 @@ # Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later +# At Astron GitLab, docker-in-docker jobs require a 'dind' tag. .dind: tags: - dind include: .gitlab-ci.common.yml - -.build-wheel: - stage: build - needs: [] - image: docker:20.10 - services: - - docker:20.10-dind - tags: - - das6 - before_script: - - apk add bash - script: - - cd docker - - USER=root ./make_wheels.sh $PYTHON_VERSION - artifacts: - paths: - - output-*/* - when: manual - allow_failure: true - -build-wheel-36: - extends: .build-wheel - variables: - PYTHON_VERSION: 36 - -build-wheel-37: - extends: .build-wheel - variables: - PYTHON_VERSION: 37 - -build-wheel-38: - extends: .build-wheel - variables: - PYTHON_VERSION: 38 - -build-wheel-39: - extends: .build-wheel - variables: - PYTHON_VERSION: 39 - -build-wheel-310: - extends: .build-wheel - variables: - PYTHON_VERSION: 310 - -.deploy-wheel: - stage: publish - image: python:latest - script: - - pip install twine - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi output-*/*.whl - when: manual - allow_failure: true - -deploy-wheel-36: - extends: .deploy-wheel - needs: ["build-wheel-36"] - -deploy-wheel-37: - extends: .deploy-wheel - needs: ["build-wheel-37"] - -deploy-wheel-38: - extends: .deploy-wheel - needs: ["build-wheel-38"] - -deploy-wheel-39: - extends: .deploy-wheel - needs: ["build-wheel-39"] - -deploy-wheel-310: - extends: .deploy-wheel - needs: ["build-wheel-310"] diff --git a/.gitlab-ci.common.yml b/.gitlab-ci.common.yml index 7e245cbd3fbe9ac1b1640b0cc0c3b5fe69011350..43852c7327ed892a206f525e88ae76d56a079128 100644 --- a/.gitlab-ci.common.yml +++ b/.gitlab-ci.common.yml @@ -27,6 +27,8 @@ stages: - publish - pages +include: .gitlab-ci.wheels.yml + # The 'IMAGE' variables allow reusing docker images between different pipelines. # See https://confluence.skatelescope.org/display/SE/Caching+Docker+images+using+GitLab+CI+registry versioning: diff --git a/.gitlab-ci.ska.yml b/.gitlab-ci.ska.yml index 4a282f5c2fe7fbd7ca32bc32a9a9c66457799ab9..8f4715f5776c32a2d11bdb189215737494bb88bf 100644 --- a/.gitlab-ci.ska.yml +++ b/.gitlab-ci.ska.yml @@ -1,6 +1,7 @@ # Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later +# At SKA GitLab, docker-in-docker jobs require a service. .dind: services: - docker:20.10-dind diff --git a/.gitlab-ci.wheels.yml b/.gitlab-ci.wheels.yml new file mode 100644 index 0000000000000000000000000000000000000000..2c1050e5a295e0befc1ecdb6413a085ea0b5e27b --- /dev/null +++ b/.gitlab-ci.wheels.yml @@ -0,0 +1,84 @@ +# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: GPL-3.0-or-later + +# GitLab CI jobs for building python wheels. +# These jobs are in a separate file since it's quite large. + +.build-wheel: + extends: .dind + stage: build + needs: [] + image: docker:20.10 + before_script: + - apk add bash + script: + - cd docker + - USER=root ./make_wheels.sh $PYTHON_VERSION + artifacts: + paths: + - output-*/* + when: manual + allow_failure: true + +build-wheel-37: + extends: .build-wheel + variables: + PYTHON_VERSION: 37 + +build-wheel-38: + extends: .build-wheel + variables: + PYTHON_VERSION: 38 + +build-wheel-39: + extends: .build-wheel + variables: + PYTHON_VERSION: 39 + +build-wheel-310: + extends: .build-wheel + variables: + PYTHON_VERSION: 310 + +build-wheel-311: + extends: .build-wheel + variables: + PYTHON_VERSION: 311 + +build-wheel-312: + extends: .build-wheel + variables: + PYTHON_VERSION: 312 + +.deploy-wheel: + stage: publish + image: python:latest + script: + - pip install twine + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi output-*/*.whl + when: manual + allow_failure: true + +deploy-wheel-37: + extends: .deploy-wheel + needs: ["build-wheel-37"] + +deploy-wheel-38: + extends: .deploy-wheel + needs: ["build-wheel-38"] + +deploy-wheel-39: + extends: .deploy-wheel + needs: ["build-wheel-39"] + +deploy-wheel-310: + extends: .deploy-wheel + needs: ["build-wheel-310"] + +deploy-wheel-311: + extends: .deploy-wheel + needs: ["build-wheel-311"] + +deploy-wheel-312: + extends: .deploy-wheel + needs: ["build-wheel-312"] diff --git a/CMakeLists.txt b/CMakeLists.txt index f1f759986fc21548d73ec9e7ffb5580c71694875..2f8a3cced41d01e30101d8cb2a17ad801a40e8af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -253,7 +253,7 @@ if(${EVERYBEAM_FOUND}) VERSION_GREATER_EQUAL "0.6.0") message( FATAL_ERROR - "DP3 needs EveryBeam version 0.5.x - with x >= 1 - but found version ${EveryBeam_VERSION}" + "DP3 needs EveryBeam version 0.5.x - with x >= 4 - but found version ${EveryBeam_VERSION}" ) endif() # TODO(AST-1336): Remove SYSTEM when EveryBeam no longer includes XTensor headers. @@ -337,6 +337,7 @@ add_subdirectory("${CMAKE_SOURCE_DIR}/external/pybind11") include_directories(${pybind11_INCLUDE_DIR}) # Include schaapcommon +set(SCHAAPCOMMON_MODULES facets h5parm) add_subdirectory("${CMAKE_SOURCE_DIR}/external/schaapcommon") include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/external/schaapcommon/include") diff --git a/docker/install_aoflagger.sh b/docker/install_aoflagger.sh index 312db6ebf20d30657935159e1204dcf23f666d9f..2ce1bf60a95332744f9020cb7ed8029f31f88183 100644 --- a/docker/install_aoflagger.sh +++ b/docker/install_aoflagger.sh @@ -3,12 +3,20 @@ # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later -# Script to install AOFlagger from source +# Script to install AOFlagger from source. set -euo pipefail +AOFLAGGER_VERSION=$1 +PYTHON_VERSION=$2 + +if [ -z "$PYTHON_VERSION" ]; then + echo "Usage: $0 <aoflagger version> <python version>" + exit 1 +fi + echo "Installing development libraries with yum" -yum -y install libpng-devel lua-devel zlib-devel +yum -y install libpng-devel zlib-devel pushd /tmp @@ -26,7 +34,7 @@ pushd aoflagger-v${AOFLAGGER_VERSION} sed -i '/find_package(PythonLibs 3 REQUIRED)/d' CMakeLists.txt # Ensure AOFlagger uses the correct python version and finds it before pybind11 does. -sed -i "s=# Include aocommon/pybind11 headers=find_package(PythonInterp ${PYMAJOR}.${PYMINOR} EXACT REQUIRED)=" CMakeLists.txt +sed -i "s=# Include aocommon/pybind11 headers=find_package(PythonInterp ${PYTHON_VERSION} EXACT REQUIRED)=" CMakeLists.txt # The patches below are already in the AOFlagger master, and should be removed # for AOFlagger > 3.2.0. See MR 204 and MR 205. @@ -97,4 +105,4 @@ rm aoflagger-v${AOFLAGGER_VERSION}.bz2 popd echo "Cleaning up development libraries" -yum -y erase libpng-devel lua-devel zlib-devel +yum -y erase libpng-devel zlib-devel diff --git a/docker/install_everybeam.sh b/docker/install_everybeam.sh index 72be9b46cb68cf1c4a074cb30f3b94448648ac93..b69e77e55c6b8028abee9c30e064ca4cc55ead73 100644 --- a/docker/install_everybeam.sh +++ b/docker/install_everybeam.sh @@ -7,7 +7,11 @@ set -euo pipefail -yum -y install wget +EVERYBEAM_VERSION=$1 + +# On CentOS7, the 'blas' package does not provide cblas functions. +# -> Install OpenBLAS, which does provide them. CMake will prefer OpenBLAS. +yum -y install openblas-devel wget pushd /tmp @@ -34,3 +38,6 @@ echo "Cleaning up unnecessary EveryBeam files" rm -r EveryBeam popd + +echo "Cleaning up development libraries" +yum -y erase openblas-devel wget diff --git a/docker/install_fftw.sh b/docker/install_fftw.sh index 280f073a4e02debbc16b5d28c8c9e53ff02ef212..7d1105ba9288bcadab5fe7afbe4e4ff33d419a31 100644 --- a/docker/install_fftw.sh +++ b/docker/install_fftw.sh @@ -7,6 +7,8 @@ set -euo pipefail +FFTW_VERSION=$1 + pushd /tmp echo "Downloading & unpacking FFTW ${FFTW_VERSION}" diff --git a/docker/install_hdf5.sh b/docker/install_hdf5.sh index 6b51e9fce635cec2af18990ea91654a4b56b4208..64d3abd0feec23d033f10b1ff91f495de3ebfa6f 100644 --- a/docker/install_hdf5.sh +++ b/docker/install_hdf5.sh @@ -3,10 +3,12 @@ # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later -# Script to install HDF5 from source +# Script to install HDF5 from source. CentOS 7 has a too old version. set -euo pipefail +HDF5_VERSION=$1 + echo "Installing zlib with yum" yum -y install zlib-devel @@ -36,4 +38,5 @@ rm hdf5-${HDF5_VERSION}.tar.gz popd +echo "Cleaning up development libraries" yum -y erase zlib-devel diff --git a/docker/install_lua.sh b/docker/install_lua.sh index 8c7e245f5a48891e4895ef6a38ec7d3450854eb1..82c3552315ab4724b0347a3e0b24d09d96f66389 100644 --- a/docker/install_lua.sh +++ b/docker/install_lua.sh @@ -3,9 +3,13 @@ # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later -# Script to install lua +# Script to install Lua 5.3, which is required for aoflagger. +# CentOS 7 only has Lua 5.1. + set -euo pipefail +LUA_VERSION=$1 + pushd /tmp echo "Downloading Lua" diff --git a/docker/make_wheels.sh b/docker/make_wheels.sh index 9ef5e9249cac43e59368d9c7696d72ae8a5f6497..238cbe7948b51562784345b96b4075896102feb7 100755 --- a/docker/make_wheels.sh +++ b/docker/make_wheels.sh @@ -12,16 +12,15 @@ # If <python versions> is empty, it becomes: 310 39 38 37 36 . set -euo pipefail -for py_version in ${@:-310 39 38 37 36}; do +for py_version in ${@:-312 311 310 39 38 37}; do pushd .. - # Use the Python3.10 dockerfile as a template, even for Python3.10. - dockerfile=$(mktemp make_wheels.docker.XXXXXX) - cat docker/py310_wheel.docker > $dockerfile - sed -i "s=\(casacore:master_wheel\)310=\1${py_version}=" $dockerfile - - ## Build docker image from docker-file. The current wheel is created there - docker build -t dp3-py${py_version}-$USER -f $dockerfile . + ## Build docker image from docker-file. The current wheel is created there. + [ ${py_version:1} -le 7 ] && py_unicode="m" || py_unicode= + docker build -t dp3-py${py_version}-$USER -f docker/py_wheel.docker \ + --build-arg PYMAJOR=${py_version:0:1} \ + --build-arg PYMINOR=${py_version:1} \ + --build-arg PYUNICODE=${py_unicode} . ## Create a docker container from that image, and extract the wheel containerid=$(docker create dp3-py${py_version}-$USER) echo "Docker container ID is: $containerid" @@ -29,6 +28,5 @@ for py_version in ${@:-310 39 38 37 36}; do docker rm ${containerid} docker image rm dp3-py${py_version}-$USER - rm $dockerfile popd done diff --git a/docker/py310_wheel.docker b/docker/py_wheel.docker similarity index 74% rename from docker/py310_wheel.docker rename to docker/py_wheel.docker index aebfd701759610f68255e67b160bddec1141acd9..d3cf06547d3b23a5f02caa4debc91ac45ef4c55f 100644 --- a/docker/py310_wheel.docker +++ b/docker/py_wheel.docker @@ -1,36 +1,46 @@ # Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: GPL-3.0-or-later -FROM quay.io/casacore/casacore:master_wheel310 +# Also put PYMAJOR and PYMINOR above 'FROM', so 'FROM' can use them. +ARG PYMAJOR +ARG PYMINOR -ENV AOFLAGGER_VERSION 3.2.0 -ENV EVERYBEAM_VERSION 0.5.4 -ENV HDF5_VERSION 1.12.2 -ENV FFTW_VERSION 3.3.8 -ENV LUA_VERSION 5.3.6 +FROM quay.io/casacore/casacore:py${PYMAJOR}${PYMINOR}_master + +ARG AOFLAGGER_VERSION=3.2.0 +ARG EVERYBEAM_VERSION=0.5.6 +ARG HDF5_VERSION=1.12.2 +ARG FFTW_VERSION=3.3.8 +ARG LUA_VERSION=5.3.6 +# ARG's must be repeated after each 'FROM' +ARG PYMAJOR +ARG PYMINOR +ARG PYUNICODE + +ENV THREADS=4 COPY docker/install_boost.sh / RUN bash /install_boost.sh COPY docker/install_fftw.sh / -RUN bash /install_fftw.sh +RUN bash /install_fftw.sh ${FFTW_VERSION} COPY docker/install_hdf5.sh / -RUN bash /install_hdf5.sh +RUN bash /install_hdf5.sh ${HDF5_VERSION} COPY docker/install_lua.sh / -RUN bash /install_lua.sh +RUN bash /install_lua.sh ${LUA_VERSION} # Create fake libpython to stop the linker from complaining. The wheel should find the user's libpython at runtime. RUN touch /usr/lib64/libpython${PYMAJOR}.${PYMINOR}${PYUNICODE}.so # EveryBeam uses FFTW, HDF5 COPY docker/install_everybeam.sh / -RUN bash /install_everybeam.sh +RUN bash /install_everybeam.sh ${EVERYBEAM_VERSION} # AOFlagger uses FFTW, HDF5, LUA. COPY docker/install_aoflagger.sh / -RUN bash /install_aoflagger.sh +RUN bash /install_aoflagger.sh ${AOFLAGGER_VERSION} ${PYMAJOR}.${PYMINOR} ADD . /dp3 WORKDIR /dp3 diff --git a/docker/ubuntu_20_04_base b/docker/ubuntu_20_04_base index 9ac805b3a182b214cc503d5c08464f7dc60c700e..3bfc8f9bb8b33062e9947762a183f7170ec00720 100644 --- a/docker/ubuntu_20_04_base +++ b/docker/ubuntu_20_04_base @@ -2,7 +2,7 @@ FROM ubuntu:20.04 # TODO: needs to be bumped before next DP3 release # ENV IDG_VERSION=0.8 -ENV EVERYBEAM_VERSION=v0.5.4 +ENV EVERYBEAM_VERSION=v0.5.6 ENV IDG_VERSION=6b61c038883ad3f807d20047c4f9e1a1f0b8d98a ENV AOFLAGGER_VERSION=65d5fba4f4c12797386d3fd9cd76734956a8b233 @@ -27,7 +27,6 @@ RUN export DEBIAN_FRONTEND="noninteractive" && \ libboost-test-dev \ libcfitsio-dev \ libfftw3-dev \ - libgsl-dev \ libgtkmm-3.0-dev \ libhdf5-serial-dev \ liblua5.3-dev \ diff --git a/docker/ubuntu_22_04_base b/docker/ubuntu_22_04_base index 76ce982c9901abd9fe5eb4453ad0579fcf6e9b28..5954af52d4daf0fe24e1a9f844e9ef685db8d061 100644 --- a/docker/ubuntu_22_04_base +++ b/docker/ubuntu_22_04_base @@ -2,7 +2,7 @@ FROM ubuntu:22.04 # TODO: needs to be bumped before next DP3 release # ENV IDG_VERSION=0.8 -ENV EVERYBEAM_VERSION=v0.5.4 +ENV EVERYBEAM_VERSION=v0.5.6 ENV IDG_VERSION=6b61c038883ad3f807d20047c4f9e1a1f0b8d98a ENV AOFLAGGER_VERSION=65d5fba4f4c12797386d3fd9cd76734956a8b233 @@ -29,7 +29,6 @@ RUN export DEBIAN_FRONTEND="noninteractive" && \ libboost-test-dev \ libcfitsio-dev \ libfftw3-dev \ - libgsl-dev \ libgtkmm-3.0-dev \ libhdf5-serial-dev \ liblua5.3-dev \ diff --git a/external/schaapcommon b/external/schaapcommon index 0b03827e4f1ac66245396152d2da255a3dcf50c5..3cc5c03cbc25f146be511518003d49a321d4d720 160000 --- a/external/schaapcommon +++ b/external/schaapcommon @@ -1 +1 @@ -Subproject commit 0b03827e4f1ac66245396152d2da255a3dcf50c5 +Subproject commit 3cc5c03cbc25f146be511518003d49a321d4d720