Skip to content
Snippets Groups Projects
Commit ccff8636 authored by Maik Nijhuis's avatar Maik Nijhuis
Browse files

AST-1495 Remove GSL dependency

parent 66525837
No related branches found
No related tags found
No related merge requests found
# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# At Astron GitLab, docker-in-docker jobs require a 'dind' tag.
.dind:
tags:
- dind
include: .gitlab-ci.common.yml
.build-wheel:
stage: build
needs: []
image: docker:20.10
services:
- docker:20.10-dind
tags:
- das6
before_script:
- apk add bash
script:
- cd docker
- USER=root ./make_wheels.sh $PYTHON_VERSION
artifacts:
paths:
- output-*/*
when: manual
allow_failure: true
build-wheel-36:
extends: .build-wheel
variables:
PYTHON_VERSION: 36
build-wheel-37:
extends: .build-wheel
variables:
PYTHON_VERSION: 37
build-wheel-38:
extends: .build-wheel
variables:
PYTHON_VERSION: 38
build-wheel-39:
extends: .build-wheel
variables:
PYTHON_VERSION: 39
build-wheel-310:
extends: .build-wheel
variables:
PYTHON_VERSION: 310
.deploy-wheel:
stage: publish
image: python:latest
script:
- pip install twine
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi output-*/*.whl
when: manual
allow_failure: true
deploy-wheel-36:
extends: .deploy-wheel
needs: ["build-wheel-36"]
deploy-wheel-37:
extends: .deploy-wheel
needs: ["build-wheel-37"]
deploy-wheel-38:
extends: .deploy-wheel
needs: ["build-wheel-38"]
deploy-wheel-39:
extends: .deploy-wheel
needs: ["build-wheel-39"]
deploy-wheel-310:
extends: .deploy-wheel
needs: ["build-wheel-310"]
......@@ -27,6 +27,8 @@ stages:
- publish
- pages
include: .gitlab-ci.wheels.yml
# The 'IMAGE' variables allow reusing docker images between different pipelines.
# See https://confluence.skatelescope.org/display/SE/Caching+Docker+images+using+GitLab+CI+registry
versioning:
......
# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# At SKA GitLab, docker-in-docker jobs require a service.
.dind:
services:
- docker:20.10-dind
......
# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# GitLab CI jobs for building python wheels.
# These jobs are in a separate file since it's quite large.
.build-wheel:
extends: .dind
stage: build
needs: []
image: docker:20.10
before_script:
- apk add bash
script:
- cd docker
- USER=root ./make_wheels.sh $PYTHON_VERSION
artifacts:
paths:
- output-*/*
when: manual
allow_failure: true
build-wheel-37:
extends: .build-wheel
variables:
PYTHON_VERSION: 37
build-wheel-38:
extends: .build-wheel
variables:
PYTHON_VERSION: 38
build-wheel-39:
extends: .build-wheel
variables:
PYTHON_VERSION: 39
build-wheel-310:
extends: .build-wheel
variables:
PYTHON_VERSION: 310
build-wheel-311:
extends: .build-wheel
variables:
PYTHON_VERSION: 311
build-wheel-312:
extends: .build-wheel
variables:
PYTHON_VERSION: 312
.deploy-wheel:
stage: publish
image: python:latest
script:
- pip install twine
- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi output-*/*.whl
when: manual
allow_failure: true
deploy-wheel-37:
extends: .deploy-wheel
needs: ["build-wheel-37"]
deploy-wheel-38:
extends: .deploy-wheel
needs: ["build-wheel-38"]
deploy-wheel-39:
extends: .deploy-wheel
needs: ["build-wheel-39"]
deploy-wheel-310:
extends: .deploy-wheel
needs: ["build-wheel-310"]
deploy-wheel-311:
extends: .deploy-wheel
needs: ["build-wheel-311"]
deploy-wheel-312:
extends: .deploy-wheel
needs: ["build-wheel-312"]
......@@ -253,7 +253,7 @@ if(${EVERYBEAM_FOUND})
VERSION_GREATER_EQUAL "0.6.0")
message(
FATAL_ERROR
"DP3 needs EveryBeam version 0.5.x - with x >= 1 - but found version ${EveryBeam_VERSION}"
"DP3 needs EveryBeam version 0.5.x - with x >= 4 - but found version ${EveryBeam_VERSION}"
)
endif()
# TODO(AST-1336): Remove SYSTEM when EveryBeam no longer includes XTensor headers.
......@@ -337,6 +337,7 @@ add_subdirectory("${CMAKE_SOURCE_DIR}/external/pybind11")
include_directories(${pybind11_INCLUDE_DIR})
# Include schaapcommon
set(SCHAAPCOMMON_MODULES facets h5parm)
add_subdirectory("${CMAKE_SOURCE_DIR}/external/schaapcommon")
include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/external/schaapcommon/include")
......
......@@ -3,12 +3,20 @@
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# Script to install AOFlagger from source
# Script to install AOFlagger from source.
set -euo pipefail
AOFLAGGER_VERSION=$1
PYTHON_VERSION=$2
if [ -z "$PYTHON_VERSION" ]; then
echo "Usage: $0 <aoflagger version> <python version>"
exit 1
fi
echo "Installing development libraries with yum"
yum -y install libpng-devel lua-devel zlib-devel
yum -y install libpng-devel zlib-devel
pushd /tmp
......@@ -26,7 +34,7 @@ pushd aoflagger-v${AOFLAGGER_VERSION}
sed -i '/find_package(PythonLibs 3 REQUIRED)/d' CMakeLists.txt
# Ensure AOFlagger uses the correct python version and finds it before pybind11 does.
sed -i "s=# Include aocommon/pybind11 headers=find_package(PythonInterp ${PYMAJOR}.${PYMINOR} EXACT REQUIRED)=" CMakeLists.txt
sed -i "s=# Include aocommon/pybind11 headers=find_package(PythonInterp ${PYTHON_VERSION} EXACT REQUIRED)=" CMakeLists.txt
# The patches below are already in the AOFlagger master, and should be removed
# for AOFlagger > 3.2.0. See MR 204 and MR 205.
......@@ -97,4 +105,4 @@ rm aoflagger-v${AOFLAGGER_VERSION}.bz2
popd
echo "Cleaning up development libraries"
yum -y erase libpng-devel lua-devel zlib-devel
yum -y erase libpng-devel zlib-devel
......@@ -7,7 +7,11 @@
set -euo pipefail
yum -y install wget
EVERYBEAM_VERSION=$1
# On CentOS7, the 'blas' package does not provide cblas functions.
# -> Install OpenBLAS, which does provide them. CMake will prefer OpenBLAS.
yum -y install openblas-devel wget
pushd /tmp
......@@ -34,3 +38,6 @@ echo "Cleaning up unnecessary EveryBeam files"
rm -r EveryBeam
popd
echo "Cleaning up development libraries"
yum -y erase openblas-devel wget
......@@ -7,6 +7,8 @@
set -euo pipefail
FFTW_VERSION=$1
pushd /tmp
echo "Downloading & unpacking FFTW ${FFTW_VERSION}"
......
......@@ -3,10 +3,12 @@
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# Script to install HDF5 from source
# Script to install HDF5 from source. CentOS 7 has a too old version.
set -euo pipefail
HDF5_VERSION=$1
echo "Installing zlib with yum"
yum -y install zlib-devel
......@@ -36,4 +38,5 @@ rm hdf5-${HDF5_VERSION}.tar.gz
popd
echo "Cleaning up development libraries"
yum -y erase zlib-devel
......@@ -3,9 +3,13 @@
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
# Script to install lua
# Script to install Lua 5.3, which is required for aoflagger.
# CentOS 7 only has Lua 5.1.
set -euo pipefail
LUA_VERSION=$1
pushd /tmp
echo "Downloading Lua"
......
......@@ -12,16 +12,15 @@
# If <python versions> is empty, it becomes: 310 39 38 37 36 .
set -euo pipefail
for py_version in ${@:-310 39 38 37 36}; do
for py_version in ${@:-312 311 310 39 38 37}; do
pushd ..
# Use the Python3.10 dockerfile as a template, even for Python3.10.
dockerfile=$(mktemp make_wheels.docker.XXXXXX)
cat docker/py310_wheel.docker > $dockerfile
sed -i "s=\(casacore:master_wheel\)310=\1${py_version}=" $dockerfile
## Build docker image from docker-file. The current wheel is created there
docker build -t dp3-py${py_version}-$USER -f $dockerfile .
## Build docker image from docker-file. The current wheel is created there.
[ ${py_version:1} -le 7 ] && py_unicode="m" || py_unicode=
docker build -t dp3-py${py_version}-$USER -f docker/py_wheel.docker \
--build-arg PYMAJOR=${py_version:0:1} \
--build-arg PYMINOR=${py_version:1} \
--build-arg PYUNICODE=${py_unicode} .
## Create a docker container from that image, and extract the wheel
containerid=$(docker create dp3-py${py_version}-$USER)
echo "Docker container ID is: $containerid"
......@@ -29,6 +28,5 @@ for py_version in ${@:-310 39 38 37 36}; do
docker rm ${containerid}
docker image rm dp3-py${py_version}-$USER
rm $dockerfile
popd
done
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
FROM quay.io/casacore/casacore:master_wheel310
# Also put PYMAJOR and PYMINOR above 'FROM', so 'FROM' can use them.
ARG PYMAJOR
ARG PYMINOR
ENV AOFLAGGER_VERSION 3.2.0
ENV EVERYBEAM_VERSION 0.5.4
ENV HDF5_VERSION 1.12.2
ENV FFTW_VERSION 3.3.8
ENV LUA_VERSION 5.3.6
FROM quay.io/casacore/casacore:py${PYMAJOR}${PYMINOR}_master
ARG AOFLAGGER_VERSION=3.2.0
ARG EVERYBEAM_VERSION=0.5.6
ARG HDF5_VERSION=1.12.2
ARG FFTW_VERSION=3.3.8
ARG LUA_VERSION=5.3.6
# ARG's must be repeated after each 'FROM'
ARG PYMAJOR
ARG PYMINOR
ARG PYUNICODE
ENV THREADS=4
COPY docker/install_boost.sh /
RUN bash /install_boost.sh
COPY docker/install_fftw.sh /
RUN bash /install_fftw.sh
RUN bash /install_fftw.sh ${FFTW_VERSION}
COPY docker/install_hdf5.sh /
RUN bash /install_hdf5.sh
RUN bash /install_hdf5.sh ${HDF5_VERSION}
COPY docker/install_lua.sh /
RUN bash /install_lua.sh
RUN bash /install_lua.sh ${LUA_VERSION}
# Create fake libpython to stop the linker from complaining. The wheel should find the user's libpython at runtime.
RUN touch /usr/lib64/libpython${PYMAJOR}.${PYMINOR}${PYUNICODE}.so
# EveryBeam uses FFTW, HDF5
COPY docker/install_everybeam.sh /
RUN bash /install_everybeam.sh
RUN bash /install_everybeam.sh ${EVERYBEAM_VERSION}
# AOFlagger uses FFTW, HDF5, LUA.
COPY docker/install_aoflagger.sh /
RUN bash /install_aoflagger.sh
RUN bash /install_aoflagger.sh ${AOFLAGGER_VERSION} ${PYMAJOR}.${PYMINOR}
ADD . /dp3
WORKDIR /dp3
......
......@@ -2,7 +2,7 @@ FROM ubuntu:20.04
# TODO: needs to be bumped before next DP3 release
# ENV IDG_VERSION=0.8
ENV EVERYBEAM_VERSION=v0.5.4
ENV EVERYBEAM_VERSION=v0.5.6
ENV IDG_VERSION=6b61c038883ad3f807d20047c4f9e1a1f0b8d98a
ENV AOFLAGGER_VERSION=65d5fba4f4c12797386d3fd9cd76734956a8b233
......@@ -27,7 +27,6 @@ RUN export DEBIAN_FRONTEND="noninteractive" && \
libboost-test-dev \
libcfitsio-dev \
libfftw3-dev \
libgsl-dev \
libgtkmm-3.0-dev \
libhdf5-serial-dev \
liblua5.3-dev \
......
......@@ -2,7 +2,7 @@ FROM ubuntu:22.04
# TODO: needs to be bumped before next DP3 release
# ENV IDG_VERSION=0.8
ENV EVERYBEAM_VERSION=v0.5.4
ENV EVERYBEAM_VERSION=v0.5.6
ENV IDG_VERSION=6b61c038883ad3f807d20047c4f9e1a1f0b8d98a
ENV AOFLAGGER_VERSION=65d5fba4f4c12797386d3fd9cd76734956a8b233
......@@ -29,7 +29,6 @@ RUN export DEBIAN_FRONTEND="noninteractive" && \
libboost-test-dev \
libcfitsio-dev \
libfftw3-dev \
libgsl-dev \
libgtkmm-3.0-dev \
libhdf5-serial-dev \
liblua5.3-dev \
......
Subproject commit 0b03827e4f1ac66245396152d2da255a3dcf50c5
Subproject commit 3cc5c03cbc25f146be511518003d49a321d4d720
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment