Skip to content
Snippets Groups Projects
Commit a95a64ea authored by Marcel Loose's avatar Marcel Loose :sunglasses:
Browse files

Fix CI/CD pipeline

parent edd6c429
No related branches found
No related tags found
1 merge request!205Fix CI/CD pipeline
......@@ -18,7 +18,7 @@ versioning:
script:
# Unshallowing ensures that 'git describe' works
- git fetch --unshallow
- ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
- ./Docker/fetch_commit_hashes.sh | tee commits.txt > versions.env
- echo RAPTHOR_VERSION=$(./Docker/get_scm_version.sh) >> versions.env
# Use hash of commits and CI/CD build files to determine tag of base image
- sha_files=".gitlab-ci.yml ci/ubuntu_22_04-* commits.txt"
......@@ -54,6 +54,13 @@ build-docker-base:
then
[ "$BUILD_DOCKER_IMAGE" = "1" ] && DOCKER_CACHE="--no-cache" || DOCKER_CACHE=""
docker build ${DOCKER_CACHE} \
--build-arg AOFLAGGER_COMMIT \
--build-arg DP3_COMMIT \
--build-arg DYSCO_COMMIT \
--build-arg EVERYBEAM_COMMIT \
--build-arg IDG_COMMIT \
--build-arg SAGECAL_COMMIT \
--build-arg WSCLEAN_COMMIT \
--file ci/ubuntu_22_04-base \
--tag $BASE_IMAGE \
.
......
#!/bin/sh -e
git ls-remote https://git.astron.nl/RD/idg.git HEAD | awk '{ print "IDG_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/EveryBeam.git v0.6.2 | awk '{ print "EVERYBEAM_COMMIT="$1 }'
git ls-remote https://github.com/aroffringa/dysco.git HEAD | awk '{ print "DYSCO_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD | awk '{ print "AOFLAGGER_COMMIT="$1 }'
# git ls-remote https://gitlab.com/aroffringa/wsclean.git v3.5 | awk '{ print "WSCLEAN_COMMIT="$1 }'
echo "WSCLEAN_COMMIT=ed6e7225532ce49222e04897d3d30184cd4a3bff"
git ls-remote https://github.com/nlesc-dirac/sagecal.git HEAD | awk '{ print "SAGECAL_COMMIT="$1 }'
# git ls-remote https://git.astron.nl/RD/DP3.git HEAD | awk '{ print "DP3_COMMIT="$1 }'
echo "DP3_COMMIT=86fba9228652932409845ebcf165dcc77f0dc4cb"
#!/bin/sh -e
git ls-remote https://git.astron.nl/RD/idg.git HEAD | awk '{ print "IDG_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/EveryBeam.git HEAD | awk '{ print "EVERYBEAM_COMMIT="$1 }'
git ls-remote https://github.com/aroffringa/dysco.git HEAD | awk '{ print "DYSCO_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD | awk '{ print "AOFLAGGER_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/wsclean.git HEAD | awk '{ print "WSCLEAN_COMMIT="$1 }'
git ls-remote https://github.com/nlesc-dirac/sagecal.git HEAD | awk '{ print "SAGECAL_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/DP3.git HEAD | awk '{ print "DP3_COMMIT="$1 }'
FROM ubuntu:22.04 as builder
FROM ubuntu:22.04 AS builder
# This Dockerfile builds a docker image containing all Rapthor's dependencies
......@@ -6,6 +6,14 @@ FROM ubuntu:22.04 as builder
ARG PORTABLE=FALSE
ARG TARGET_CPU=haswell
ARG IDG_COMMIT=master
ARG EVERYBEAM_COMMIT=master
ARG DYSCO_COMMIT=master
ARG AOFLAGGER_COMMIT=master
ARG WSCLEAN_COMMIT=master
ARG SAGECAL_COMMIT=master
ARG DP3_COMMIT=master
# Install all build-time dependencies
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
......@@ -39,14 +47,11 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
wget
# Prepare the environment for installing the Rapthor source dependencies.
RUN mkdir /src
RUN git config --global alias.shallow-clone "!git clone --depth 1 --recurse-submodules --shallow-submodules"
WORKDIR /src
# Install IDG
# A shallow clone of IDG breaks its version number. This is an issue for DP3
# since it has a minimal version requirement.
RUN git clone --recurse-submodules https://git.astron.nl/RD/idg.git
RUN git clone --no-checkout https://git.astron.nl/RD/idg.git
RUN git -C idg checkout ${IDG_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_WITH_PYTHON=OFF \
......@@ -58,10 +63,10 @@ RUN cmake \
-G Ninja
RUN ninja -C /src/idg/build install
# Install EveryBeam
# Do not compile python bindings, they will interfere with the ones in the
# binary wheel on PyPI.
RUN git shallow-clone https://git.astron.nl/RD/EveryBeam.git
# Install EveryBeam. Do not compile python bindings, they will interfere with
# the ones in the binary wheel on PyPI.
RUN git clone --no-checkout https://git.astron.nl/RD/EveryBeam.git
RUN git -C EveryBeam checkout ${EVERYBEAM_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_WITH_PYTHON=OFF \
......@@ -74,7 +79,8 @@ RUN cmake \
RUN ninja -C /src/EveryBeam/build install
# Install Dysco
RUN git shallow-clone https://github.com/aroffringa/dysco.git
RUN git clone --no-checkout https://github.com/aroffringa/dysco.git
RUN git -C dysco checkout ${DYSCO_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DPORTABLE=${PORTABLE} \
......@@ -85,7 +91,8 @@ RUN cmake \
RUN ninja -C /src/dysco/build install
# Install AOFlagger
RUN git shallow-clone https://gitlab.com/aroffringa/aoflagger.git
RUN git clone --no-checkout https://gitlab.com/aroffringa/aoflagger.git
RUN git -C aoflagger checkout ${AOFLAGGER_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DPORTABLE=${PORTABLE} \
......@@ -96,7 +103,8 @@ RUN cmake \
RUN ninja -C /src/aoflagger/build install
# Install WSClean
RUN git shallow-clone https://gitlab.com/aroffringa/wsclean.git
RUN git clone --no-checkout https://gitlab.com/aroffringa/wsclean.git
RUN git -C wsclean checkout ${WSCLEAN_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_TESTING=OFF \
......@@ -108,7 +116,8 @@ RUN cmake \
RUN ninja -C /src/wsclean/build install
# Install SAGECal libdirac
RUN git shallow-clone https://github.com/nlesc-dirac/sagecal.git
RUN git clone --no-checkout https://github.com/nlesc-dirac/sagecal.git
RUN git -C sagecal checkout ${SAGECAL_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DLIB_ONLY=1 \
......@@ -118,7 +127,8 @@ RUN cmake \
RUN ninja -C /src/sagecal/build install
# Install DP3
RUN git shallow-clone https://git.astron.nl/RD/DP3.git
RUN git clone --no-checkout https://git.astron.nl/RD/DP3.git
RUN git -C DP3 checkout ${DP3_COMMIT}
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_TESTING=OFF \
......@@ -158,10 +168,28 @@ RUN python3 -m pip install --upgrade ./rapthor && \
# The image will now be rebuilt without adding the sources, in order to
# reduce the size of the image.
#---------------------------------------------------------------------------
FROM ubuntu:22.04 as runner
FROM ubuntu:22.04 AS runner
COPY --from=builder /usr/local /usr/local
ARG IDG_COMMIT=master
ARG EVERYBEAM_COMMIT=master
ARG DYSCO_COMMIT=master
ARG AOFLAGGER_COMMIT=master
ARG WSCLEAN_COMMIT=master
ARG SAGECAL_COMMIT=master
ARG DP3_COMMIT=master
# Add version information to the metadata of the image
LABEL \
nl.astron.rapthor.idg.version=${IDG_COMMIT} \
nl.astron.rapthor.everybeam.version=${EVERYBEAM_COMMIT} \
nl.astron.rapthor.dysco.version=${DYSCO_COMMIT} \
nl.astron.rapthor.aoflagger.version=${AOFLAGGER_COMMIT} \
nl.astron.rapthor.wsclean.version=${WSCLEAN_COMMIT} \
nl.astron.rapthor.sagecal.version=${SAGECAL_COMMIT} \
nl.astron.rapthor.dp3.version=${DP3_COMMIT}
# Only install run-time required packages
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment