Skip to content
Snippets Groups Projects
Commit e63b59e4 authored by Marcel Loose's avatar Marcel Loose :sunglasses:
Browse files

Sync with development version

parent 35d93bd6
No related branches found
No related tags found
No related merge requests found
Pipeline #104797 passed
Stage: versioning
Stage: build
Stage: install
Stage: prepare_tests
Stage: run_tests
Stage: deploy
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 27
− 17
View file @ e63b59e4
......@@ -36,7 +36,7 @@ stages:
- |
if test -n "$RELEASE"
then
echo "Checking out branch $CI_COMMIT_BRANCH"
echo "Checking out branch '$CI_COMMIT_BRANCH'"
git checkout $CI_COMMIT_BRANCH
fi
......@@ -66,7 +66,6 @@ stages:
.run_tests:
stage: run_tests
image: $INTEGRATION_IMAGE
extends: .release
before_script:
- !reference [.release, before_script]
- mkdir -p workdir
......@@ -84,7 +83,6 @@ stages:
.docs:
stage: docs
extends: .release
image: $INTEGRATION_IMAGE
.finalize:
......@@ -115,14 +113,21 @@ prepare_release:
- |
if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
then
echo "*** Release $RELEASE already exists. Bailing out! ***"
echo "*** Release '$RELEASE' already exists. Bailing out! ***"
touch .tag.exists
exit 1
fi
- !reference [.setup_git, before_script]
script:
# Update dockerPull image URI in CWL steps with a tagged version
- sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$RELEASE," steps/*.cwl
# Determine the name of the docker image tag. If this is a versioned release,
# strip off the leading "V" or "v", because that is what `setuptools_scm`
# does when setting LINC_VERSION in the `versioning` job. LINC_VERSION
# is used later in the `deploy_docker` job to tag the docker image to be
# pushed to Docker Hub. Hence, the same tag must be used in the `dockerPull`
# URI in the CWL steps, so update the CWL files.
- LINC_TAG=$(echo $RELEASE | sed -r 's,^[Vv]([0-9]+),\1,')
- echo "Updating dockerPull URI to use image tag '$LINC_TAG'"
- sed -ri "/dockerPull/s,(astronrd/linc).*,\1:$LINC_TAG," steps/*.cwl
- git add -u steps/*.cwl
# Only commit if there are changes
- |
......@@ -140,11 +145,14 @@ prepare_release:
versioning:
stage: versioning
image: bitnami/git
extends: .release
image: python
before_script:
- !reference [.release, before_script]
- pip install setuptools_scm
script:
- ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
- echo LINC_VERSION=$(git describe --tags --always) >> versions.env
# Use a sub-command, to catch the exit status from `setuptools_scm`
- (echo -n "LINC_VERSION="; python -m setuptools_scm) >> versions.env
# Use hash of commits to determine version of base image (and rebuild if necessary)
- echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
- echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
......@@ -353,12 +361,14 @@ deploy_docker:
stage: deploy
extends: .deploy
script:
- echo "Deploying to DockerHub, using $LINC_VERSION as image tag"
# Replace characters that are now allowed in a tag string with a dash
- LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
- echo "Deploying to DockerHub, using tag '$LINC_TAG'"
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:$LINC_VERSION
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:latest
- docker push $CI_PROJECT_PATH:$LINC_VERSION
- docker push $CI_PROJECT_PATH:latest
- docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_TAG
- docker tag $INTEGRATION_IMAGE astronrd/linc:latest
- docker push astronrd/linc:$LINC_TAG
- docker push astronrd/linc:latest
rules:
# Run on the default branch or on a release branch
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
......@@ -371,8 +381,8 @@ deploy_docker_tag_stable:
when: manual
script:
- docker pull $INTEGRATION_IMAGE
- docker tag $INTEGRATION_IMAGE $CI_PROJECT_PATH:stable
- docker push $CI_PROJECT_PATH:stable
- docker tag $INTEGRATION_IMAGE astronrd/linc:stable
- docker push astronrd/linc:stable
### Stage: finalize
......@@ -390,7 +400,7 @@ rollback_release:
- |
if test -f .tag.exists
then
echo "*** Not removing existing tag $RELEASE! ***"
echo "*** Not removing existing tag '$RELEASE'! ***"
else
git push origin -d $RELEASE
fi
......
Docker/Dockerfile-base
+ 42
− 35
View file @ e63b59e4
FROM ubuntu:22.04 as builder
FROM ubuntu:22.04 AS builder
# This Docker image builds the dependencies for the Rapthor pipeline.
# It lives on the head of its dependencies.
......@@ -9,8 +9,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get install -y \
bison \
build-essential \
casacore-data \
casacore-dev \
cmake \
flex \
gfortran \
......@@ -37,16 +35,38 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
python3 \
python3-casacore \
python3-numpy \
python3-pip \
wcslib-dev \
wget && \
mkdir -p /src
wget
WORKDIR /src
# Build portable binaries by default
# Suppress warning from pip when installing packages as root
ENV PIP_ROOT_USER_ACTION=ignore
# Do not use `pip` from the Debian repository, but fetch it from PyPA.
# This way, we are sure that the latest versions of `pip`, `setuptools`, and
# `wheel` are installed in /usr/local, the only directory we're going to copy
# over to the next build stage.
RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
python3 get-pip.py
# Install required python packages
RUN python3 -m pip install --no-cache-dir --upgrade \
nodejs-wheel \
toil[cwl]
# Build binaries targeted at the oldest CPU that supports AVX2
ARG TARGET_CPU=haswell
ARG CASACORE_COMMIT=master
RUN git clone --no-checkout \
https://github.com/casacore/casacore.git && \
mkdir casacore/build && \
cd casacore && git checkout ${CASACORE_COMMIT} && \
cd build && \
cmake .. -DTARGET_CPU=${TARGET_CPU} && \
make install -j`nproc`
ARG LOFARSTMAN_COMMIT=main
RUN git clone --no-checkout \
https://github.com/lofar-astron/LofarStMan && \
......@@ -132,17 +152,11 @@ RUN git clone --no-checkout \
cmake .. -DTARGET_CPU=${TARGET_CPU} && \
make install -j`nproc`
# Do not use `pip` from the Debian repository, but fetch it from PyPA.
# This way, we are sure that the latest versions of `pip`, `setuptools`, and
# `wheel` are installed in /usr/local, the only directory we're going to copy
# over to the next build stage.
RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
python3 get-pip.py
# Install required python packages
RUN python3 -m pip install --no-cache-dir --upgrade \
nodejs-wheel \
toil[cwl]
ARG PYTHONCASACORE_COMMIT=master
RUN git clone --no-checkout \
https://github.com/casacore/python-casacore.git && \
cd python-casacore && git checkout ${PYTHONCASACORE_COMMIT} && \
python3 -m pip install --verbose .
# Install current version of LINC. By adding this to the base image
# we can speed up the build of the final image, because all of LINC's
......@@ -153,14 +167,14 @@ COPY . linc
# Note: LINC_VERSION should be provided as build argument
ARG LINC_VERSION=0.0.0
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LINC=${LINC_VERSION} \
python3 -m pip install --no-cache-dir --upgrade ./linc
python3 -m pip install --upgrade ./linc
#---------------------------------------------------------------------------
# The image will now be rebuilt without adding the sources, in order to
# reduce the size of the image.
#---------------------------------------------------------------------------
FROM ubuntu:22.04 as runner
FROM ubuntu:22.04 AS runner
COPY --from=builder /usr/local /usr/local
RUN chmod +rx /usr/local/bin/*
......@@ -168,18 +182,22 @@ RUN chmod +rx /usr/local/bin/*
SHELL ["/bin/bash", "-c"]
# Set default versions. Can be overridden from the command-line
ARG CASACORE_COMMIT=master
ARG LOFARSTMAN_COMMIT=main
ARG DYSCO_COMMIT=master
ARG SAGECAL_COMMIT=master
ARG IDG_COMMIT=master
ARG AOFLAGGER_COMMIT=master
ARG LOFARBEAM_COMMIT=master
ARG EVERYBEAM_COMMIT=master
ARG SAGECAL_COMMIT=master
ARG DP3_COMMIT=master
ARG WSCLEAN_COMMIT=master
ARG PYTHONCASACORE_COMMIT=master
# Add version information to the metadata of the image
LABEL \
nl.astron.nl.linc.casacore.version=${CASACORE_COMMIT} \
nl.astron.nl.linc.python-casacore.version=${PYTHONCASACORE_COMMIT} \
nl.astron.linc.lofarstman.version=${LOFARSTMAN_COMMIT} \
nl.astron.linc.dysco.version=${DYSCO_COMMIT} \
nl.astron.linc.sagecal.version=${SAGECAL_COMMIT} \
......@@ -194,7 +212,6 @@ LABEL \
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
casacore-tools \
gdb \
git \
libatkmm-1.6-1v5 \
......@@ -203,13 +220,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libboost-program-options1.74.0 \
libboost-python1.74.0 \
libcairomm-1.0-1v5 \
libcasa-casa6 \
libcasa-fits6 \
libcasa-measures6 \
libcasa-ms6 \
libcasa-python3-6 \
libcasa-scimath6 \
libcasa-tables6 \
libcfitsio9 \
libfftw3-double3 \
libfftw3-single3 \
......@@ -229,7 +239,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libsigc++-2.0-0v5 \
libstdc++6 \
python3 \
python3-casacore \
python3-distutils \
rsync \
wget && \
......@@ -238,11 +247,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
# Install WSRT Measures (extra casacore data)
# Note: The file on the ftp site is updated daily. When warnings regarding leap
# seconds appear, ignore them or regenerate the docker image.
RUN wget -q -O /WSRT_Measures.ztar \
ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
cd /var/lib/casacore/data && \
tar xfz /WSRT_Measures.ztar && \
rm /WSRT_Measures.ztar
RUN mkdir -p /usr/local/share/casacore/data && \
wget -qO - ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | \
tar -C /usr/local/share/casacore/data -xzf -
# Try to run the compiled tools to make sure they run without
# a problem (e.g. no missing libraries).
......
Docker/fetch_latest_commits.sh
+ 11
− 9
View file @ e63b59e4
#!/bin/sh
git ls-remote https://github.com/lofar-astron/LofarStMan HEAD | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
git ls-remote https://github.com/aroffringa/dysco.git HEAD | awk '{ print "DYSCO_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/idg.git HEAD | awk '{ print "IDG_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD | awk '{ print "AOFLAGGER_COMMIT="$1 }'
git ls-remote https://github.com/nlesc-dirac/sagecal HEAD | awk '{ print "SAGECAL_COMMIT="$1 }'
git ls-remote https://github.com/lofar-astron/LOFARBeam.git HEAD | awk '{ print "LOFARBEAM_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/EveryBeam.git HEAD | awk '{ print "EVERYBEAM_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/DP3.git HEAD | awk '{ print "DP3_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/wsclean.git HEAD | awk '{ print "WSCLEAN_COMMIT="$1 }'
git ls-remote https://github.com/casacore/casacore.git v3.6.1 | awk '{ print "CASACORE_COMMIT="$1 }'
git ls-remote https://github.com/lofar-astron/LofarStMan HEAD | awk '{ print "LOFARSTMAN_COMMIT="$1 }'
git ls-remote https://github.com/aroffringa/dysco.git HEAD | awk '{ print "DYSCO_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/idg.git HEAD | awk '{ print "IDG_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/aoflagger.git HEAD | awk '{ print "AOFLAGGER_COMMIT="$1 }'
git ls-remote https://github.com/lofar-astron/LOFARBeam.git HEAD | awk '{ print "LOFARBEAM_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/EveryBeam.git HEAD | awk '{ print "EVERYBEAM_COMMIT="$1 }'
git ls-remote https://github.com/nlesc-dirac/sagecal HEAD | awk '{ print "SAGECAL_COMMIT="$1 }'
git ls-remote https://git.astron.nl/RD/DP3.git HEAD | awk '{ print "DP3_COMMIT="$1 }'
git ls-remote https://gitlab.com/aroffringa/wsclean.git HEAD | awk '{ print "WSCLEAN_COMMIT="$1 }'
git ls-remote https://github.com/casacore/python-casacore.git v3.6.1 | awk '{ print "PYTHONCASACORE_COMMIT="$1 }'
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment