Skip to content
Snippets Groups Projects
Commit ef3a718e authored by Mick Veldhuis's avatar Mick Veldhuis
Browse files

RAP-880 Add Docker Image

parent 0964b072
Branches
No related tags found
1 merge request!10RAP-880 Add Docker Image
docker/
docs/build/
venv/
tests/
.git/
......@@ -13,6 +13,7 @@ workflow:
stages:
- prepare
- test
- images
include:
- template: Security/SAST.gitlab-ci.yml
......@@ -38,3 +39,15 @@ validate:
needs: ["trigger_prepare"]
script:
- cwltool --validate workflows/pipeline.cwl tests/pipeline_input.json
docker_build:
stage: images
image: docker:latest
tags:
- dind
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_REF_PROTECTED == "true"'
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build -f docker/pipeline/Dockerfile . --tag $CI_REGISTRY_IMAGE/preprocess:latest
- docker push $CI_REGISTRY_IMAGE/preprocess:latest
FROM ubuntu:22.04 AS builder
# This Docker image builds the dependencies for the Pre-Processing Pipeline.
# Credits: this image is based on the Docker images for LINC and Rapthor.
# Install all build-time dependencies
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
build-essential \
casacore-dev \
cmake \
gfortran \
flex \
bison \
git \
libarmadillo-dev \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-program-options-dev \
libboost-python-dev \
libboost-test-dev \
libcfitsio-dev \
libfftw3-dev \
libgsl-dev \
libgtkmm-3.0-dev \
libhdf5-serial-dev \
liblua5.3-dev \
libpng-dev \
ninja-build \
python3-dev \
wcslib-dev \
wget
# Prepare the environment for installing the Rapthor source dependencies.
WORKDIR /src
RUN git config --global alias.shallow-clone "!git clone --depth 1 --recurse-submodules --shallow-submodules"
# By default, build non-portable code, for the oldest CPU that supports AVX2
ARG PORTABLE=FALSE
ARG TARGET_CPU=haswell
# Install casacore
RUN git shallow-clone https://github.com/casacore/casacore.git
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DPORTABLE=${PORTABLE} \
-DTARGET_CPU=${TARGET_CPU} \
-H/src/casacore \
-B/src/casacore/build \
-G Ninja
RUN ninja -C /src/casacore/build install
# Install LofarStMan
RUN git shallow-clone https://github.com/lofar-astron/LofarStMan
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DPORTABLE=${PORTABLE} \
-DTARGET_CPU=${TARGET_CPU} \
-H/src/LofarStMan \
-B/src/LofarStMan/build \
-G Ninja
RUN ninja -C /src/LofarStMan/build install
# Install EveryBeam
# Do not compile python bindings, they will interfere with the ones in the
# binary wheel on PyPI.
RUN git shallow-clone https://git.astron.nl/RD/EveryBeam.git
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_WITH_PYTHON=OFF \
-DBUILD_TESTING=OFF \
-DPORTABLE=${PORTABLE} \
-DTARGET_CPU=${TARGET_CPU} \
-H/src/EveryBeam \
-B/src/EveryBeam/build \
-G Ninja
RUN ninja -C /src/EveryBeam/build install
# Install Dysco
RUN git shallow-clone https://github.com/aroffringa/dysco.git
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DPORTABLE=${PORTABLE} \
-DTARGET_CPU=${TARGET_CPU} \
-H/src/dysco \
-B/src/dysco/build \
-G Ninja
RUN ninja -C /src/dysco/build install
# Install AOFlagger
RUN git shallow-clone https://gitlab.com/aroffringa/aoflagger.git
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DPORTABLE=${PORTABLE} \
-DTARGET_CPU=${TARGET_CPU} \
-H/src/aoflagger \
-B/src/aoflagger/build \
-G Ninja
RUN ninja -C /src/aoflagger/build install
# Install SAGECal libdirac
RUN git shallow-clone https://github.com/nlesc-dirac/sagecal.git
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DLIB_ONLY=1 \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-H/src/sagecal \
-B/src/sagecal/build \
-G Ninja
RUN ninja -C /src/sagecal/build install
# Install DP3
RUN git shallow-clone https://git.astron.nl/RD/DP3.git
RUN cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DBUILD_TESTING=OFF \
-DPORTABLE=${PORTABLE} \
-DTARGET_CPU=${TARGET_CPU} \
-DLIBDIRAC_PREFIX=/usr/local/ \
-H/src/DP3 \
-B/src/DP3/build \
-G Ninja
RUN ninja -C /src/DP3/build install
# Do not use `pip` from the Debian repository, but fetch it from PyPA.
# This way, we are sure that the latest versions of `pip`, `setuptools`, and
# `wheel` are installed in /usr/local, the only directory we're going to copy
# over to the next build stage.
RUN wget https://bootstrap.pypa.io/get-pip.py && \
python3 get-pip.py
# Install all Python dependencies for Rapthor
RUN python3 -m pip install \
cwltool \
toil[cwl]
#---------------------------------------------------------------------------
# The image will now be rebuilt without adding the sources, in order to
# reduce the size of the image.
#---------------------------------------------------------------------------
FROM ubuntu:22.04 AS runner
COPY --from=builder /usr/local /usr/local
# Only install run-time required packages
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates \
casacore-tools \
libatkmm-1.6-1v5 \
libblas3 \
libboost-filesystem1.74.0 \
libboost-program-options1.74.0 \
libcairomm-1.0-1v5 \
libcasa-casa6 \
libcasa-fits6 \
libcasa-measures6 \
libcasa-ms6 \
libcasa-scimath6 \
libcasa-tables6 \
libcfitsio9 \
libfftw3-double3 \
libfftw3-single3 \
libglibmm-2.4-1v5 \
libgomp1 \
libgsl27 \
libgtkmm-3.0-1v5 \
libhdf5-103-1 \
libhdf5-cpp-103-1 \
liblapack3 \
liblua5.3-0 \
libpangomm-1.4-1v5 \
libpng16-16 \
libpython3.10 \
libsigc++-2.0-0v5 \
libstdc++6 \
nodejs \
python3 \
python3-distutils \
python3-six \
wget
RUN rm -rf /var/lib/apt/lists/*
# Install the casacore measures data. We purposely do not install these from
# the Ubuntu repository, but download the latest version directly from the
# ASTRON ftp site.
# Note: The file on the ftp site is updated daily. When warnings regarding
# leap seconds appear, ignore them or regenerate the docker image.
RUN mkdir -p /usr/local/share/casacore/data && \
wget -qO - ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | \
tar -C /usr/local/share/casacore/data -xzf -
# Try to run the compiled tools to make sure they run without
# a problem (e.g. no missing libraries).
RUN aoflagger --version && \
DP3 --version
# Similarly, try to run the CWL runners, to make sure the workflows can
# fully be run inside a container as well.
RUN cwltool --version && \
toil-cwl-runner --version
# Install current version of the Pre-Processing Pipeline.
COPY . /usr/local/share/prep
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment