Merge branch 'rap-880-add-docker-image' into 'main'

RAP-880 Add Docker Image

See merge request !10

.dockerignore

+docker/
+docs/build/
+venv/
+tests/
+.git/

.gitlab-ci.yml

@@ -13,6 +13,7 @@ workflow:
 stages:
   - prepare
   - test
+  - images
 
 include:
   - template: Security/SAST.gitlab-ci.yml
@@ -38,3 +39,15 @@ validate:
   needs: ["trigger_prepare"]
   script:
     - cwltool --validate workflows/pipeline.cwl tests/pipeline_input.json
+
+docker_build:
+  stage: images
+  image: docker:latest
+  tags:
+    - dind
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_REF_PROTECTED == "true"'
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker build -f docker/pipeline/Dockerfile . --tag $CI_REGISTRY_IMAGE/preprocess:latest
+    - docker push $CI_REGISTRY_IMAGE/preprocess:latest

docker/pipeline/Dockerfile

+FROM ubuntu:22.04 AS builder
+
+# This Docker image builds the dependencies for the Pre-Processing Pipeline.
+# Credits: this image is based on the Docker images for LINC and Rapthor.
+
+# Install all build-time dependencies
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+        build-essential \
+        casacore-dev \
+        cmake \
+        gfortran \
+        flex \
+        bison \
+        git \
+        libarmadillo-dev \
+        libboost-date-time-dev \
+        libboost-filesystem-dev \
+        libboost-program-options-dev \
+        libboost-python-dev \
+        libboost-test-dev \
+        libcfitsio-dev \
+        libfftw3-dev \
+        libgsl-dev \
+        libgtkmm-3.0-dev \
+        libhdf5-serial-dev \
+        liblua5.3-dev \
+        libpng-dev \
+        ninja-build \
+        python3-dev \
+        wcslib-dev \
+        wget
+
+# Prepare the environment for installing the Rapthor source dependencies.
+WORKDIR /src
+RUN git config --global alias.shallow-clone "!git clone --depth 1 --recurse-submodules --shallow-submodules"
+
+# By default, build non-portable code, for the oldest CPU that supports AVX2
+ARG PORTABLE=FALSE
+ARG TARGET_CPU=haswell
+
+# Install casacore
+RUN git shallow-clone https://github.com/casacore/casacore.git
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DPORTABLE=${PORTABLE} \
+    -DTARGET_CPU=${TARGET_CPU} \
+    -H/src/casacore \
+    -B/src/casacore/build \
+    -G Ninja
+RUN ninja -C /src/casacore/build install
+
+# Install LofarStMan
+RUN git shallow-clone https://github.com/lofar-astron/LofarStMan
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DPORTABLE=${PORTABLE} \
+    -DTARGET_CPU=${TARGET_CPU} \
+    -H/src/LofarStMan \
+    -B/src/LofarStMan/build \
+    -G Ninja
+RUN ninja -C /src/LofarStMan/build install
+
+# Install EveryBeam
+# Do not compile python bindings, they will interfere with the ones in the 
+# binary wheel on PyPI.
+RUN git shallow-clone https://git.astron.nl/RD/EveryBeam.git
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DBUILD_WITH_PYTHON=OFF \
+    -DBUILD_TESTING=OFF \
+    -DPORTABLE=${PORTABLE} \
+    -DTARGET_CPU=${TARGET_CPU} \
+    -H/src/EveryBeam \
+    -B/src/EveryBeam/build \
+    -G Ninja
+RUN ninja -C /src/EveryBeam/build install
+
+# Install Dysco 
+RUN git shallow-clone https://github.com/aroffringa/dysco.git
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DPORTABLE=${PORTABLE} \
+    -DTARGET_CPU=${TARGET_CPU} \
+    -H/src/dysco \
+    -B/src/dysco/build \
+    -G Ninja
+RUN ninja -C /src/dysco/build install
+
+# Install AOFlagger
+RUN git shallow-clone https://gitlab.com/aroffringa/aoflagger.git
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DPORTABLE=${PORTABLE} \
+    -DTARGET_CPU=${TARGET_CPU} \
+    -H/src/aoflagger \
+    -B/src/aoflagger/build \
+    -G Ninja
+RUN ninja -C /src/aoflagger/build install
+
+# Install SAGECal libdirac
+RUN git shallow-clone https://github.com/nlesc-dirac/sagecal.git
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DLIB_ONLY=1 \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+    -H/src/sagecal \
+    -B/src/sagecal/build \
+    -G Ninja
+RUN ninja -C /src/sagecal/build install
+
+# Install DP3
+RUN git shallow-clone https://git.astron.nl/RD/DP3.git
+RUN cmake \
+    -DCMAKE_BUILD_TYPE:STRING=Release \
+    -DBUILD_TESTING=OFF \
+    -DPORTABLE=${PORTABLE} \
+    -DTARGET_CPU=${TARGET_CPU} \
+    -DLIBDIRAC_PREFIX=/usr/local/ \
+    -H/src/DP3 \
+    -B/src/DP3/build \
+    -G Ninja
+RUN ninja -C /src/DP3/build install
+
+# Do not use `pip` from the Debian repository, but fetch it from PyPA.
+# This way, we are sure that the latest versions of `pip`, `setuptools`, and
+# `wheel` are installed in /usr/local, the only directory we're going to copy
+# over to the next build stage.
+RUN wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+# Install all Python dependencies for Rapthor
+RUN python3 -m pip install \
+    cwltool \
+    toil[cwl]
+
+#---------------------------------------------------------------------------
+# The image will now be rebuilt without adding the sources, in order to
+# reduce the size of the image.
+#---------------------------------------------------------------------------
+FROM ubuntu:22.04 AS runner
+
+COPY --from=builder /usr/local /usr/local
+
+# Only install run-time required packages
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        casacore-tools \
+        libatkmm-1.6-1v5 \
+        libblas3 \
+        libboost-filesystem1.74.0 \
+        libboost-program-options1.74.0 \
+        libcairomm-1.0-1v5 \
+        libcasa-casa6 \
+        libcasa-fits6 \
+        libcasa-measures6 \
+        libcasa-ms6 \
+        libcasa-scimath6 \
+        libcasa-tables6 \
+        libcfitsio9 \
+        libfftw3-double3 \
+        libfftw3-single3 \
+        libglibmm-2.4-1v5 \
+        libgomp1 \
+        libgsl27 \
+        libgtkmm-3.0-1v5 \
+        libhdf5-103-1 \
+        libhdf5-cpp-103-1 \
+        liblapack3 \
+        liblua5.3-0 \
+        libpangomm-1.4-1v5 \
+        libpng16-16 \
+        libpython3.10 \
+        libsigc++-2.0-0v5 \
+        libstdc++6 \
+        nodejs \
+        python3 \
+        python3-distutils \
+        python3-six \
+        wget
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# Install the casacore measures data. We purposely do not install these from
+# the Ubuntu repository, but download the latest version directly from the
+# ASTRON ftp site.
+# Note: The file on the ftp site is updated daily. When warnings regarding
+# leap seconds appear, ignore them or regenerate the docker image.
+RUN mkdir -p /usr/local/share/casacore/data && \
+wget -qO - ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar | \
+    tar -C /usr/local/share/casacore/data -xzf -
+
+# Try to run the compiled tools to make sure they run without
+# a problem (e.g. no missing libraries).
+RUN aoflagger --version && \
+    DP3 --version
+
+# Similarly, try to run the CWL runners, to make sure the workflows can
+# fully be run inside a container as well.
+RUN cwltool --version && \
+    toil-cwl-runner --version
+
+# Install current version of the Pre-Processing Pipeline.
+COPY . /usr/local/share/prep