Dockerfile

FROM ubuntu:22.04 as builder

# This Docker image builds the dependencies for the Rapthor pipeline.
# It is designed to stay at the head of its dependencies.

# Install all required dependencies
RUN apt-get update && apt-get install -y \
    bison \
    build-essential \
    cmake \
    flex \
    gfortran \
    git \
    libblas-dev \
    libboost-date-time-dev \
    libboost-filesystem-dev \
    libboost-numpy-dev \
    libboost-program-options-dev \
    libboost-python-dev \
    libboost-system-dev \
    libboost-test-dev \
    libcfitsio-dev \
    libfftw3-dev \
    libgsl-dev \
    libgtkmm-3.0-dev \
    libhdf5-serial-dev \
    liblapack-dev \
    liblua5.3-dev \
    libncurses5-dev \
    libpng-dev \
    libpython3-dev \
    pkg-config \
    python3 \
    wcslib-dev \
    wget && \
    rm -rf /var/lib/apt/lists/*

# Set working directory
WORKDIR /src

# Install latest pip, setuptools, and wheel
RUN wget -q https://bootstrap.pypa.io/get-pip.py && \
    python3 get-pip.py && rm get-pip.py && \
    python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel

# Install required Python packages
RUN python3 -m pip install --no-cache-dir \
    numpy \
    nodejs-wheel \
    toil[cwl]

# Install the latest casacore measures data
RUN mkdir -p /usr/local/share/casacore/data && \
    wget -qO - https://www.astron.nl/iers/WSRT_Measures.ztar | \
    tar -C /usr/local/share/casacore/data -xzf -

# Build dependencies with CMake
# Target haswell for better x64 performance. Use PORTABLE=TRUE if you want to target a broader range of CPU architectures.
# We tried using znver2 here so we can run on both wn-dc and wn-la machines on SURF's Spider.
ARG PORTABLE=TRUE
# ARG TARGET_CPU=haswell
# ARG TARGET_CPU=znver2 << Also does not work on Spider's wn-dc nodes

# Casacore
ARG CASACORE_COMMIT=master
RUN git clone --no-checkout https://github.com/casacore/casacore.git && \
    git -C casacore checkout ${CASACORE_COMMIT} && \
    mkdir casacore/build && \
    cd casacore/build && \
    cmake .. && \
    make install -j$(nproc) && \
    cd ../.. && rm -rf casacore

# LofarStMan
ARG LOFARSTMAN_COMMIT=main
RUN git clone --no-checkout https://github.com/lofar-astron/LofarStMan.git && \
    git -C LofarStMan checkout ${LOFARSTMAN_COMMIT} && \
    mkdir LofarStMan/build && \
    cd LofarStMan/build && \
    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j$(nproc) && \
    cd ../.. && rm -rf LofarStMan

# AOFlagger
RUN apt update && \
    apt install -y software-properties-common && \
    add-apt-repository -y universe && \
    apt update && \
    apt install -y pybind11-dev
ARG AOFLAGGER_COMMIT=master
RUN git clone --no-checkout https://gitlab.com/aroffringa/aoflagger.git && \
    git -C aoflagger checkout ${AOFLAGGER_COMMIT} && \
    mkdir aoflagger/build && \
    cd aoflagger/build && \
    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j$(nproc) && \
    cd ../.. && rm -rf aoflagger

# EveryBeam
ARG EVERYBEAM_COMMIT=master
RUN git clone --no-checkout https://git.astron.nl/RD/EveryBeam.git && \
    git -C EveryBeam checkout ${EVERYBEAM_COMMIT} && \
    mkdir EveryBeam/build && \
    cd EveryBeam/build && \
    cmake .. && \
    make install -j$(nproc) && \
    cd ../.. # Note: not removing the directory here so we can build pytohn wrapper later

# DP3
ARG DP3_COMMIT=master
RUN git clone --no-checkout https://git.astron.nl/RD/DP3.git && \
    git -C DP3 checkout ${DP3_COMMIT} && \
    mkdir DP3/build && \
    cd DP3/build && \
    cmake .. -DPORTABLE=${PORTABLE} -DLIBDIRAC_PREFIX=/usr/local/ -DMETADATA_COMPRESSION_DEFAULT=True && \
    make install -j$(nproc) && \
    cd ../.. && rm -rf DP3

# Python-Casacore
ARG PYTHONCASACORE_COMMIT=master
RUN git clone --no-checkout https://github.com/casacore/python-casacore.git && \
    git -C python-casacore checkout ${PYTHONCASACORE_COMMIT} && \
    cd python-casacore && \
    CASACORE_DATA=/usr/local/share/casacore/data python3 -m pip install -v . && \
    cd .. && rm -rf python-casacore

# Install EveryBeam Python package
RUN python3 -m pip install ./EveryBeam && \
    rm -rf EveryBeam # Remove directory here now the python wrapper is installed

#---------------------------------------------------------------------------
# The image will now be rebuilt without adding the sources, in order to
# reduce the size of the image.
#---------------------------------------------------------------------------
FROM ubuntu:22.04 AS runner

COPY --from=builder /usr/local /usr/local
RUN chmod +rx /usr/local/bin/*

SHELL ["/bin/bash", "-c"]

# Set default versions. Can be overridden from the command-line
ARG AOFLAGGER_COMMIT=master
ARG CASACORE_COMMIT=master
ARG DP3_COMMIT=master
ARG EVERYBEAM_COMMIT=master
ARG IDG_COMMIT=master
ARG LOFARSTMAN_COMMIT=main
ARG PYTHONCASACORE_COMMIT=master
ARG SAGECAL_COMMIT=master
ARG WSCLEAN_COMMIT=master

# Add version information to the metadata of the image
LABEL \
    nl.astron.linc.aoflagger.version=${AOFLAGGER_COMMIT} \
    nl.astron.linc.casacore.version=${CASACORE_COMMIT} \
    nl.astron.linc.dp3.version=${DP3_COMMIT} \
    nl.astron.linc.everybeam.version=${EVERYBEAM_COMMIT} \
    nl.astron.linc.idg.version=${IDG_COMMIT} \
    nl.astron.linc.lofarstman.version=${LOFARSTMAN_COMMIT} \
    nl.astron.linc.python-casacore.version=${PYTHONCASACORE_COMMIT} \
    nl.astron.linc.sagecal.version=${SAGECAL_COMMIT} \
    nl.astron.linc.wsclean.version=${WSCLEAN_COMMIT}

# Only install run-time required packages
RUN export DEBIAN_FRONTEND=noninteractive && \
    apt-get update && \
    apt-get install -y \
    git \
    libblas3 \
    libboost-filesystem1.74.0 \
    libboost-program-options1.74.0 \
    libboost-python1.74.0 \
    libcfitsio9 \
    libfftw3-double3 \
    libfftw3-single3 \
    libgfortran5 \
    libglib2.0-0 \
    libgomp1 \
    libgsl27 \
    libgtkmm-3.0-1v5 \
    libhdf5-103-1 \
    libhdf5-cpp-103-1 \
    liblapack3 \
    liblua5.3-0 \
    libpng16-16 \
    libpython3.10 \
    libstdc++6 \
    python3 \
    python3-distutils \
    wget && \
    rm -rf /var/lib/apt/lists/*

# Install ms_tools
RUN python3 -m pip install "git+https://git.astron.nl/ldv/ms_tools.git#egg=ms_tools"

# Install lofar_quality and link scripts
RUN git clone https://git.astron.nl/ldv/ldv-images.git /opt/ldv-images && \
    git clone https://git.astron.nl/ldv/lofar_quality.git /opt/lofar_quality && \
    python3 -m pip install /opt/lofar_quality && \
    ln -sf /opt/ldv-images/lofar_legacy/scripts/fixbeaminfo /usr/local/bin/fixbeaminfo && \
    ln -sf /opt/ldv-images/lofar_legacy/scripts/fixbeaminfo /usr/local/bin/fixinfo && \
    ln -sf /opt/ldv-images/lofar_legacy/scripts/fix_common_ms_issues.py /usr/local/bin/fix_common_ms_issues.py && \
    ln -sf /opt/ldv-images/lofar_legacy/scripts/fix_weightedsum_uvw.py /usr/local/bin/fix_weightedsum_uvw.py && \
    ln -sf /opt/ldv-images/lofar_legacy/scripts/fix_weightspectrum.py /usr/local/bin/fix_weightspectrum.py && \
    ln -sf /opt/lofar_quality/bin/inspect_utils.py /usr/local/bin/inspect_utils.py

# Make sure scripts added to bin are executable
RUN chmod +rx /usr/local/bin/*

# Ensure shared libraries are up to date
RUN ldconfig

# Test if tools are installed properly
RUN inspect_utils.py --help && \
    fix_weightedsum_uvw.py --help && \
    fix_weightspectrum.py --help && \
    fix_common_ms_issues.py --help && \
    aoflagger --version && \
    DP3 --version