Build desired software versions in Docker image

For each of the packages that are cloned from Git it is now possible to specify a branch or tag name. Also, the list of packages installed in the "runner" build was cleaned up to reduce the size of the final container image.

Build desired software versions in Docker image
267dde1b · Marcel Loose · b6e58b5b · 267dde1b · 267dde1b · 267dde1b
Commit 267dde1b authored 3 years ago by Marcel Loose
--- a/.dockerignore
+++ b/.dockerignore
+.git/
+build/
+dist/
+docs/build/
+*.egg-info/
+venv/
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,21 +120,36 @@ run_hba_calibrator:
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
-    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
    - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
+  after_script:
+    - find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_calibrator_logs.tar.gz
+    when: on_failure

 run_hba_target:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
-    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results workflows/HBA_target.cwl test_jobs/HBA_target.json
+    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
    - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
+  after_script:
+    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_target_logs.tar.gz
+    when: on_failure

 build_doc:
  stage: docs
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
+  before_script:
+    - apt-get update
+    - apt-get install -y make python3-sphinx
  script:
    - cd docs
    - make html

--- a/Docker/Dockerfile-base
+++ b/Docker/Dockerfile-base
@@ -10,8 +10,7 @@ FROM ubuntu:20.04 as builder
 # increment the value.
 ENV DOCKER_BUILD_FORCE_GENERATION=0

-ARG portable=True
-
+# Install all build-time dependencies
 RUN export DEBIAN_FRONTEND=noninteractive && \ 
    apt-get update && \
    apt-get install -y \
@@ -24,194 +23,206 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
        gfortran \ 
        git \
        libblas-dev \
+        libboost-date-time-dev \
+        libboost-filesystem-dev \
+        libboost-numpy-dev \
+        libboost-program-options-dev \
+        libboost-python-dev \
+        libboost-system-dev \
+        libboost-test-dev \
        libcfitsio-dev \
        libfftw3-dev \
        libgsl-dev \
+        libgtkmm-3.0-dev \
+        libhdf5-serial-dev \
        liblapack-dev \
        liblua5.3-dev \
-        libhdf5-serial-dev \
-        libboost-filesystem-dev \
-        libboost-system-dev \
-        libboost-date-time-dev \
-        libboost-program-options-dev \
-        libboost-test-dev \
-        libgtkmm-3.0-dev \
        libncurses5-dev \
        libpng-dev \
        libpython3-dev \
        pkg-config \
+        python3 \
+        python3-casacore \
        python3-numpy \
        wcslib-dev \
        wget && \
-    mkdir -p /src/
+    mkdir -p /src
+
+WORKDIR /src

-WORKDIR /src/
+# Build portable binaries by default
+ARG PORTABLE=TRUE

-RUN \
-  git clone https://github.com/lofar-astron/LofarStMan --depth 1 && \
+ARG LOFARSTMAN_VERSION=master
+RUN git clone --depth 1 --branch ${LOFARSTMAN_VERSION} \
+        https://github.com/lofar-astron/LofarStMan && \
    mkdir LofarStMan/build && \
    cd LofarStMan/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j`nproc`

-RUN \
-  git clone https://github.com/aroffringa/dysco.git --depth 1 && \
+ARG DYSCO_VERSION=master
+RUN git clone --depth 1 --branch ${DYSCO_VERSION} \
+        https://github.com/aroffringa/dysco.git && \
    mkdir dysco/build && \
    cd dysco/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j`nproc`
    
+ARG IDG_VERSION=master
 # IDG doesn't work with --depth 1, because it needs all branches to
 # determine its version :-(
-RUN \
-  git clone https://git.astron.nl/RD/idg.git && \
+RUN git clone --branch ${IDG_VERSION} \
+        https://git.astron.nl/RD/idg.git && \
    mkdir idg/build && \
    cd idg/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+    cmake .. && \
    make install -j`nproc`

-RUN \
-  git clone https://gitlab.com/aroffringa/aoflagger.git --depth 1 && \
+ARG AOFLAGGER_VERSION=master
+RUN git clone --depth 1 --branch ${AOFLAGGER_VERSION} \
+        https://gitlab.com/aroffringa/aoflagger.git && \
    mkdir aoflagger/build && \
    cd aoflagger/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j`nproc`

-RUN \
-  apt-get install -y libboost-python-dev libboost-numpy-dev && \
-  git clone https://github.com/lofar-astron/LOFARBeam.git --depth 1 && \
+ARG LOFARBEAM_VERSION=master
+RUN git clone  --depth 1 --branch ${LOFARBEAM_VERSION} \
+        https://github.com/lofar-astron/LOFARBeam.git && \
    mkdir LOFARBeam/build && \
    cd LOFARBeam/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+    cmake .. && \
    make install -j`nproc`

-RUN \
-  git clone https://git.astron.nl/RD/EveryBeam.git --depth 1 && \
+ARG EVERYBEAM_VERSION=master
+RUN git clone --depth 1  --branch ${EVERYBEAM_VERSION} \
+        https://git.astron.nl/RD/EveryBeam.git && \
    mkdir EveryBeam/build && \
    cd EveryBeam/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+    cmake .. && \
    make install -j`nproc`

-RUN \
-  git clone https://git.astron.nl/RD/DP3.git --depth 1 && \
+ARG DP3_VERSION=master
+RUN git clone --depth 1 --branch ${DP3_VERSION} \
+        https://git.astron.nl/RD/DP3.git && \
    mkdir DP3/build && \
    cd DP3/build && \
-  cmake ../ \
-    -DCMAKE_INSTALL_PREFIX=/usr/local/ \
-    -DPORTABLE=${portable} \
-    -DAOFlagger_DIR=/usr/local/ && \
+    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j`nproc`

-RUN \
-  git clone https://gitlab.com/aroffringa/wsclean.git --depth 1 && \
+ARG WSCLEAN_VERSION=master
+RUN git clone --depth 1 --branch ${WSCLEAN_VERSION} \
+        https://gitlab.com/aroffringa/wsclean.git && \
    mkdir wsclean/build && \
    cd wsclean/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+    cmake .. -DPORTABLE=${PORTABLE} && \
    make install -j`nproc`

+# Do not use `pip` from the Debian repository, but fetch it from PyPA.
+# This way, we are sure that the latest versions of `pip`, `setuptools`, and
+# `wheel` are installed in /usr/local, the only directory we're going to copy
+# over to the next build stage.
+RUN wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+# Install required python packages
+RUN python3 -m pip install --no-cache-dir --upgrade \
+    cwl-runner \
+    cwltool
+
+# Install current version of Prefactor CWL. By adding this to the base image
+# we can speed up the build of the final image, because all of Prefactor's
+# dependencies will have been installed already and (probably) don't need to
+# be updated.
+COPY . prefactor
+RUN python3 -m pip install --no-cache-dir --upgrade ./prefactor
+
+
 #---------------------------------------------------------------------------
-# The image is now rebuild without adding the source, to get a smaller image
-# Further binary and/or Python dependencies are also added here.
+# The image will now be rebuilt without adding the sources, in order to
+# reduce the size of the image.
 #---------------------------------------------------------------------------
 FROM ubuntu:20.04 as runner

-COPY --from=builder /usr/local/ /usr/local
+COPY --from=builder /usr/local /usr/local
 RUN chmod +rx /usr/local/bin/*

 SHELL ["/bin/bash", "-c"]

+# Set default versions. Can be overridden from the command-line
+ARG LOFARSTMAN_VERSION=master
+ARG DYSCO_VERSION=master
+ARG IDG_VERSION=master
+ARG AOFLAGGER_VERSION=master
+ARG LOFARBEAM_VERSION=master
+ARG EVERYBEAM_VERSION=master
+ARG DP3_VERSION=master
+ARG WSCLEAN_VERSION=master
+
+# Add version information to the metadata of the image
+LABEL \
+    nl.astron.prefactor.lofarstman.version=${LOFARSTMAN_VERSION} \
+    nl.astron.prefactor.dysco.version=${DYSCO_VERSION} \
+    nl.astron.prefactor.idg.version=${IDG_VERSION} \
+    nl.astron.prefactor.aoflagger.version=${AOFLAGGER_VERSION} \
+    nl.astron.prefactor.lofarbeam.version=${LOFARBEAM_VERSION} \
+    nl.astron.prefactor.everybeam.version=${EVERYBEAM_VERSION} \
+    nl.astron.prefactor.dp3.version=${DP3_VERSION} \
+    nl.astron.prefactor.wsclean.version=${WSCLEAN_VERSION}
+
+# Only install run-time required packages 
 RUN export DEBIAN_FRONTEND=noninteractive && \
    apt-get update && \
    apt-get install -y \
-        bison \
-    	git \
-        casacore-data \
        casacore-tools \
-        flex \
-	gfortran \
-        libboost-date-time1.71 \
-        libboost-filesystem1.71 \
-        libboost-numpy1.71 \
-        libboost-program-options1.71 \
-        libboost-python1.71 \
-        libboost-system1.71 \
-        libboost-test1.71 \
+        libatkmm-1.6-1v5 \
+        libboost-date-time1.71.0 \
+        libboost-filesystem1.71.0 \
+        libboost-program-options1.71.0 \
+        libboost-python1.71.0 \
+        libcairomm-1.0-1v5 \
        libcasa-casa4 \
        libcasa-fits4 \
        libcasa-measures4 \
        libcasa-ms4 \
+        libcasa-python3-4 \
+        libcasa-scimath4 \
        libcasa-tables4 \
        libcfitsio8 \
-        libfftw3-3 \
+        libfftw3-double3 \
+        libfftw3-single3 \
+        libglibmm-2.4-1v5 \
+        libgomp1 \
        libgsl23 \
-        liblua5.3-0 \
-        libncurses5 \ 
-        libgtkmm-3.0 \
+        libgtkmm-3.0-1v5 \
+        libhdf5-103 \
        libhdf5-cpp-103 \
+        liblapack3 \
+        liblua5.3-0 \
+        libpangomm-1.4-1v5 \
        libpng16-16 \
+        libpython3.8 \
+        libsigc++-2.0-0v5 \
+        libstdc++6 \
        nodejs \
-        pybind11-dev   \
        python3 \
        python3-casacore \
-	python3-h5py \
-	python3-matplotlib \
-        python3-numpy \
-	python3-pip \
-        python3-scipy \
-	python3-sphinx \
-	tar \
-	wget \
-        wcslib-dev && \
+        python3-distutils \
+        wget && \
    rm -rf /var/lib/apt/lists/*

-ARG TOIL_VERSION=3.20.0
-ARG CASACORE_TAG=master   
-ARG LSMTool_TAG=master
-ARG LoSoTo_TAG=master
-ARG STMAN_TAG=master
-ARG Dysco_TAG=master
-ARG LofarBeam_TAG=master
-ARG DP3_TAG=master
-ARG IDG_TAG=master
-ARG Everybeam_TAG
-ARG AOFlagger_TAG
-ARG RMextract_TAG=master
-ARG WSClean_TAG
-
-LABEL \
-  casacore.version.tag=${CASACORE_TAG} \
-  toil.version=${TOIL_VERSION} \
-  LSMTool.version.tag=${LSMTool_TAG} \
-  LoSoTo.version.tag=${LoSoTo_TAG} \
-  STMAN.version.tag=${STMAN_TAG} \
-  Dysco.version.tag=${Dysco_TAG} \
-  LOFARBeam.version.tag=${LofarBeam_TAG} \
-  DP3.version.tag=${DP3_TAG} \
-  IDG.version.tag=${IDG_TAG} \
-  Everybeam.version.tag=${Everybeam_TAG} \
-  AOFlagger.version.tag=${AOFlagger_TAG} \
-  RMextract.version.tag=${RMextract_TAG} \
-  WSClean.version.tag=${WSClean_TAG}
-
-ENV LC_ALL=C.UTF-8 \
-    PYTHONPATH="/usr/local/bin:/usr/local/lib/python3.8/site-packages"
-
-RUN \
 # Install WSRT Measures (extra casacore data)
 # Note: The file on the ftp site is updated daily. When warnings regarding leap
 # seconds appear, ignore them or regenerate the docker image.
-  wget -q -O /WSRT_Measures.ztar ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar \
-    && cd /var/lib/casacore/data \
-    && tar xfz /WSRT_Measures.ztar \
-    && rm /WSRT_Measures.ztar && \
-  \
-# Install cwtool & cwl-runner
-  python3 -m pip install --upgrade pip && \
-  python3 -m pip install cwltool cwl-runner && \
-  \
+RUN wget -q -O /WSRT_Measures.ztar \
+        ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
+    cd /var/lib/casacore/data && \
+    tar xfz /WSRT_Measures.ztar && \
+    rm /WSRT_Measures.ztar
+
 # Try to run the compiled tools to make sure they run without
 # a problem (e.g. no missing libraries).
-  aoflagger --version && \
+RUN aoflagger --version && \
    DP3 --version && \
    wsclean --version
-  
\ No newline at end of file
--- a/Docker/Dockerfile-prefactor
+++ b/Docker/Dockerfile-prefactor
 ARG BASE_TAG=latest
 FROM $BASE_TAG

-ENV PREFACTOR_VERSION='master' \
-    PREFACTOR_NAME=prefactor \
-    PREFACTOR_DATA_ROOT=/usr/local/share/prefactor
+ENV PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
+    PYTHONPATH=/usr/local/bin

 # Install Prefactor CWL
-COPY . /${PREFACTOR_NAME}
-RUN cd /${PREFACTOR_NAME} && \
-  pip3 install --upgrade $PWD && \
-  cd .. && \
-  rm -rfv ${PREFACTOR_NAME} && \
-  \
-# A user 'lofaruser' is added (this might be legacy -- not sure this is
-# still relevant).
-  mkdir /home/lofaruser/ && \ 
-  groupadd -r lofaruser && \
-  useradd --no-log-init -r -g lofaruser lofaruser && \
-  chown lofaruser:lofaruser /home/lofaruser
+COPY . /tmp/prefactor

-WORKDIR /home/lofaruser
+RUN python3 -m pip install --no-cache-dir --upgrade /tmp/prefactor && \
+    rm -rf /tmp/prefactor

-USER lofaruser
+# # A user 'lofaruser' is added (this might be legacy -- not sure this is
+# # still relevant).
+# RUN groupadd -r lofaruser && \
+#     useradd -r -l -m -g lofaruser lofaruser
+
+# WORKDIR /home/lofaruser
+
+# USER lofaruser
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,7 @@ setup(
        'Topic :: Scientific/Engineering :: Astronomy'],
    platforms='any',
    install_requires=[
-        'aplpy', 'astropy', 'bdsf', 'losoto', 'lsmtool',
+        'aplpy', 'astropy', 'bdsf', 'h5py', 'losoto', 'lsmtool',
        'matplotlib', 'numpy', 'python-casacore', 'RMextract', 'scipy'],
    scripts=glob.glob('scripts/*'),
    data_files=data_files