Merge branch 'ubuntu21-dockerfile' into 'master'

Base dockerfile on Ubuntu 20.04 See merge request eosc/prefactor3-cwl!58 Former-commit-id: df6ad411

Merge branch 'ubuntu21-dockerfile' into 'master'
54743fbf · Andre Offringa · 227847a7 · 4ff4a1f4 · 54743fbf · 227847a7
Commit 54743fbf authored 3 years ago by Andre Offringa
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
-FROM lofareosc/lofar-pipeline:latest
+FROM ubuntu:20.04 as builder
-USER root
+ARG portable=True
+RUN export DEBIAN_FRONTEND=noninteractive && \ 
+    apt-get update && \
+    apt-get install -y \
+        bison \
+        build-essential \
+        casacore-data \
+        casacore-dev \
+        cmake \
+        flex \
+        gfortran \ 
+        git \
+	libblas-dev \
+        libcfitsio-dev \
+        libfftw3-dev \
+        libgsl-dev \
+	liblapack-dev \
+        liblua5.3-dev \
+        libhdf5-serial-dev \
+        libboost-filesystem-dev \
+        libboost-system-dev \
+        libboost-date-time-dev \
+        libboost-program-options-dev \
+        libboost-test-dev \
+        libgtkmm-3.0-dev \
+        libncurses5-dev \
+        libpng-dev \
+        libpython3-dev \
+        pkg-config \
+        python3-numpy \
+        wcslib-dev \        
+        wget && \
+    mkdir -p /src/
+WORKDIR /src/
+RUN \
+  git clone https://github.com/lofar-astron/LofarStMan --depth 1 && \
+  mkdir LofarStMan/build && \
+  cd LofarStMan/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+  make install -j`nproc`
+RUN \
+  git clone https://github.com/aroffringa/dysco.git --depth 1 && \
+  mkdir dysco/build && \
+  cd dysco/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+  make install -j`nproc`
+# IDG doesn't work with --depth 1, because it needs all branches to
+# determine its version :-(
+RUN \
+  git clone https://git.astron.nl/RD/idg.git && \
+  mkdir idg/build && \
+  cd idg/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+  make install -j`nproc`
+RUN \
+  git clone https://gitlab.com/aroffringa/aoflagger.git --depth 1 && \
+  mkdir aoflagger/build && \
+  cd aoflagger/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+  make install -j`nproc`
+RUN \
+  apt-get install -y libboost-python-dev libboost-numpy-dev && \
+  git clone https://github.com/lofar-astron/LOFARBeam.git --depth 1 && \
+  mkdir LOFARBeam/build && \
+  cd LOFARBeam/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+  make install -j`nproc`
+RUN \
+  git clone https://git.astron.nl/RD/EveryBeam.git --depth 1 && \
+  mkdir EveryBeam/build && \
+  cd EveryBeam/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+  make install -j`nproc`
+# We currently checkout a specific version of DP3 because there's
+# currently a bug in the solver in the master branch. Once this is
+# solved we should again use master, and the clone cmd should be changed
+# back to:
+#  git clone https://git.astron.nl/RD/DP3.git --depth 1 && \
+RUN \
+  git clone https://git.astron.nl/RD/DP3.git && \
+  cd DP3 && git checkout 1eadf43fee5ac76f48611fdd1fd464ebda8c0b6d && cd .. && \
+  mkdir DP3/build && \
+  cd DP3/build && \
+  cmake ../ \
+    -DCMAKE_INSTALL_PREFIX=/usr/local/ \
+    -DPORTABLE=${portable} \
+    -DAOFlagger_DIR=/usr/local/ && \
+  make install -j`nproc`
+RUN \
+  git clone https://gitlab.com/aroffringa/wsclean.git --depth 1 && \
+  mkdir wsclean/build && \
+  cd wsclean/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
+  make install -j`nproc`
+#---------------------------------------------------------------------------
+# The image is now rebuild without adding the source, to get a smaller image
+# Further binary and/or Python dependencies are also added here.
+#---------------------------------------------------------------------------
+FROM ubuntu:20.04 as runner
+COPY --from=builder /usr/local/ /usr/local
+RUN chmod +rx /usr/local/bin/*
+SHELL ["/bin/bash", "-c"]
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+        bison \
+    	git \
+        casacore-data \
+	casacore-tools \
+        flex \
+	gfortran \
+        libboost-date-time1.71 \
+        libboost-filesystem1.71 \
+        libboost-numpy1.71 \
+        libboost-program-options1.71 \
+        libboost-python1.71 \
+        libboost-system1.71 \
+        libboost-test1.71 \
+        libcasa-casa4 \
+	libcasa-fits4 \
+	libcasa-measures4 \
+	libcasa-ms4 \
+	libcasa-tables4 \
+        libcfitsio8 \
+        libfftw3-3 \
+	libgsl23 \
+        liblua5.3-0 \
+        libncurses5 \ 
+        libgtkmm-3.0 \
+        libhdf5-cpp-103 \
+        libpng16-16 \
+	nodejs \
+        pybind11-dev   \
+        python3 \
+        python3-casacore \
+	python3-h5py \
+	python3-matplotlib \
+        python3-numpy \
+	python3-pip \
+        python3-scipy \
+	python3-sphinx \
+	tar \
+	wget \
+        wcslib-dev && \
+    rm -rf /var/lib/apt/lists/*
+ARG TOIL_VERSION=3.20.0
+ARG CASACORE_TAG=master   
+ARG LSMTool_TAG=master
+ARG LoSoTo_TAG=master
+ARG STMAN_TAG=master
+ARG Dysco_TAG=master
+ARG LofarBeam_TAG=master
+ARG DP3_TAG=master
+ARG IDG_TAG=master
+ARG Everybeam_TAG
+ARG AOFlagger_TAG
+ARG RMextract_TAG=master
+ARG WSClean_TAG
+LABEL \
+  casacore.version.tag=${CASACORE_TAG} \
+  toil.version=${TOIL_VERSION} \
+  LSMTool.version.tag=${LSMTool_TAG} \
+  LoSoTo.version.tag=${LoSoTo_TAG} \
+  STMAN.version.tag=${STMAN_TAG} \
+  Dysco.version.tag=${Dysco_TAG} \
+  LOFARBeam.version.tag=${LofarBeam_TAG} \
+  DP3.version.tag=${DP3_TAG} \
+  IDG.version.tag=${IDG_TAG} \
+  Everybeam.version.tag=${Everybeam_TAG} \
+  AOFlagger.version.tag=${AOFlagger_TAG} \
+  RMextract.version.tag=${RMextract_TAG} \
+  WSClean.version.tag=${WSClean_TAG}
 ENV PREFACTOR_VERSION='master' \
    PREFACTOR_NAME=prefactor \
    LC_ALL=C.UTF-8 \
    PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
-    PYTHONPATH="/usr/local/bin:${PYTHONPATH}"
+    PYTHONPATH="/usr/local/bin:/usr/local/lib/python3.8/site-packages"
 RUN \
-  apt-get update && \
+# Install WSRT Measures (extra casacore data)
-  apt-get install -y vim wget casacore-tools && \
+# Note: The file on the ftp site is updated daily. When warnings regarding leap
-  rm -rf /var/cache/apt 
+# seconds appear, ignore them or regenerate the docker image.
-RUN \
+  wget -q -O /WSRT_Measures.ztar ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar \
-  pip3 install --upgrade pip && \
+    && cd /var/lib/casacore/data \
-  pip3 install h5py
+    && tar xfz /WSRT_Measures.ztar \
-RUN \
+    && rm /WSRT_Measures.ztar && \
-  pip3 install --upgrade pip && \
+  \
-  git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \
+# Install cwtool & cwl-runner
+  python3 -m pip install --upgrade pip && \
+  python3 -m pip install cwltool cwl-runner && \
+  \
+# Install Prefactor
+  git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} --branch ${PREFACTOR_VERSION} --depth 1 && \
  cd ${PREFACTOR_NAME} && \
-  git checkout ${PREFACTOR_VERSION} && \
-  git pull && \
  pip3 install --upgrade $PWD && \
  cd .. && \
-  rm -rfv ${PREFACTOR_NAME}
+  rm -rfv ${PREFACTOR_NAME} && \
+  \
+# A user 'lofaruser' is added (this might be legacy -- not sure this is
+# still relevant).
+  mkdir /home/lofaruser/ && \ 
+  groupadd -r lofaruser && \
+  useradd --no-log-init -r -g lofaruser lofaruser && \
+  chown lofaruser:lofaruser /home/lofaruser && \
+  \
+# Try to run the compiled tools to make sure they run without
+# a problem (e.g. no missing libraries).
+  aoflagger --version && \
+  DP3 --version && \
+  wsclean --version
-USER lofaruser
+WORKDIR /home/lofaruser
+USER lofaruser
--- a/Docker/Dockerfile-base
+++ b/Docker/Dockerfile-base
-FROM kernsuite/base:6 as builder
-SHELL ["/bin/bash", "-c"]
-RUN export DEBIAN_FRONTEND=noninteractive && \ 
-    apt-get update && \
-    apt-get install -y \
-        gfortran \ 
-        flex \
-        bison \
-        wcslib-dev \        
-        libncurses5-dev \
-        casacore-data \
-        casacore-dev \
-        libboost-python-dev \
-        libcfitsio-dev \
-        python-dev \
-        python3-numpy \
-        libcasa* \
-        cmake \
-        build-essential \
-        liblua5.3-dev \
-        libhdf5-serial-dev \
-        libarmadillo-dev \
-        libboost-filesystem-dev \
-        libboost-system-dev \
-        libboost-date-time-dev \
-        libboost-numpy-dev \
-        libboost-signals-dev \ 
-        libboost-program-options-dev \
-        libboost-test-dev \
-        pybind11-dev   \
-        libxml2-dev \
-        libpng-dev \
-        pkg-config \
-        libgtkmm-3.0-dev \
-        git \
-        wget \
-        libfftw3-dev \
-        libgsl-dev
-RUN mkdir -p /src/
-WORKDIR /src/
-RUN \
-  git clone https://github.com/lofar-astron/LofarStMan && \
-  mkdir LofarStMan/build && \
-  cd LofarStMan/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://github.com/aroffringa/dysco.git && \
-  mkdir dysco/build && \
-  cd dysco/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://git.astron.nl/RD/idg.git && \
-  mkdir idg/build && \
-  cd idg/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://gitlab.com/aroffringa/aoflagger.git && \
-  mkdir aoflagger/build && \
-  cd aoflagger/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DPORTABLE=True && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://github.com/lofar-astron/LOFARBeam.git && \
-  mkdir LOFARBeam/build && \
-  cd LOFARBeam/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3.6 && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://git.astron.nl/RD/EveryBeam.git && \
-  mkdir EveryBeam/build && \
-  cd EveryBeam/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://git.astron.nl/RD/DP3.git && \
-  mkdir DP3/build && \
-  cd DP3/build && \
-  cmake ../ \
-    -DCMAKE_INSTALL_PREFIX=/usr/local/ \
-    -DPYTHON_EXECUTABLE=/usr/bin/python3.6 \
-    -DPORTABLE=True \
-    -DAOFlagger_DIR=/usr/local/ && \
-  make install -j`nproc --all`
-RUN \
-  git clone https://gitlab.com/aroffringa/wsclean.git && \
-  mkdir wsclean/build && \
-  cd wsclean/build && \
-  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
-  make install -j`nproc --all`
-#---------------------------------------------------------------------------
-# The image is now rebuild without adding the source, to get a smaller image
-# Further binary and/or Python dependencies are also added here.
-#---------------------------------------------------------------------------
-FROM kernsuite/base:6 as runner
-COPY --from=builder /usr/local/ /usr/local
-RUN chmod +rx /usr/local/bin/*
-SHELL ["/bin/bash", "-c"]
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update && \
-    apt-get install -y \
-        casacore-data \
-        wcslib-dev \        
-        libarmadillo8 \
-        bison \
-        libncurses5 \ 
-        flex \
-        libboost-date-time1.65.1 \
-        libboost-filesystem1.65.1 \
-        libboost-numpy1.65.1 \
-        libboost-python1.65.1 \
-        libboost-program-options1.65.1 \
-        libboost-system1.65.1 \
-        libboost-signals1.65.1 \
-        libboost-test1.65.1 \
-        libboost-python1.65-dev \
-        libstationresponse3 \
-        liblua5.3-dev \
-        libcasa-* \
-        pybind11-dev   \
-        libcfitsio5 \
-        libcfitsio-dev \
-        libgtkmm-3.0 \
-        libfftw3-3 \
-        libhdf5-cpp-100 \
-        libpng16-16 \
-        libxml2 \
-        python3.7 \
-        python3-casacore \
-	dysco \
-        python3-numpy \
-        python3-scipy && \
-    rm -rf /var/lib/apt/lists/*
-ARG CASACORE_TAG=master
-LABEL casacore.version.tag=${CASACORE_TAG}
-ARG TOIL_VERSION=3.20.0
-LABEL toil.version=${TOIL_VERSION}
-ARG LSMTool_TAG=master
-LABEL LSMTool.version.tag=${LSMTool_TAG}
-ARG LoSoTo_TAG=master
-LABEL LoSoTo.version.tag=${LoSoTo_TAG}
-ARG STMAN_TAG=master
-LABEL STMAN.version.tag=${STMAN_TAG}
-ARG Dysco_TAG=master
-LABEL Dysco.version.tag=${Dysco_TAG}
-ARG LofarBeam_TAG=master
-LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
-ARG DP3_TAG=master
-LABEL DP3.version.tag=${DP3_TAG}
-ARG IDG_TAG=master
-LABEL IDG.version.tag=${IDG_TAG}
-ARG Everybeam_TAG
-LABEL Everybeam.version.tag=${Everybeam_TAG}
-ARG Aoflagger_TAG
-LABEL Aoflagger.version.tag=${Aoflagger_TAG}
-ARG RMextract_TAG
-LABEL RMextract.version.tag=${RMExtract_TAG}
-ARG wsclean_TAG
-LABEL wsclean.version.tag=${wsclean_TAG}
-RUN \
-  apt-get --allow-releaseinfo-change update -y && \
-  apt-get install -y nodejs python3-pip git gfortran && \
-  python3 \
-    -m pip install cwltool cwl-runner \
-    -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
-       "toil[cwl]"==${TOIL_VERSION} \
-       matplotlib && \
-  python3 \
-    -m pip install \
-    -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo" \
-    ipython "git+https://github.com/lofar-astron/RMextract.git@${RMextract_TAG}#egg=RMextract"
-RUN \
-  mkdir /home/lofaruser/ && \ 
-  groupadd -r lofaruser && \
-  useradd --no-log-init -r -g lofaruser lofaruser && \
-  chown lofaruser:lofaruser /home/lofaruser
-WORKDIR /home/lofaruser
-USER lofaruser
-ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
--- a/Docker/build_docker.sh
+++ b/Docker/build_docker.sh
@@ -3,8 +3,5 @@ set -e
 DOCKER_TAG=latest
-docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
+docker build ${PWD} -f Dockerfile -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
-docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
-docker build --no-cache ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
 docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}