Merge branch 'add-base-dockerfile' into 'master'

Add a base docker file and base the main dockerfile on this

See merge request eosc/prefactor3-cwl!27

Former-commit-id: 5b1574c4 [formerly 9635ff9f]
Former-commit-id: 9b30b935
Former-commit-id: 8f5a6d36

Docker/Dockerfile

@@ -6,12 +6,19 @@ ENV PREFACTOR_VERSION='master' \
     PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
     PYTHONPATH="/usr/local/bin:${PYTHONPATH}"
 
-RUN apt update && apt install -y vim wget casacore-tools && rm -rf /var/cache/apt 
-RUN pip3 install --upgrade pip && \
-    git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \
-    cd ${PREFACTOR_NAME} && git checkout ${PREFACTOR_VERSION} && git pull && \
-    pip3 install --upgrade $PWD && cd .. && \
-    rm -rfv ${PREFACTOR_NAME}
+RUN \
+  apt-get update && \
+  apt-get install -y vim wget casacore-tools && \
+  rm -rf /var/cache/apt 
+RUN \
+  pip3 install --upgrade pip && \
+  git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \
+  cd ${PREFACTOR_NAME} && \
+  git checkout ${PREFACTOR_VERSION} && \
+  git pull && \
+  pip3 install --upgrade $PWD && \
+  cd .. && \
+  rm -rfv ${PREFACTOR_NAME}
 
-ENTRYPOINT ["/home/lofaruser/.entrypoint"]
 USER lofaruser
+

Docker/Dockerfile-base

+FROM kernsuite/base:6 as builder
+SHELL ["/bin/bash", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \ 
+    apt-get update && \
+    apt-get install -y \
+        gfortran \ 
+        flex \
+        bison \
+        wcslib-dev \        
+        libncurses5-dev \
+        casacore-data \
+        casacore-dev \
+        libboost-python-dev \
+        libcfitsio-dev \
+        python-dev \
+        python3-numpy \
+        libcasa* \
+        cmake \
+        build-essential \
+        liblua5.3-dev \
+        libhdf5-serial-dev \
+        libarmadillo-dev \
+        libboost-filesystem-dev \
+        libboost-system-dev \
+        libboost-date-time-dev \
+        libboost-numpy-dev \
+        libboost-signals-dev \ 
+        libboost-program-options-dev \
+        libboost-test-dev \
+        pybind11-dev   \
+        libxml2-dev \
+        libpng-dev \
+        pkg-config \
+        libgtkmm-3.0-dev \
+        git \
+        wget \
+        libfftw3-dev \
+        libgsl-dev
+        
+RUN mkdir -p /src/
+WORKDIR /src/
+
+RUN \
+  git clone https://github.com/lofar-astron/LofarStMan && \
+  mkdir LofarStMan/build && \
+  cd LofarStMan/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+  make install -j`nproc --all`
+
+RUN \
+  git clone https://github.com/aroffringa/dysco.git && \
+  mkdir dysco/build && \
+  cd dysco/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+  make install -j`nproc --all`
+    
+RUN \
+  git clone https://git.astron.nl/RD/idg.git && \
+  mkdir idg/build && \
+  cd idg/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+  make install -j`nproc --all`
+
+RUN \
+  git clone https://gitlab.com/aroffringa/aoflagger.git && \
+  mkdir aoflagger/build && \
+  cd aoflagger/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DPORTABLE=True && \
+  make install -j`nproc --all`
+
+RUN \
+  git clone https://github.com/lofar-astron/LOFARBeam.git && \
+  mkdir LOFARBeam/build && \
+  cd LOFARBeam/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPYTHON_EXECUTABLE=/usr/bin/python3.6 && \
+  make install -j`nproc --all`
+
+RUN \
+  git clone https://git.astron.nl/RD/EveryBeam.git && \
+  mkdir EveryBeam/build && \
+  cd EveryBeam/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+  make install -j`nproc --all`
+    
+RUN \
+  git clone https://git.astron.nl/RD/DP3.git && \
+  mkdir DP3/build && \
+  cd DP3/build && \
+  cmake ../ \
+    -DCMAKE_INSTALL_PREFIX=/usr/local/ \
+    -DPYTHON_EXECUTABLE=/usr/bin/python3.6 \
+    -DPORTABLE=True \
+    -DAOFlagger_DIR=/usr/local/ && \
+  make install -j`nproc --all`
+
+RUN \
+  git clone https://gitlab.com/aroffringa/wsclean.git && \
+  mkdir wsclean/build && \
+  cd wsclean/build && \
+  cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=True && \
+  make install -j`nproc --all`
+
+#---------------------------------------------------------------------------
+# The image is now rebuild without adding the source, to get a smaller image
+# Further binary and/or Python dependencies are also added here.
+#---------------------------------------------------------------------------
+FROM kernsuite/base:6 as runner
+
+COPY --from=builder /usr/local/ /usr/local
+RUN chmod +rx /usr/local/bin/*
+
+SHELL ["/bin/bash", "-c"]
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+        casacore-data \
+        wcslib-dev \        
+        libarmadillo8 \
+        bison \
+        libncurses5 \ 
+        flex \
+        libboost-date-time1.65.1 \
+        libboost-filesystem1.65.1 \
+        libboost-numpy1.65.1 \
+        libboost-python1.65.1 \
+        libboost-program-options1.65.1 \
+        libboost-system1.65.1 \
+        libboost-signals1.65.1 \
+        libboost-test1.65.1 \
+        libboost-python1.65-dev \
+        libstationresponse3 \
+        liblua5.3-dev \
+        libcasa-* \
+        pybind11-dev   \
+        libcfitsio5 \
+        libcfitsio-dev \
+        libgtkmm-3.0 \
+        libfftw3-3 \
+        libhdf5-cpp-100 \
+        libpng16-16 \
+        libxml2 \
+        python3.7 \
+        python3-casacore \
+	dysco \
+        python3-numpy \
+        python3-scipy && \
+    rm -rf /var/lib/apt/lists/*
+
+ARG CASACORE_TAG=master
+LABEL casacore.version.tag=${CASACORE_TAG}
+    
+ARG TOIL_VERSION=3.20.0
+LABEL toil.version=${TOIL_VERSION}
+
+ARG LSMTool_TAG=master
+LABEL LSMTool.version.tag=${LSMTool_TAG}
+
+ARG LoSoTo_TAG=master
+LABEL LoSoTo.version.tag=${LoSoTo_TAG}
+
+ARG STMAN_TAG=master
+LABEL STMAN.version.tag=${STMAN_TAG}
+
+ARG Dysco_TAG=master
+LABEL Dysco.version.tag=${Dysco_TAG}
+
+ARG LofarBeam_TAG=master
+LABEL LOFARBeam.version.tag=${LofarBeam_TAG}
+
+ARG DP3_TAG=master
+LABEL DP3.version.tag=${DP3_TAG}
+
+ARG IDG_TAG=master
+LABEL IDG.version.tag=${IDG_TAG}
+
+ARG Everybeam_TAG
+LABEL Everybeam.version.tag=${Everybeam_TAG}
+
+ARG Aoflagger_TAG
+LABEL Aoflagger.version.tag=${Aoflagger_TAG}
+
+ARG RMextract_TAG
+LABEL RMextract.version.tag=${RMExtract_TAG}
+
+ARG wsclean_TAG
+LABEL wsclean.version.tag=${wsclean_TAG}
+    
+RUN \
+  apt-get --allow-releaseinfo-change update -y && \
+  apt-get install -y nodejs python3-pip git gfortran && \
+  python3 \
+    -m pip install cwltool cwl-runner \
+    -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" \
+       "toil[cwl]"==${TOIL_VERSION} \
+       matplotlib && \
+  python3 \
+    -m pip install \
+    -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo" \
+    ipython "git+https://github.com/lofar-astron/RMextract.git@${RMextract_TAG}#egg=RMextract"
+
+RUN \
+  mkdir /home/lofaruser/ && \ 
+  groupadd -r lofaruser && \
+  useradd --no-log-init -r -g lofaruser lofaruser && \
+  chown lofaruser:lofaruser /home/lofaruser
+
+WORKDIR /home/lofaruser
+
+USER lofaruser
+ENV PYTHONPATH=/usr/local/lib/python3.6/site-packages/
+

Docker/build_docker.sh

 #! /bin/bash
 set -e
 
-
 DOCKER_TAG=latest
 
+docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
+docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
+
 docker build ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
 docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}