diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..c0ef1859c3910115fecd9868fdaef4f2da59fc1f
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,6 @@
+.git/
+build/
+dist/
+docs/build/
+*.egg-info/
+venv/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6b9999479a2a86b229ef780a7640dcd52a0cc278..6208b88961c46f89439c39ed488880318315ce69 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,21 +120,36 @@ run_hba_calibrator:
needs: ["versioning", "download_data"]
image: $INTEGRATION_IMAGE
script:
- - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+ - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
+ after_script:
+ - find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
+ artifacts:
+ paths:
+ - hba_calibrator_logs.tar.gz
+ when: on_failure
run_hba_target:
stage: tests
needs: ["versioning", "download_data"]
image: $INTEGRATION_IMAGE
script:
- - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results workflows/HBA_target.cwl test_jobs/HBA_target.json
+ - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
+ after_script:
+ - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
+ artifacts:
+ paths:
+ - hba_target_logs.tar.gz
+ when: on_failure
build_doc:
stage: docs
needs: ["versioning", "download_data"]
image: $INTEGRATION_IMAGE
+ before_script:
+ - apt-get update
+ - apt-get install -y make python3-sphinx
script:
- cd docs
- make html
diff --git a/Docker/Dockerfile-base b/Docker/Dockerfile-base
index 393372f27bb07a17afd96880465654c974965270..40956bc5346efe029b3291ff1bbd7fd5df91d573 100644
--- a/Docker/Dockerfile-base
+++ b/Docker/Dockerfile-base
@@ -10,8 +10,7 @@ FROM ubuntu:20.04 as builder
# increment the value.
ENV DOCKER_BUILD_FORCE_GENERATION=0
-ARG portable=True
-
+# Install all build-time dependencies
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
@@ -23,195 +22,207 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
flex \
gfortran \
git \
- libblas-dev \
- libcfitsio-dev \
- libfftw3-dev \
- libgsl-dev \
- liblapack-dev \
- liblua5.3-dev \
- libhdf5-serial-dev \
- libboost-filesystem-dev \
- libboost-system-dev \
+ libblas-dev \
libboost-date-time-dev \
+ libboost-filesystem-dev \
+ libboost-numpy-dev \
libboost-program-options-dev \
+ libboost-python-dev \
+ libboost-system-dev \
libboost-test-dev \
+ libcfitsio-dev \
+ libfftw3-dev \
+ libgsl-dev \
libgtkmm-3.0-dev \
+ libhdf5-serial-dev \
+ liblapack-dev \
+ liblua5.3-dev \
libncurses5-dev \
libpng-dev \
libpython3-dev \
pkg-config \
+ python3 \
+ python3-casacore \
python3-numpy \
- wcslib-dev \
+ wcslib-dev \
wget && \
- mkdir -p /src/
-
-WORKDIR /src/
-
-RUN \
- git clone https://github.com/lofar-astron/LofarStMan --depth 1 && \
- mkdir LofarStMan/build && \
- cd LofarStMan/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
- make install -j`nproc`
-
-RUN \
- git clone https://github.com/aroffringa/dysco.git --depth 1 && \
- mkdir dysco/build && \
- cd dysco/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
- make install -j`nproc`
+ mkdir -p /src
+
+WORKDIR /src
+
+# Build portable binaries by default
+ARG PORTABLE=TRUE
+
+ARG LOFARSTMAN_VERSION=master
+RUN git clone --depth 1 --branch ${LOFARSTMAN_VERSION} \
+ https://github.com/lofar-astron/LofarStMan && \
+ mkdir LofarStMan/build && \
+ cd LofarStMan/build && \
+ cmake .. -DPORTABLE=${PORTABLE} && \
+ make install -j`nproc`
+
+ARG DYSCO_VERSION=master
+RUN git clone --depth 1 --branch ${DYSCO_VERSION} \
+ https://github.com/aroffringa/dysco.git && \
+ mkdir dysco/build && \
+ cd dysco/build && \
+ cmake .. -DPORTABLE=${PORTABLE} && \
+ make install -j`nproc`
+ARG IDG_VERSION=master
# IDG doesn't work with --depth 1, because it needs all branches to
# determine its version :-(
-RUN \
- git clone https://git.astron.nl/RD/idg.git && \
- mkdir idg/build && \
- cd idg/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
- make install -j`nproc`
-
-RUN \
- git clone https://gitlab.com/aroffringa/aoflagger.git --depth 1 && \
- mkdir aoflagger/build && \
- cd aoflagger/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
- make install -j`nproc`
-
-RUN \
- apt-get install -y libboost-python-dev libboost-numpy-dev && \
- git clone https://github.com/lofar-astron/LOFARBeam.git --depth 1 && \
- mkdir LOFARBeam/build && \
- cd LOFARBeam/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
- make install -j`nproc`
-
-RUN \
- git clone https://git.astron.nl/RD/EveryBeam.git --depth 1 && \
- mkdir EveryBeam/build && \
- cd EveryBeam/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
- make install -j`nproc`
-
-RUN \
- git clone https://git.astron.nl/RD/DP3.git --depth 1 && \
- mkdir DP3/build && \
- cd DP3/build && \
- cmake ../ \
- -DCMAKE_INSTALL_PREFIX=/usr/local/ \
- -DPORTABLE=${portable} \
- -DAOFlagger_DIR=/usr/local/ && \
- make install -j`nproc`
-
-RUN \
- git clone https://gitlab.com/aroffringa/wsclean.git --depth 1 && \
- mkdir wsclean/build && \
- cd wsclean/build && \
- cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DPORTABLE=${portable} && \
- make install -j`nproc`
+RUN git clone --branch ${IDG_VERSION} \
+ https://git.astron.nl/RD/idg.git && \
+ mkdir idg/build && \
+ cd idg/build && \
+ cmake .. && \
+ make install -j`nproc`
+
+ARG AOFLAGGER_VERSION=master
+RUN git clone --depth 1 --branch ${AOFLAGGER_VERSION} \
+ https://gitlab.com/aroffringa/aoflagger.git && \
+ mkdir aoflagger/build && \
+ cd aoflagger/build && \
+ cmake .. -DPORTABLE=${PORTABLE} && \
+ make install -j`nproc`
+
+ARG LOFARBEAM_VERSION=master
+RUN git clone --depth 1 --branch ${LOFARBEAM_VERSION} \
+ https://github.com/lofar-astron/LOFARBeam.git && \
+ mkdir LOFARBeam/build && \
+ cd LOFARBeam/build && \
+ cmake .. && \
+ make install -j`nproc`
+
+ARG EVERYBEAM_VERSION=master
+RUN git clone --depth 1 --branch ${EVERYBEAM_VERSION} \
+ https://git.astron.nl/RD/EveryBeam.git && \
+ mkdir EveryBeam/build && \
+ cd EveryBeam/build && \
+ cmake .. && \
+ make install -j`nproc`
+
+ARG DP3_VERSION=master
+RUN git clone --depth 1 --branch ${DP3_VERSION} \
+ https://git.astron.nl/RD/DP3.git && \
+ mkdir DP3/build && \
+ cd DP3/build && \
+ cmake .. -DPORTABLE=${PORTABLE} && \
+ make install -j`nproc`
+
+ARG WSCLEAN_VERSION=master
+RUN git clone --depth 1 --branch ${WSCLEAN_VERSION} \
+ https://gitlab.com/aroffringa/wsclean.git && \
+ mkdir wsclean/build && \
+ cd wsclean/build && \
+ cmake .. -DPORTABLE=${PORTABLE} && \
+ make install -j`nproc`
+
+# Do not use `pip` from the Debian repository, but fetch it from PyPA.
+# This way, we are sure that the latest versions of `pip`, `setuptools`, and
+# `wheel` are installed in /usr/local, the only directory we're going to copy
+# over to the next build stage.
+RUN wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
+
+# Install required python packages
+RUN python3 -m pip install --no-cache-dir --upgrade \
+ cwl-runner \
+ cwltool
+
+# Install current version of Prefactor CWL. By adding this to the base image
+# we can speed up the build of the final image, because all of Prefactor's
+# dependencies will have been installed already and (probably) don't need to
+# be updated.
+COPY . prefactor
+RUN python3 -m pip install --no-cache-dir --upgrade ./prefactor
+
#---------------------------------------------------------------------------
-# The image is now rebuild without adding the source, to get a smaller image
-# Further binary and/or Python dependencies are also added here.
+# The image will now be rebuilt without adding the sources, in order to
+# reduce the size of the image.
#---------------------------------------------------------------------------
FROM ubuntu:20.04 as runner
-COPY --from=builder /usr/local/ /usr/local
+COPY --from=builder /usr/local /usr/local
RUN chmod +rx /usr/local/bin/*
SHELL ["/bin/bash", "-c"]
+# Set default versions. Can be overridden from the command-line
+ARG LOFARSTMAN_VERSION=master
+ARG DYSCO_VERSION=master
+ARG IDG_VERSION=master
+ARG AOFLAGGER_VERSION=master
+ARG LOFARBEAM_VERSION=master
+ARG EVERYBEAM_VERSION=master
+ARG DP3_VERSION=master
+ARG WSCLEAN_VERSION=master
+
+# Add version information to the metadata of the image
+LABEL \
+ nl.astron.prefactor.lofarstman.version=${LOFARSTMAN_VERSION} \
+ nl.astron.prefactor.dysco.version=${DYSCO_VERSION} \
+ nl.astron.prefactor.idg.version=${IDG_VERSION} \
+ nl.astron.prefactor.aoflagger.version=${AOFLAGGER_VERSION} \
+ nl.astron.prefactor.lofarbeam.version=${LOFARBEAM_VERSION} \
+ nl.astron.prefactor.everybeam.version=${EVERYBEAM_VERSION} \
+ nl.astron.prefactor.dp3.version=${DP3_VERSION} \
+ nl.astron.prefactor.wsclean.version=${WSCLEAN_VERSION}
+
+# Only install run-time required packages
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
- bison \
- git \
- casacore-data \
- casacore-tools \
- flex \
- gfortran \
- libboost-date-time1.71 \
- libboost-filesystem1.71 \
- libboost-numpy1.71 \
- libboost-program-options1.71 \
- libboost-python1.71 \
- libboost-system1.71 \
- libboost-test1.71 \
+ casacore-tools \
+ libatkmm-1.6-1v5 \
+ libboost-date-time1.71.0 \
+ libboost-filesystem1.71.0 \
+ libboost-program-options1.71.0 \
+ libboost-python1.71.0 \
+ libcairomm-1.0-1v5 \
libcasa-casa4 \
- libcasa-fits4 \
- libcasa-measures4 \
- libcasa-ms4 \
- libcasa-tables4 \
+ libcasa-fits4 \
+ libcasa-measures4 \
+ libcasa-ms4 \
+ libcasa-python3-4 \
+ libcasa-scimath4 \
+ libcasa-tables4 \
libcfitsio8 \
- libfftw3-3 \
- libgsl23 \
- liblua5.3-0 \
- libncurses5 \
- libgtkmm-3.0 \
+ libfftw3-double3 \
+ libfftw3-single3 \
+ libglibmm-2.4-1v5 \
+ libgomp1 \
+ libgsl23 \
+ libgtkmm-3.0-1v5 \
+ libhdf5-103 \
libhdf5-cpp-103 \
+ liblapack3 \
+ liblua5.3-0 \
+ libpangomm-1.4-1v5 \
libpng16-16 \
- nodejs \
- pybind11-dev \
+ libpython3.8 \
+ libsigc++-2.0-0v5 \
+ libstdc++6 \
+ nodejs \
python3 \
python3-casacore \
- python3-h5py \
- python3-matplotlib \
- python3-numpy \
- python3-pip \
- python3-scipy \
- python3-sphinx \
- tar \
- wget \
- wcslib-dev && \
+ python3-distutils \
+ wget && \
rm -rf /var/lib/apt/lists/*
-ARG TOIL_VERSION=3.20.0
-ARG CASACORE_TAG=master
-ARG LSMTool_TAG=master
-ARG LoSoTo_TAG=master
-ARG STMAN_TAG=master
-ARG Dysco_TAG=master
-ARG LofarBeam_TAG=master
-ARG DP3_TAG=master
-ARG IDG_TAG=master
-ARG Everybeam_TAG
-ARG AOFlagger_TAG
-ARG RMextract_TAG=master
-ARG WSClean_TAG
-
-LABEL \
- casacore.version.tag=${CASACORE_TAG} \
- toil.version=${TOIL_VERSION} \
- LSMTool.version.tag=${LSMTool_TAG} \
- LoSoTo.version.tag=${LoSoTo_TAG} \
- STMAN.version.tag=${STMAN_TAG} \
- Dysco.version.tag=${Dysco_TAG} \
- LOFARBeam.version.tag=${LofarBeam_TAG} \
- DP3.version.tag=${DP3_TAG} \
- IDG.version.tag=${IDG_TAG} \
- Everybeam.version.tag=${Everybeam_TAG} \
- AOFlagger.version.tag=${AOFlagger_TAG} \
- RMextract.version.tag=${RMextract_TAG} \
- WSClean.version.tag=${WSClean_TAG}
-
-ENV LC_ALL=C.UTF-8 \
- PYTHONPATH="/usr/local/bin:/usr/local/lib/python3.8/site-packages"
-
-RUN \
# Install WSRT Measures (extra casacore data)
# Note: The file on the ftp site is updated daily. When warnings regarding leap
# seconds appear, ignore them or regenerate the docker image.
- wget -q -O /WSRT_Measures.ztar ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar \
- && cd /var/lib/casacore/data \
- && tar xfz /WSRT_Measures.ztar \
- && rm /WSRT_Measures.ztar && \
- \
-# Install cwtool & cwl-runner
- python3 -m pip install --upgrade pip && \
- python3 -m pip install cwltool cwl-runner && \
- \
+RUN wget -q -O /WSRT_Measures.ztar \
+ ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
+ cd /var/lib/casacore/data && \
+ tar xfz /WSRT_Measures.ztar && \
+ rm /WSRT_Measures.ztar
+
# Try to run the compiled tools to make sure they run without
# a problem (e.g. no missing libraries).
- aoflagger --version && \
- DP3 --version && \
- wsclean --version
-
\ No newline at end of file
+RUN aoflagger --version && \
+ DP3 --version && \
+ wsclean --version
diff --git a/Docker/Dockerfile-prefactor b/Docker/Dockerfile-prefactor
index bc6424ee3a762ea3bf29f588b8acce7b6b42c63b..f25a6eb63c6148755d86106f51a870a82711b1f2 100644
--- a/Docker/Dockerfile-prefactor
+++ b/Docker/Dockerfile-prefactor
@@ -1,24 +1,20 @@
ARG BASE_TAG=latest
FROM $BASE_TAG
-ENV PREFACTOR_VERSION='master' \
- PREFACTOR_NAME=prefactor \
- PREFACTOR_DATA_ROOT=/usr/local/share/prefactor
+ENV PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
+ PYTHONPATH=/usr/local/bin
# Install Prefactor CWL
-COPY . /${PREFACTOR_NAME}
-RUN cd /${PREFACTOR_NAME} && \
- pip3 install --upgrade $PWD && \
- cd .. && \
- rm -rfv ${PREFACTOR_NAME} && \
- \
-# A user 'lofaruser' is added (this might be legacy -- not sure this is
-# still relevant).
- mkdir /home/lofaruser/ && \
- groupadd -r lofaruser && \
- useradd --no-log-init -r -g lofaruser lofaruser && \
- chown lofaruser:lofaruser /home/lofaruser
+COPY . /tmp/prefactor
-WORKDIR /home/lofaruser
+RUN python3 -m pip install --no-cache-dir --upgrade /tmp/prefactor && \
+ rm -rf /tmp/prefactor
-USER lofaruser
+# # A user 'lofaruser' is added (this might be legacy -- not sure this is
+# # still relevant).
+# RUN groupadd -r lofaruser && \
+# useradd -r -l -m -g lofaruser lofaruser
+
+# WORKDIR /home/lofaruser
+
+# USER lofaruser
diff --git a/setup.py b/setup.py
index bcb636599bcc056edb9ac2d4492285cfe0fb1a15..14dfc6967178531b4d09705c4015586b0f442d6e 100644
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,7 @@ setup(
'Topic :: Scientific/Engineering :: Astronomy'],
platforms='any',
install_requires=[
- 'aplpy', 'astropy', 'bdsf', 'losoto', 'lsmtool',
+ 'aplpy', 'astropy', 'bdsf', 'h5py', 'losoto', 'lsmtool',
'matplotlib', 'numpy', 'python-casacore', 'RMextract', 'scipy'],
scripts=glob.glob('scripts/*'),
data_files=data_files