Skip to content
Snippets Groups Projects
Commit 9dfced6d authored by Matthijs van der Wild's avatar Matthijs van der Wild
Browse files

Merge branch 'no-container' into 'master'

Removed Dockerfiles

See merge request !29
parents 24531ebc 2725bcbc
No related branches found
No related tags found
1 merge request!29Removed Dockerfiles
Showing
with 33 additions and 335 deletions
#!/bin/bash
set -e
exec "$@"
FROM ubuntu:20.04 as builder
# This Docker image builds the dependencies for the Rapthor pipeline.
# It lives on the head of its dependencies.
# The Docker image is cached and not updated on every build.
# When this file changes it forces an update of the Docker image.
# Increment this value to get a fresh build.
# Note decrementing may give an older cached value, therefore only
# increment the value.
ENV DOCKER_BUILD_FORCE_GENERATION=0
# Install all build-time dependencies
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
bison \
build-essential \
casacore-data \
casacore-dev \
cmake \
flex \
gfortran \
git \
libblas-dev \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-numpy-dev \
libboost-program-options-dev \
libboost-python-dev \
libboost-system-dev \
libboost-test-dev \
libcfitsio-dev \
libfftw3-dev \
libgsl-dev \
libgtkmm-3.0-dev \
libhdf5-serial-dev \
liblapack-dev \
liblua5.3-dev \
libncurses5-dev \
libpng-dev \
libpython3-dev \
pkg-config \
python3 \
python3-casacore \
python3-numpy \
wcslib-dev \
wget && \
mkdir -p /src
WORKDIR /src
# Build portable binaries by default
ARG PORTABLE=TRUE
ARG LOFARSTMAN_VERSION=master
RUN git clone --depth 1 --branch ${LOFARSTMAN_VERSION} \
https://github.com/lofar-astron/LofarStMan && \
mkdir LofarStMan/build && \
cd LofarStMan/build && \
cmake .. -DPORTABLE=${PORTABLE} && \
make install -j`nproc`
ARG DYSCO_VERSION=master
RUN git clone --depth 1 --branch ${DYSCO_VERSION} \
https://github.com/aroffringa/dysco.git && \
mkdir dysco/build && \
cd dysco/build && \
cmake .. -DPORTABLE=${PORTABLE} && \
make install -j`nproc`
ARG IDG_VERSION=master
# IDG doesn't work with --depth 1, because it needs all branches to
# determine its version :-(
RUN git clone --branch ${IDG_VERSION} \
https://git.astron.nl/RD/idg.git && \
mkdir idg/build && \
cd idg/build && \
cmake .. && \
make install -j`nproc`
ARG AOFLAGGER_VERSION=master
RUN git clone --depth 1 --branch ${AOFLAGGER_VERSION} \
https://gitlab.com/aroffringa/aoflagger.git && \
mkdir aoflagger/build && \
cd aoflagger/build && \
cmake .. -DPORTABLE=${PORTABLE} && \
make install -j`nproc`
ARG LOFARBEAM_VERSION=master
RUN git clone --depth 1 --branch ${LOFARBEAM_VERSION} \
https://github.com/lofar-astron/LOFARBeam.git && \
mkdir LOFARBeam/build && \
cd LOFARBeam/build && \
cmake .. && \
make install -j`nproc`
ARG EVERYBEAM_VERSION=master
RUN git clone --depth 1 --branch ${EVERYBEAM_VERSION} \
https://git.astron.nl/RD/EveryBeam.git && \
mkdir EveryBeam/build && \
cd EveryBeam/build && \
cmake .. && \
make install -j`nproc`
ARG DP3_VERSION=master
RUN git clone --depth 1 --branch ${DP3_VERSION} \
https://git.astron.nl/RD/DP3.git && \
mkdir DP3/build && \
cd DP3/build && \
cmake .. -DPORTABLE=${PORTABLE} && \
make install -j`nproc`
ARG WSCLEAN_VERSION=master
RUN git clone --depth 1 --branch ${WSCLEAN_VERSION} \
https://gitlab.com/aroffringa/wsclean.git && \
mkdir wsclean/build && \
cd wsclean/build && \
cmake .. -DPORTABLE=${PORTABLE} && \
make install -j`nproc`
# Do not use `pip` from the Debian repository, but fetch it from PyPA.
# This way, we are sure that the latest versions of `pip`, `setuptools`, and
# `wheel` are installed in /usr/local, the only directory we're going to copy
# over to the next build stage.
RUN wget https://bootstrap.pypa.io/get-pip.py && \
python3 get-pip.py
# Install required python packages
RUN python3 -m pip install --no-cache-dir --upgrade \
cwl-runner \
cwltool
# Install current version of Prefactor CWL. By adding this to the base image
# we can speed up the build of the final image, because all of Prefactor's
# dependencies will have been installed already and (probably) don't need to
# be updated.
COPY . prefactor
RUN python3 -m pip install --no-cache-dir --upgrade ./prefactor
#---------------------------------------------------------------------------
# The image will now be rebuilt without adding the sources, in order to
# reduce the size of the image.
#---------------------------------------------------------------------------
FROM ubuntu:20.04 as runner
COPY --from=builder /usr/local /usr/local
RUN chmod +rx /usr/local/bin/*
SHELL ["/bin/bash", "-c"]
# Set default versions. Can be overridden from the command-line
ARG LOFARSTMAN_VERSION=master
ARG DYSCO_VERSION=master
ARG IDG_VERSION=master
ARG AOFLAGGER_VERSION=master
ARG LOFARBEAM_VERSION=master
ARG EVERYBEAM_VERSION=master
ARG DP3_VERSION=master
ARG WSCLEAN_VERSION=master
# Add version information to the metadata of the image
LABEL \
nl.astron.prefactor.lofarstman.version=${LOFARSTMAN_VERSION} \
nl.astron.prefactor.dysco.version=${DYSCO_VERSION} \
nl.astron.prefactor.idg.version=${IDG_VERSION} \
nl.astron.prefactor.aoflagger.version=${AOFLAGGER_VERSION} \
nl.astron.prefactor.lofarbeam.version=${LOFARBEAM_VERSION} \
nl.astron.prefactor.everybeam.version=${EVERYBEAM_VERSION} \
nl.astron.prefactor.dp3.version=${DP3_VERSION} \
nl.astron.prefactor.wsclean.version=${WSCLEAN_VERSION}
# Only install run-time required packages
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
casacore-tools \
libatkmm-1.6-1v5 \
libboost-date-time1.71.0 \
libboost-filesystem1.71.0 \
libboost-program-options1.71.0 \
libboost-python1.71.0 \
libcairomm-1.0-1v5 \
libcasa-casa4 \
libcasa-fits4 \
libcasa-measures4 \
libcasa-ms4 \
libcasa-python3-4 \
libcasa-scimath4 \
libcasa-tables4 \
libcfitsio8 \
libfftw3-double3 \
libfftw3-single3 \
libglibmm-2.4-1v5 \
libgomp1 \
libgsl23 \
libgtkmm-3.0-1v5 \
libhdf5-103 \
libhdf5-cpp-103 \
liblapack3 \
liblua5.3-0 \
libpangomm-1.4-1v5 \
libpng16-16 \
libpython3.8 \
libsigc++-2.0-0v5 \
libstdc++6 \
nodejs \
python3 \
python3-casacore \
python3-distutils \
wget && \
rm -rf /var/lib/apt/lists/*
# Install WSRT Measures (extra casacore data)
# Note: The file on the ftp site is updated daily. When warnings regarding leap
# seconds appear, ignore them or regenerate the docker image.
RUN wget -q -O /WSRT_Measures.ztar \
ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
cd /var/lib/casacore/data && \
tar xfz /WSRT_Measures.ztar && \
rm /WSRT_Measures.ztar
# Try to run the compiled tools to make sure they run without
# a problem (e.g. no missing libraries).
RUN aoflagger --version && \
DP3 --version && \
wsclean --version
ARG BASE_TAG=latest
FROM $BASE_TAG
ENV PREFACTOR_DATA_ROOT=/usr/local/share/prefactor \
PYTHONPATH=/usr/local/bin
# Install Prefactor CWL
COPY . /tmp/prefactor
RUN python3 -m pip install --no-cache-dir --upgrade /tmp/prefactor && \
rm -rf /tmp/prefactor
# # A user 'lofaruser' is added (this might be legacy -- not sure this is
# # still relevant).
# RUN groupadd -r lofaruser && \
# useradd -r -l -m -g lofaruser lofaruser
# WORKDIR /home/lofaruser
# USER lofaruser
FROM lofareosc/prefactor3-cwl
RUN groupadd -r lofaruser && \
useradd -r -l -m -g lofaruser lofaruser
RUN apt-get update && \
apt-get install -y \
python3-regex
RUN python3 -m pip install -Iv pyvo bdsf
#==0.9.3
RUN apt-get update && \
apt-get install -y \
git \
cmake \
python3-scipy \
python3-statsmodels \
python3-astroquery \
python3-magic && \
mkdir -p /src
# ensure that `python` and associated commands
# are available. Needed for facetselfcal.py
RUN cd "$(dirname $(which python3))" \
&& ln -s idle3 idle \
&& ln -s pydoc3 pydoc \
&& ln -s python3 python \
&& ln -s python3-config python-config
WORKDIR /home/lofaruser
USER lofaruser
#RUN git clone https://github.com/lonbar/VLBI-cwl.git /home/lofaruser/VLBI-cwl
ENV PYTHONPATH=/home/lofaruser/VLBI-cwl/scripts/:$PYTHONPATH \
PATH=/home/lofaruser/VLBI-cwl/scripts:$PATH
# Install VLBI CWL
COPY . /home/lofaruser/VLBI-cwl
#! /bin/bash
set -e
DOCKER_TAG=latest
docker build ${PWD} -f Docker/Dockerfile-vlbi -t vlbi-cwl:${DOCKER_TAG} #lofareosc/prefactor3-cwl:${DOCKER_TAG}
#docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}
...@@ -95,7 +95,7 @@ requirements: ...@@ -95,7 +95,7 @@ requirements:
ramMin: $(inputs.memory) ramMin: $(inputs.memory)
hints: hints:
DockerRequirement: - class: DockerRequirement
dockerPull: vlbi-cwl dockerPull: vlbi-cwl
stdout: aoflag.log stdout: aoflag.log
......
...@@ -68,7 +68,7 @@ outputs: ...@@ -68,7 +68,7 @@ outputs:
hints: hints:
- class: DockerRequirement - class: DockerRequirement
dockerPull: astronrd/linc dockerPull: vlbi-cwl
- class: InlineJavascriptRequirement - class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement - class: InitialWorkDirRequirement
listing: listing:
......
...@@ -85,8 +85,8 @@ requirements: ...@@ -85,8 +85,8 @@ requirements:
json.dump(cwl_output, fp) json.dump(cwl_output, fp)
hints: hints:
DockerRequirement: - class: DockerRequirement
dockerPull: vlbi-cwl:latest dockerPull: vlbi-cwl
stdout: compareStationMismatch.log stdout: compareStationMismatch.log
stderr: compareStationMismatch_err.log stderr: compareStationMismatch_err.log
...@@ -96,10 +96,6 @@ outputs: ...@@ -96,10 +96,6 @@ outputs:
The files containing the stdout The files containing the stdout
and stderr from the step. and stderr from the step.
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
requirements: requirements:
- class: InitialWorkDirRequirement - class: InitialWorkDirRequirement
listing: listing:
...@@ -108,5 +104,9 @@ requirements: ...@@ -108,5 +104,9 @@ requirements:
- class: InplaceUpdateRequirement - class: InplaceUpdateRequirement
inplaceUpdate: true inplaceUpdate: true
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
stdout: dp3_applycal.log stdout: dp3_applycal.log
stderr: dp3_applycal_err.log stderr: dp3_applycal_err.log
...@@ -121,7 +121,7 @@ requirements: ...@@ -121,7 +121,7 @@ requirements:
hints: hints:
- class: DockerRequirement - class: DockerRequirement
dockerPull: vlbi-cwl:latest dockerPull: vlbi-cwl
- class: InitialWorkDirRequirement - class: InitialWorkDirRequirement
listing: listing:
- entry: $(inputs.msin) - entry: $(inputs.msin)
......
...@@ -179,12 +179,12 @@ outputs: ...@@ -179,12 +179,12 @@ outputs:
glob: dp3_phaseup_err.log glob: dp3_phaseup_err.log
doc: The file containing the stderr from the step. doc: The file containing the stderr from the step.
hints:
DockerRequirement:
dockerPull: vlbi-cwl
requirements: requirements:
- class: InlineJavascriptRequirement - class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
stdout: dp3_phaseup.log stdout: dp3_phaseup.log
stderr: dp3_phaseup_err.log stderr: dp3_phaseup_err.log
...@@ -93,6 +93,10 @@ requirements: ...@@ -93,6 +93,10 @@ requirements:
- class: ResourceRequirement - class: ResourceRequirement
coresMin: 6 coresMin: 6
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
outputs: outputs:
- id: logfile - id: logfile
type: File[] type: File[]
...@@ -126,9 +130,5 @@ outputs: ...@@ -126,9 +130,5 @@ outputs:
doc: | doc: |
The flagging information of the output data as a parsed JSON string. The flagging information of the output data as a parsed JSON string.
hints:
DockerRequirement:
dockerPull: vlbi-cwl:latest
stdout: dp3_prep_target.log stdout: dp3_prep_target.log
stderr: dp3_prep_target_err.log stderr: dp3_prep_target_err.log
...@@ -59,8 +59,6 @@ outputs: ...@@ -59,8 +59,6 @@ outputs:
glob: dp3_target_phaseup_err.log glob: dp3_target_phaseup_err.log
doc: DP3 processing error log file. doc: DP3 processing error log file.
hints: hints:
- class: DockerRequirement - class: DockerRequirement
dockerPull: vlbi-cwl dockerPull: vlbi-cwl
......
...@@ -146,10 +146,10 @@ outputs: ...@@ -146,10 +146,10 @@ outputs:
and stderr from the step. and stderr from the step.
hints: hints:
- class: DockerRequirement
dockerPull: vlbi-cwl:latest
- class: ResourceRequirement - class: ResourceRequirement
coresMin: 6 coresMin: 6
- class: DockerRequirement
dockerPull: vlbi-cwl
stdout: predict_ateam.log stdout: predict_ateam.log
stderr: predict_ateam_err.log stderr: predict_ateam_err.log
...@@ -42,6 +42,10 @@ requirements: ...@@ -42,6 +42,10 @@ requirements:
with open('./out.json', 'w') as fp: with open('./out.json', 'w') as fp:
json.dump(output, fp) json.dump(output, fp)
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
outputs: outputs:
- id: source_id - id: source_id
type: string type: string
...@@ -67,9 +71,5 @@ outputs: ...@@ -67,9 +71,5 @@ outputs:
The files containing the stdout The files containing the stdout
and stderr outputs from the step. and stderr outputs from the step.
hints:
DockerRequirement:
dockerPull: vlbi-cwl
stdout: prep_delay.log stdout: prep_delay.log
stderr: prep_delay_err.log stderr: prep_delay_err.log
...@@ -104,6 +104,10 @@ requirements: ...@@ -104,6 +104,10 @@ requirements:
with open('./out.json', 'w') as fp: with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp) json.dump(cwl_output, fp)
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
outputs: outputs:
- id: filenames - id: filenames
type: File type: File
...@@ -127,9 +131,5 @@ outputs: ...@@ -127,9 +131,5 @@ outputs:
glob: sort_concatmap.log glob: sort_concatmap.log
doc: The file containing the stdout output from the step. doc: The file containing the stdout output from the step.
hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
stdout: sort_concatmap.log stdout: sort_concatmap.log
stderr: sort_concatmap_err.log stderr: sort_concatmap_err.log
...@@ -164,7 +164,7 @@ outputs: ...@@ -164,7 +164,7 @@ outputs:
hints: hints:
- class: DockerRequirement - class: DockerRequirement
dockerPull: vlbi-cwl:latest dockerPull: vlbi-cwl
stdout: summary.log stdout: summary.log
stderr: summary_err.log stderr: summary_err.log
...@@ -63,10 +63,10 @@ requirements: ...@@ -63,10 +63,10 @@ requirements:
subprocess.run(f'python3 {selfcal}/facetselfcal.py {msin} --helperscriptspath {selfcal} --helperscriptspathh5merge {h5merge} --weightspectrum-clipvalue 30.0 --auto --imagename {imagename}', shell = True) subprocess.run(f'python3 {selfcal}/facetselfcal.py {msin} --helperscriptspath {selfcal} --helperscriptspathh5merge {h5merge} --weightspectrum-clipvalue 30.0 --auto --imagename {imagename}', shell = True)
hints: hints:
- class: DockerRequirement
dockerPull: vlbi-cwl
- class: ResourceRequirement - class: ResourceRequirement
coresMin: 6 coresMin: 6
- class: DockerRequirement
dockerPull: vlbi-cwl
stdout: target_solve.log stdout: target_solve.log
stderr: target_solve_err.log stderr: target_solve_err.log
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment