diff --git a/.gitattributes b/.gitattributes
index e3fa62677708f0dce0d86c373df1b7919158aad1..3003752d3fbb1515990e9bf043f218938091b15f 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1613,6 +1613,7 @@ CEP/Pipeline/recipes/sip/nodes/setupparmdb.py eol=lf
CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py eol=lf
CEP/Pipeline/recipes/sip/nodes/vdsmaker.py eol=lf
CEP/Pipeline/recipes/sip/pipeline.cfg.in eol=lf
+CEP/Pipeline/recipes/sip/pipeline.cfg.thead01.cep4 -text
CEP/Pipeline/recipes/sip/plugins/PipelineStep_addMapfile.py -text
CEP/Pipeline/recipes/sip/plugins/PipelineStep_changeMapfile.py -text
CEP/Pipeline/recipes/sip/plugins/PipelineStep_combineParsets.py -text
@@ -2344,6 +2345,13 @@ CMake/variants/variants.node503 -text
CMake/variants/variants.node521 -text
CMake/variants/variants.phi -text
CMake/variants/variants.sharkbay -text
+Docker/docker-build-all.sh -text
+Docker/lofar-base/Dockerfile.tmpl -text
+Docker/lofar-base/bashrc -text
+Docker/lofar-base/chuser.sh -text
+Docker/lofar-base/subversion_servers -text
+Docker/lofar-outputproc/Dockerfile.tmpl -text
+Docker/lofar-pipeline/Dockerfile.tmpl -text
JAVA/GUI/Plotter/dist/lib/sgt.jar -text svneol=unset#unset
JAVA/GUI/Plotter/doc/Plotter.EAP -text
JAVA/GUI/Plotter/doc/javadoc/resources/inherit.gif -text
diff --git a/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py b/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py
index 9bb498e5514559e110c153312b27aaef97677843..82950986177ebb0cf1e16735b99f0861fa2f5ad0 100644
--- a/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py
+++ b/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py
@@ -78,6 +78,8 @@ def run_remote_command(config, logger, host, command, env, arguments = None):
except:
method = None
+ logger.info("********************** Remote method is %s" % method)
+
if method == "paramiko":
try:
key_filename = config.get('remote', 'key_filename')
@@ -94,13 +96,17 @@ def run_remote_command(config, logger, host, command, env, arguments = None):
return run_via_mpiexec_cep(logger, command, arguments, host)
elif method == "slurm_srun_cep3":
return run_via_slurm_srun_cep3(logger, command, arguments, host)
+ elif method == "custom_cmdline":
+ return run_via_custom_cmdline(logger, host, command, env, arguments, config)
else:
return run_via_ssh(logger, host, command, env, arguments)
def run_via_slurm_srun_cep3(logger, command, arguments, host):
+ logger.debug("Dispatching command to %s with srun" % host)
for arg in arguments:
command = command + " " + str(arg)
commandstring = ["srun","-N 1","-n 1","-w",host, "/bin/sh", "-c", "hostname && " + command]
+ #commandstring = ["srun","-N 1","--cpu_bind=map_cpu:none","-w",host, "/bin/sh", "-c", "hostname && " + command]
# we have a bug that crashes jobs when too many get startet at the same time
# temporary NOT 100% reliable workaround
#from random import randint
@@ -179,6 +185,53 @@ def run_via_ssh(logger, host, command, environment, arguments):
process.kill = lambda : os.kill(process.pid, signal.SIGKILL)
return process
+def run_via_custom_cmdline(logger, host, command, environment, arguments, config):
+ """
+ Dispatch a remote command via a customisable command line
+
+ We return a Popen object pointing at the running executable, to which we add a
+ kill method for shutting down the connection if required.
+
+ The command line is taken from the "remote.cmdline" configuration option,
+ with the following strings replaced:
+
+ {host} := host to execute command on
+ {command} := bash command line to be executed
+ {uid} := uid of the calling user
+
+ {image} := docker.image configuration option
+ {slurm_job_id} := the SLURM job id to allocate resources in
+
+ """
+ commandArray = ["%s=%s" % (key, value) for key, value in environment.items()]
+ commandArray.append(command)
+ commandArray.extend(re.escape(str(arg)) for arg in arguments)
+ commandStr = " ".join(commandArray)
+
+ try:
+ image = config.get('docker', 'image')
+ except:
+ image = "lofar"
+
+ # Construct the full command line, except for {command}, as that itself
+ # can contain spaces which we don't want to split on.
+ full_command_line = config.get('remote', 'cmdline').format(
+ uid = os.geteuid(),
+ slurm_job_id = os.environ.get("SLURM_JOB_ID"),
+ docker_image = image,
+ host = host,
+ command = "{command}"
+ ).split(' ')
+
+ # Fill in {command} somewhere
+ full_command_line = [x.format(command = commandStr) for x in full_command_line]
+
+ logger.debug("Dispatching command to %s with custom_cmdline: %s" % (host, full_command_line))
+
+ process = spawn_process(full_command_line, logger)
+ process.kill = lambda : os.kill(process.pid, signal.SIGKILL)
+ return process
+
def run_via_paramiko(logger, host, command, environment, arguments, key_filename):
"""
Dispatch a remote command via paramiko.
@@ -270,8 +323,7 @@ class ComputeJob(object):
"PYTHONPATH": os.environ.get('PYTHONPATH'),
"LD_LIBRARY_PATH": os.environ.get('LD_LIBRARY_PATH'),
"LOFARROOT" : os.environ.get('LOFARROOT'),
- "QUEUE_PREFIX" : os.environ.get('QUEUE_PREFIX',''),
- "LOFAR_OBSID" : os.environ.get('LOFAR_OBSID','')
+ "QUEUE_PREFIX" : os.environ.get('QUEUE_PREFIX','')
},
arguments = [id, jobhost, jobport]
)
diff --git a/CEP/Pipeline/recipes/sip/pipeline.cfg.thead01.cep4 b/CEP/Pipeline/recipes/sip/pipeline.cfg.thead01.cep4
new file mode 100644
index 0000000000000000000000000000000000000000..62ad298207d88a02f6ed3c1c416a505127967774
--- /dev/null
+++ b/CEP/Pipeline/recipes/sip/pipeline.cfg.thead01.cep4
@@ -0,0 +1,89 @@
+# This pipeline.cfg was used on the CEP4 test system to show how Docker and SLURM
+# can be used to distribute and run jobs.
+#
+# Is called as follows (inside a SLURM job allocation, f.e. "salloc -N 2"). Replace directory names
+# where needed:
+#
+# # Create this on every node
+# CONFIGDIR=/globalhome/mol/regression_test
+# HOSTS=(`scontrol show hostnames $SLURM_JOB_NODELIST`)
+#
+# # $CONFIGDIR is local, and ocntains both the working_dir (for input/output data) and the config files
+# # /share is a shared disk for the vds files, map files, logs.
+#
+# docker run --rm -e SLURM_JOB_ID=${SLURM_JOB_ID} -e LUSER=${UID} -v $HOME/.ssh:/home/lofar/.ssh:ro -v $CONFIGDIR:$CONFIGDIR -v /shared:/shared --net=host lofar-patched /bin/bash -c "\"$EXTRA_CMDS;python $CONFIGDIR/regression_test_runner.py preprocessing_pipeline --pipelinecfg $CONFIGDIR/pipeline.cfg --testdata $CONFIGDIR/data --computehost1 ${HOSTS[0]} --computehost2 ${HOSTS[1]} --workdir $CONFIGDIR/working_dir --rundir $CONFIGDIR/rundir\""
+
+
+[DEFAULT]
+lofarroot = /opt/lofar
+casaroot = /opt/casacore
+pyraproot = /opt/python-casacore/pyrap
+hdf5root =
+wcsroot = /opt/wcslib
+pythonpath = /opt/lofar/lib/python2.7/site-packages
+# runtime dir is a global FS (nfs, lustre) to exchange small files (parsets, vds, map files, etc)
+runtime_directory = /shared/mol/regression_test/rundir
+recipe_directories = [%(pythonpath)s/lofarpipe/recipes]
+# working dir is the local dir in which input/output dataproducts reside
+working_directory = /globalhome/mol/regression_test/working_dir
+task_files = [%(lofarroot)s/share/pipeline/tasks.cfg]
+
+[layout]
+job_directory = %(runtime_directory)s/%(job_name)s
+
+[cluster]
+clusterdesc = %(lofarroot)s/share/local.clusterdesc
+
+[deploy]
+engine_ppath = %(pythonpath)s:%(pyraproot)s/lib:/opt/cep/pythonlibs/lib/python/site-packages
+engine_lpath = %(lofarroot)s/lib:%(casaroot)s/lib:%(pyraproot)s/lib:%(hdf5root)s/lib:%(wcsroot)s/lib
+
+[logging]
+log_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/pipeline.log
+xml_stat_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/statistics.xml
+
+[feedback]
+# Method of providing feedback to LOFAR.
+# Valid options:
+# messagebus Send feedback and status using LCS/MessageBus
+# none Do NOT send feedback and status
+method = none
+
+[docker]
+image = lofar-patched
+
+[remote]
+#method = slurm_srun_cep3
+#method = ssh_docker
+method = custom_cmdline
+max_per_node = 1
+
+# We take the following path to start a remote container:
+#
+# [container] --SSH--> [host] --SRUN--> [worker node] --DOCKER--> [container]
+#
+# This path is needed because running SRUN from within the container needs a lot of cluster-specific infra
+# (SLURM config files, Munge keys, correct Munge user ID, munged).
+#
+# Throughout this path, we maintain:
+# * userid, which is set to the user that started the master script container (-e LUSER=`id -u`)
+# * environment (sudo -p, docker -e K=V, etc)
+# * pseudo-tty to prevent buffering logs (ssh -tt, docker -t)
+
+#
+# host -> worker node: srun -w {host} -N 1 -n 1 --jobid={slurm_job_id}
+# Needs the specific host, because the test system does not have a global file system.
+#
+# worker node -> container: docker run -t --rm -e LUSER={uid} -w g -v /home/mol/.ssh:/home/lofar/.ssh:ro -v /globalhome/mol/regression_test:/globalhome/mol/regression_test -v /shared:/shared --net=host {docker_image}
+# Starts the container on the worker node, with pretty much the same parameters as the master container:
+#
+# --rm: don't linger resources when the container exits
+# -e LUSER=(uid): set the user to run as (the calling user)
+# -h {host}: set container hostname (for easier debugging) (doesnt work yet, using --net=host instead)
+# -v: map the directories for input/output data, shared storage (parsets, vds files, etc)
+#
+# /bin/bash -c
+#
+# Required because the pipeline framework needs some bash functionality in the commands it starts.
+#cmdline = ssh -n -tt -x localhost srun -w {host} -N 1 -n 1 --jobid={slurm_job_id} docker run -t --rm -e LUSER={uid} -w g -v /home/mol/.ssh:/home/lofar/.ssh:ro -v /globalhome/mol/regression_test:/globalhome/mol/regression_test -v /shared:/shared --net=host {docker_image} /bin/bash -c
+cmdline = ssh -n -tt -x localhost srun -w {host} -N 1 -n 1 --jobid={slurm_job_id} docker run -t --rm -e LUSER={uid} -v %(runtime_directory)s:%(runtime_directory)s -v %(working_directory)s:%(working_directory)s --net=host {docker_image} /bin/bash -c "\"{command}\""
diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index c1990c2763c8a3e6ce317952a67fff78c7aae35d..1e8ed2516bee6c38dc4c7c2eb12901c084f97c5a 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -16,6 +16,7 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(LOFAR_PACKAGE_LIST_INCLUDED TRUE)
set(Calibration_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration)
set(DP3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3)
+ set(Docker_SOURCE_DIR ${CMAKE_SOURCE_DIR}/Docker)
set(GSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/GSM)
set(Imager_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Imager)
set(LMWCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/LMWCommon)
diff --git a/CMake/UpdatePackageVersion.cmake.in b/CMake/UpdatePackageVersion.cmake.in
index 6732c27242cc2db0c4d64eb43aca04dc8ae306e7..84f72ac731cefeec683bb723e84d0824dae066a1 100644
--- a/CMake/UpdatePackageVersion.cmake.in
+++ b/CMake/UpdatePackageVersion.cmake.in
@@ -77,7 +77,11 @@ if(_result EQUAL 0)
endif(_root MATCHES ".+")
if(_rvers MATCHES "^/trunk/")
set(_rvers "trunk")
+ set(_branch "trunk")
elseif(_rvers MATCHES "^/tags/" OR _rvers MATCHES "^/branches/")
+ # branch = directory right under tags/ or branches/
+ string(REGEX REPLACE "^/([^/]*/[^/]*)/.*$" "\\1" _branch "${_rvers}")
+
# We have a task branch (for a bug) or a release branch.
# Remove all but version and replace _ in version by .
string(REGEX REPLACE "^/[^/]*/([^/]*)/.*$" "\\1" _rvers "${_rvers}")
@@ -121,6 +125,8 @@ file(WRITE "${_file}.tmp"
" // Get the info for this package.\n"
" static Version getInfo();\n"
"\n"
+ " // Get the branch that was checked out (relative to repository root).\n"
+ " static std::string getBranch();\n"
" // Get the version number of the package (as given in repository).\n"
" static std::string getVersion();\n"
" // Get the svn version number of the package according to CMakeLists.txt.\n"
@@ -168,12 +174,15 @@ file(APPEND "${_file}.tmp"
" Version ${_pkg}Version::getInfo()\n"
" {\n"
" return Version (\"@PACKAGE_NAME@\",\n"
+ " getBranch(),\n"
" getVersion(), getConfVersion(),\n"
" getRevision(), getPackageRevision(),\n"
" getNrChangedFiles(),\n"
" getBuildTime(), getBuildUser(), getBuildMachine());\n"
" }\n"
"\n"
+ " std::string ${_pkg}Version::getBranch()\n"
+ " { return \"${_branch}\"; }\n"
" std::string ${_pkg}Version::getVersion()\n"
" { return \"${_rvers}\"; }\n"
" std::string ${_pkg}Version::getConfVersion()\n"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ccb788fe33c4bb7b2458a73210da07d583a68bda..ea914605bb0e6cfd80c3f8dcb4117ad3c8697aa1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -18,6 +18,7 @@ include(LofarGeneral)
## ---------------------------------------------------------------------------
include(LofarPackage)
if(NOT DEFINED BUILD_PACKAGES)
+ lofar_add_package(Docker)
lofar_add_package(LCS)
lofar_add_package(CEP)
lofar_add_package(RTCP)
diff --git a/Docker/CMakeLists.txt b/Docker/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..060eea1d0c446ffeb5ac0433df0432c95964c845
--- /dev/null
+++ b/Docker/CMakeLists.txt
@@ -0,0 +1,65 @@
+# $Id$
+
+lofar_package(Docker 1.0 DEPENDS Common)
+
+include(LofarPackageVersion)
+
+# Build version info
+set(docker_LIB_SRCS
+ Package__Version.cc
+)
+
+lofar_add_library(docker ${docker_LIB_SRCS})
+
+lofar_add_bin_program(versiondocker versiondocker.cc)
+
+#
+# For given directories, we generate the Dockerfile
+# by parsing their Dockerfile.tmpl through "docker-template".
+#
+# "docker-template" is a script that fills in variables with
+# respect to the build info (branch name, build time, etc)
+#
+
+# Directories with Dockerfile.tmpl to parse
+set(DOCKER_TEMPLATE_DIRS
+ lofar-base
+ lofar-pipeline
+ lofar-outputproc)
+
+# Note: "docker-template" only works as long as the sources are still around,
+# since it uses svn to query information from them.
+lofar_add_bin_scripts(docker-template)
+lofar_add_sbin_scripts(docker-build-all.sh)
+
+# Convert Dockerfile.tmpl -> Dockerfile in ${DOCKER_TEMPLATE_DIRS}
+foreach(_dir ${DOCKER_TEMPLATE_DIRS})
+ # _src -> _dst
+ set(_src ${CMAKE_CURRENT_SOURCE_DIR}/${_dir}/Dockerfile.tmpl)
+ set(_dst ${CMAKE_CURRENT_BINARY_DIR}/${_dir}_Dockerfile)
+
+ # add generating command, and (any) target to force the generation
+ # when "all" is build.
+ add_custom_command(
+ OUTPUT ${_dst}
+ COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/docker-template < ${_src} > ${_dst}
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/docker-template ${_src} ${CMAKE_CURRENT_BINARY_DIR}/versiondocker
+ )
+ add_custom_target(${_dir}_Dockerfile_target ALL DEPENDS ${_dst})
+
+ # install resulting Dockerfile
+ install(FILES
+ ${_dst}
+ DESTINATION share/docker/${_dir}
+ RENAME Dockerfile
+ )
+endforeach()
+
+# Install everything else
+install(DIRECTORY
+ lofar-base
+ lofar-pipeline
+ lofar-outputproc
+ DESTINATION share/docker
+ USE_SOURCE_PERMISSIONS
+ PATTERN Dockerfile.tmpl EXCLUDE)
diff --git a/Docker/docker-build-all.sh b/Docker/docker-build-all.sh
new file mode 100755
index 0000000000000000000000000000000000000000..71eb409a42c9e125646f6f19af6d99e96c176c21
--- /dev/null
+++ b/Docker/docker-build-all.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+TAG=`echo '${LOFAR_TAG}' | docker-template`
+
+function build {
+ IMAGE=$1
+ docker build -t ${IMAGE}:${TAG} ${IMAGE}
+}
+
+cd ${LOFARROOT}/share/docker
+
+build lofar-base && \
+build lofar-pipeline && \
+build lofar-outputproc
+
diff --git a/Docker/docker-template b/Docker/docker-template
new file mode 100755
index 0000000000000000000000000000000000000000..8cac4023c1e08281bdd89b8e23629812c5b084b6
--- /dev/null
+++ b/Docker/docker-template
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Template engine for LOFAR
+#
+# Reads stdin, replaces variables, writes stdout.
+#
+# Variables replaced:
+# ${LOFAR_BRANCH_NAME} = Name of this branch, relative to root
+# ${LOFAR_BRANCH_URL} = Full subversion URL for this branch
+# ${LOFAR_TAG} = Tag for a Docker image
+# ${LOFAR_REVISION} = SVN revision number of this checkout (or HEAD if trunk)
+# ${NOW} = now in UTC (format: 2016-01-01 10:11:12)
+# ${BUILD_UID} = uid of building user (=caller)
+#
+
+# ----- LOFAR_BRANCH_NAME = branches/LOFAR-Task1234 -----
+# ----- LOFAR_BRANCH_NAME = trunk -----
+# ----- LOFAR_BRANCH_NAME = tags/LOFAR-Release-2_15_1 -----
+# ----- LOFAR_BRANCH_NAME = UNKNOWN -----
+
+# Make sure we obtain info about the project source!
+#PATH=$PATH:.
+VERSION_INFO=`(versiondocker || ./versiondocker) 2>/dev/null` # in cmake, executable is in .
+
+# Extract branch name w.r.t. repository root, e.g. branches/LOFAR-Task1234
+export LOFAR_BRANCH_NAME=`echo "$VERSION_INFO" | perl -ne 'print "$1" if /branch += +(.+)/;'`
+
+# ----- LOFAR_BRANCH_URL = https://svn.astron.nl/LOFAR/branches/LOFAR-Task1234 -----
+# ----- LOFAR_BRANCH_URL = https://svn.astron.nl/LOFAR/trunk -----
+# ----- LOFAR_BRANCH_URL = https://svn.astron.nl/LOFAR/tags/LOFAR-Release-2_15_1 -----
+
+export LOFAR_BRANCH_URL="https://svn.astron.nl/LOFAR/${LOFAR_BRANCH_NAME}"
+
+# ----- LOFAR_TAG = 1234 -----
+# ----- LOFAR_TAG = trunk -----
+# ----- LOFAR_TAG = 2_15_1 -----
+
+case "${LOFAR_BRANCH_NAME}" in
+ trunk) export LOFAR_TAG=trunk ;;
+ tags/*) export LOFAR_TAG=${LOFAR_BRANCH_NAME##tags/LOFAR-Release-} ;;
+ branches/*) export LOFAR_TAG=${LOFAR_BRANCH_NAME##branches/LOFAR*Task} ;;
+ *) export LOFAR_TAG=latest ;;
+esac
+
+# ----- LOFAR_REVISION = 12345 -----
+
+export LOFAR_REVISION=`echo "$VERSION_INFO" | perl -ne 'print "$1" if /overall revision += +([0-9]+)/;'`
+
+# ----- NOW = 2016-01-01 10:11:12 -----
+
+export NOW="`date -u +'%F %T'`"
+
+# ----- BUILD_UID = 1001 ----
+
+export BUILD_UID=`id -u`
+
+# ----- Process input -----
+
+# Insert our knowledge when processing stdin -> stdout
+envsubst '$LOFAR_BRANCH_NAME $LOFAR_BRANCH_URL $LOFAR_TAG $LOFAR_REVISION $NOW $BUILD_UID'
+
diff --git a/Docker/lofar-base/Dockerfile.tmpl b/Docker/lofar-base/Dockerfile.tmpl
new file mode 100644
index 0000000000000000000000000000000000000000..42d5fb3913e4db10f7c6e6aa521e0c56947f1985
--- /dev/null
+++ b/Docker/lofar-base/Dockerfile.tmpl
@@ -0,0 +1,167 @@
+#
+# base
+#
+FROM ubuntu:14.04
+
+#
+# common-environment
+#
+ENV USER=lofar
+ENV INSTALLDIR=/opt
+
+#
+# environment
+#
+ENV DEBIAN_FRONTEND=noninteractive \
+ PYTHON_VERSION=2.7
+
+#
+# versions
+#
+ENV CASACORE_VERSION=2.0.3 \
+ CASAREST_VERSION=1.4.1 \
+ PYTHON_CASACORE_VERSION=2.0.1 \
+ BOOST_VERSION=1.54
+
+#
+# set-uid
+#
+ENV UID=${BUILD_UID}
+
+#
+# set-build-options
+#
+ENV J=4
+
+#
+# Base and runtime dependencies
+#
+#RUN sed -i 's/archive.ubuntu.com/osmirror.rug.nl/' /etc/apt/sources.list
+RUN apt-get update && apt-get upgrade -y
+RUN apt-get install -y sudo python2.7 libpython2.7 && \
+ apt-get install -y libblas3 liblapacke python-numpy libcfitsio3 libwcs4 libfftw3-bin libhdf5-7 libboost-python${BOOST_VERSION}.0
+
+#
+# setup-account
+#
+RUN (getent group sudo &>/dev/null || groupadd sudo) && \
+ echo "useradd -m ${USERADD_FLAGS} ${USER}" && \
+ useradd -m -u ${UID} ${USER} && \
+ usermod -a -G sudo ${USER} && \
+ echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+ sed -i 's/requiretty/!requiretty/g' /etc/sudoers
+
+#
+# setup install dir
+#
+RUN mkdir -p ${INSTALLDIR} && chown ${USER}.${USER} ${INSTALLDIR}
+
+USER ${USER}
+
+#
+# *******************
+# Casacore
+# *******************
+#
+RUN sudo apt-get install -y wget git cmake g++ gfortran flex bison libblas-dev liblapacke-dev libfftw3-dev libhdf5-dev libboost-python-dev libcfitsio3-dev wcslib-dev && \
+ mkdir -p ${INSTALLDIR}/casacore/build && \
+ mkdir -p ${INSTALLDIR}/casacore/data && \
+ cd ${INSTALLDIR}/casacore && git clone https://github.com/casacore/casacore.git src && \
+ if [ "${CASACORE_VERSION}" != "latest" ]; then cd ${INSTALLDIR}/casacore/src && git checkout tags/v${CASACORE_VERSION}; fi && \
+ cd ${INSTALLDIR}/casacore/data && wget --retry-connrefused ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
+ cd ${INSTALLDIR}/casacore/data && tar xf WSRT_Measures.ztar && rm -f WSRT_Measures.ztar && \
+ cd ${INSTALLDIR}/casacore/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/casacore/ -DDATA_DIR=${INSTALLDIR}/casacore/data -DBUILD_PYTHON=True -DUSE_OPENMP=True -DUSE_FFTW3=TRUE ../src/ && \
+ cd ${INSTALLDIR}/casacore/build && make -j ${J} && \
+ cd ${INSTALLDIR}/casacore/build && make install && \
+ bash -c "strip ${INSTALLDIR}/casacore/{lib,bin}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/casacore/{build,src}" && \
+ sudo apt-get purge -y wget git cmake g++ gfortran flex bison libblas-dev liblapacke-dev libfftw3-dev libhdf5-dev libboost-python-dev libcfitsio3-dev wcslib-dev && \
+ sudo apt-get autoremove -y
+
+#
+# *******************
+# Casarest
+# *******************
+#
+RUN sudo apt-get install -y git cmake g++ gfortran libboost-system-dev libboost-thread-dev libhdf5-dev libcfitsio3-dev wcslib-dev && \
+ mkdir -p ${INSTALLDIR}/casarest/build && \
+ cd ${INSTALLDIR}/casarest && git clone https://github.com/casacore/casarest.git src && \
+ if [ "${CASAREST_VERSION}" != "latest" ]; then cd ${INSTALLDIR}/casarest/src && git checkout tags/v${CASAREST_VERSION}; fi && \
+ cd ${INSTALLDIR}/casarest/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/casarest -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore ../src/ && \
+ cd ${INSTALLDIR}/casarest/build && make -j ${J} && \
+ cd ${INSTALLDIR}/casarest/build && make install && \
+ bash -c "strip ${INSTALLDIR}/casarest/{lib,bin}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/casarest/{build,src}" && \
+ sudo apt-get purge -y git cmake g++ gfortran libboost-system-dev libboost-thread-dev libhdf5-dev libcfitsio3-dev wcslib-dev && \
+ sudo apt-get autoremove -y
+
+#
+# *******************
+# Pyrap
+# *******************
+#
+RUN sudo apt-get install -y git make g++ python-setuptools libboost-python-dev libcfitsio3-dev wcslib-dev && \
+ mkdir ${INSTALLDIR}/python-casacore && \
+ cd ${INSTALLDIR}/python-casacore && git clone https://github.com/casacore/python-casacore && \
+ if [ "$PYTHON_CASACORE_VERSION" != "latest" ]; then cd ${INSTALLDIR}/python-casacore/python-casacore && git checkout tags/v${PYTHON_CASACORE_VERSION}; fi && \
+ cd ${INSTALLDIR}/python-casacore/python-casacore && ./setup.py build_ext -I${INSTALLDIR}/casacore/include/ -L${INSTALLDIR}/casacore/lib/ && \
+ mkdir -p ${INSTALLDIR}/python-casacore/lib/python${PYTHON_VERSION}/site-packages/ && \
+ mkdir -p ${INSTALLDIR}/python-casacore/lib64/python${PYTHON_VERSION}/site-packages/ && \
+ export PYTHONPATH=${INSTALLDIR}/python-casacore/lib/python${PYTHON_VERSION}/site-packages:${INSTALLDIR}/python-casacore/lib64/python${PYTHON_VERSION}/site-packages:$PYTHONPATH && cd ${INSTALLDIR}/python-casacore/python-casacore && ./setup.py install --prefix=${INSTALLDIR}/python-casacore/ && \
+ bash -c "find ${INSTALLDIR}/python-casacore/lib -name '*.so' | xargs strip || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/python-casacore/python-casacore" && \
+ sudo apt-get purge -y git make g++ python-setuptools libboost-python-dev libcfitsio3-dev wcslib-dev && \
+ sudo apt-get autoremove -y
+
+#
+# *******************
+# QPID client
+# *******************
+#
+
+# Allow auto-checkout from lofar repo
+RUN mkdir /home/${USER}/.subversion
+COPY subversion_servers /home/${USER}/.subversion/servers
+
+# Run-time dependencies
+# QPID daemon legacy store would require: libaio1 libdb5.1++
+RUN sudo apt-get install -y sasl2-bin libuuid1 libnss3 libnspr4 xqilla libboost-program-options${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0
+
+# Install
+# QPID daemon legacy store would require: libaio-dev libdb5.1++-dev
+RUN sudo apt-get install -y subversion swig ruby ruby-dev python-dev libsasl2-dev pkg-config cmake libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential debhelper libsslcommon2-dev libxqilla-dev python-setuptools libboost-program-options${BOOST_VERSION}-dev libboost-filesystem${BOOST_VERSION}-dev && \
+ mkdir /opt/qpid && \
+ svn --non-interactive -q --username lofar-guest --password lofar-guest co ${LOFAR_BRANCH_URL}/LCS/MessageBus/qpid/ /opt/qpid; \
+ /opt/qpid/local/sbin/build_qpid && \
+ bash -c "strip /opt/qpid/{bin,lib}/* || true" && \
+ bash -c "rm -rf ~/sources" && \
+ sudo apt-get purge -y subversion swig ruby ruby-dev python-dev libsasl2-dev pkg-config cmake libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential debhelper libsslcommon2-dev libxqilla-dev python-setuptools libboost-program-options${BOOST_VERSION}-dev libboost-filesystem${BOOST_VERSION}-dev && \
+ sudo apt-get autoremove -y
+#
+# *******************
+# DAL
+# *******************
+#
+RUN sudo apt-get install -y git cmake g++ swig python-dev libhdf5-dev && \
+ mkdir ${INSTALLDIR}/DAL && \
+ cd ${INSTALLDIR}/DAL && git clone https://github.com/nextgen-astrodata/DAL.git src && \
+ mkdir ${INSTALLDIR}/DAL/build && cd ${INSTALLDIR}/DAL/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/DAL ../src && \
+ make -j ${J} && \
+ make install && \
+ bash -c "find ${INSTALLDIR}/DAL/lib -name '*.so' | xargs strip || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/DAL/{src,build}" && \
+ sudo apt-get purge -y git cmake g++ swig python-dev libhdf5-dev && \
+ sudo apt-get autoremove -y
+
+#
+# config
+#
+COPY bashrc /home/${USER}/.bashrc
+
+#
+# entry
+#
+COPY chuser.sh /usr/local/bin/chuser.sh
+WORKDIR /home/${USER}
+ENTRYPOINT ["sudo","-E","/usr/local/bin/chuser.sh"]
+
diff --git a/Docker/lofar-base/bashrc b/Docker/lofar-base/bashrc
new file mode 100644
index 0000000000000000000000000000000000000000..74dbbeef1254fd1c50e3454a8b19b441379788d1
--- /dev/null
+++ b/Docker/lofar-base/bashrc
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# lofar
+[ -r ${INSTALLDIR}/lofar/lofarinit.sh ] && source ${INSTALLDIR}/lofar/lofarinit.sh
+
+# qpid
+source ${INSTALLDIR}/qpid/.profile
+
+# python-casacore
+export PYTHONPATH=${PYTHONPATH}:${INSTALLDIR}/python-casacore/lib/python2.7/site-packages
+
+# casacore
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${INSTALLDIR}/casacore/lib
diff --git a/Docker/lofar-base/chuser.sh b/Docker/lofar-base/chuser.sh
new file mode 100755
index 0000000000000000000000000000000000000000..294e58c7d30e9288aa263d632c738a8ab06c0b98
--- /dev/null
+++ b/Docker/lofar-base/chuser.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Fetch the user name used in this container
+export USER=${SUDO_USER}
+
+if [ -n "${LUSER}" ]; then
+ if [ -z "${LGROUP}" ]; then
+ LGROUP=${LUSER}
+ fi
+
+ OLDID=`id -u ${USER}`
+
+ # Replace USER by LUSER:LGROUP
+ sed -i -e "s/${USER}:x:[0-9]\+:[0-9]\+/${USER}:x:${LUSER}:${LGROUP}/g" /etc/passwd
+ sed -i -e "s/${USER}:x:[0-9]\+:/${USER}:x:${LGROUP}:/g" /etc/group
+
+ # Set ownership of home dir to new user
+ chown --from=${OLDID} -R ${LUSER}:${LGROUP} ${HOME}
+fi
+
+# Switch to the updated user
+export HOME=/home/${USER}
+touch -a $HOME/.bashrc
+sudo -u ${USER} -E -s /bin/bash -c "source $HOME/.bashrc;$*"
diff --git a/Docker/lofar-base/subversion_servers b/Docker/lofar-base/subversion_servers
new file mode 100644
index 0000000000000000000000000000000000000000..986b312078ec2bdd8474d33f6dce788756d3f609
--- /dev/null
+++ b/Docker/lofar-base/subversion_servers
@@ -0,0 +1,6 @@
+[groups]
+astron = svn.astron.nl
+
+[astron]
+store-passwords = yes
+store-plaintext-passwords = yes
diff --git a/Docker/lofar-outputproc/Dockerfile.tmpl b/Docker/lofar-outputproc/Dockerfile.tmpl
new file mode 100644
index 0000000000000000000000000000000000000000..85474bfcb585b82dc4018a994dfb2322d63263ed
--- /dev/null
+++ b/Docker/lofar-outputproc/Dockerfile.tmpl
@@ -0,0 +1,33 @@
+#
+# base
+#
+FROM lofar-base:${LOFAR_TAG}
+
+#
+# *******************
+# LOFAR
+# *******************
+#
+
+# Run-time dependencies
+RUN sudo apt-get install -y liblog4cplus-1.0-4 libxml2 libboost-thread${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0 libboost-date-time${BOOST_VERSION}.0 libpng12-0 libsigc++-2.0-dev libxml++2.6-2 libboost-regex${BOOST_VERSION}.0
+
+# Tell image build information
+ENV LOFAR_BRANCH=${LOFAR_BRANCH_NAME} \
+ LOFAR_REVISION=${LOFAR_REVISION}
+
+# Install
+RUN sudo apt-get install -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libboost-regex${BOOST_VERSION} binutils-dev && \
+ mkdir -p ${INSTALLDIR}/lofar/build/gnu_opt libcfitsio3-dev wcslib-dev && \
+ cd ${INSTALLDIR}/lofar && \
+ svn --non-interactive -q --username lofar-guest --password lofar-guest co -r ${LOFAR_REVISION} -N ${LOFAR_BRANCH_URL} src; \
+ svn --non-interactive -q up src/CMake && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && cmake -DBUILD_PACKAGES=Online_OutputProc -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/lofar/ -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DLOG4CPLUS_ROOT_DIR=${INSTALLDIR}/log4cplus/ -DQPID_ROOT_DIR=/opt/qpid/ -DDAL_ROOT_DIR=${INSTALLDIR}/DAL -DUSE_OPENMP=True ${INSTALLDIR}/lofar/src/ && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && sed -i '29,31d' include/ApplCommon/PosixTime.h && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && make -j ${J} && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && make install && \
+ bash -c "strip ${INSTALLDIR}/lofar/{bin,sbin,lib64}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/lofar/{build,src}" && \
+ sudo apt-get purge -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev binutils-dev libcfitsio3-dev wcslib-dev && \
+ sudo apt-get autoremove -y
+
diff --git a/Docker/lofar-pipeline/Dockerfile.tmpl b/Docker/lofar-pipeline/Dockerfile.tmpl
new file mode 100644
index 0000000000000000000000000000000000000000..4443b94a15f41ffd3134d6279785e7aa0147983e
--- /dev/null
+++ b/Docker/lofar-pipeline/Dockerfile.tmpl
@@ -0,0 +1,58 @@
+#
+# base
+#
+FROM lofar-base:${LOFAR_TAG}
+
+ENV AOFLAGGER_VERSION=2.7.1
+
+# Run-time dependencies
+RUN sudo apt-get install -y python-xmlrunner python-scipy liblog4cplus-1.0-4 libxml2 libboost-thread${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0 libboost-date-time${BOOST_VERSION}.0 libboost-signals${BOOST_VERSION}.0 libpng12-0 libsigc++-2.0-dev libxml++2.6-2 libgsl0ldbl openssh-client libboost-regex${BOOST_VERSION}.0 && \
+ sudo apt-get -y install python-pip python-dev && \
+ sudo pip install pyfits pywcs python-monetdb && \
+ sudo apt-get -y purge python-pip python-dev && \
+ sudo apt-get -y autoremove
+
+#
+# *******************
+# AOFlagger
+# *******************
+#
+
+RUN sudo apt-get install -y wget cmake g++ libxml++2.6-dev libpng12-dev libfftw3-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-signals${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libcfitsio3-dev && \
+ mkdir -p ${INSTALLDIR}/aoflagger/build && \
+ bash -c "cd ${INSTALLDIR}/aoflagger && wget --retry-connrefused http://downloads.sourceforge.net/project/aoflagger/aoflagger-${AOFLAGGER_VERSION%%.?}.0/aoflagger-${AOFLAGGER_VERSION}.tar.bz2" && \
+ cd ${INSTALLDIR}/aoflagger && tar xf aoflagger-${AOFLAGGER_VERSION}.tar.bz2 && \
+ cd ${INSTALLDIR}/aoflagger/build && cmake -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/aoflagger ../aoflagger-${AOFLAGGER_VERSION} && \
+ cd ${INSTALLDIR}/aoflagger/build && make -j ${J} && \
+ cd ${INSTALLDIR}/aoflagger/build && make install && \
+ bash -c "strip ${INSTALLDIR}/aoflagger/{lib,bin}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/aoflagger/{build,aoflagger-${AOFLAGGER_VERSION}}" && \
+ bash -c "rm -rf ${INSTALLDIR}/aoflagger/aoflagger-${AOFLAGGER_VERSION}.tar.bz2" && \
+ sudo apt-get -y purge wget cmake g++ libxml++2.6-dev libpng12-dev libfftw3-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-signals${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libcfitsio3-dev && \
+ sudo apt-get -y autoremove
+
+#
+# *******************
+# LOFAR
+# *******************
+#
+
+# Tell image build information
+ENV LOFAR_BRANCH=${LOFAR_BRANCH_NAME} \
+ LOFAR_REVISION=${LOFAR_REVISION}
+
+# Install
+RUN sudo apt-get install -y subversion cmake g++ gfortran bison flex liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python-dev python-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libgsl0-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libboost-regex${BOOST_VERSION} binutils-dev libcfitsio3-dev wcslib-dev && \
+ mkdir -p ${INSTALLDIR}/lofar/build/gnu_opt && \
+ cd ${INSTALLDIR}/lofar && \
+ svn --non-interactive -q --username lofar-guest --password lofar-guest co -r ${LOFAR_REVISION} -N ${LOFAR_BRANCH_URL} src; \
+ svn --non-interactive -q up src/CMake && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && cmake -DBUILD_PACKAGES=Offline -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/lofar/ -DCASAREST_ROOT_DIR=${INSTALLDIR}/casarest/ -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DAOFLAGGER_ROOT_DIR=${INSTALLDIR}/aoflagger/ -DLOG4CPLUS_ROOT_DIR=${INSTALLDIR}/log4cplus/ -DQPID_ROOT_DIR=/opt/qpid/ -DUSE_OPENMP=True ${INSTALLDIR}/lofar/src/ && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && sed -i '29,31d' include/ApplCommon/PosixTime.h && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && make -j ${J} && \
+ cd ${INSTALLDIR}/lofar/build/gnu_opt && make install && \
+ bash -c "strip ${INSTALLDIR}/lofar/{bin,sbin,lib64}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/lofar/{build,src}" && \
+ sudo apt-get purge -y subversion cmake g++ gfortran bison flex liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python-dev python-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libgsl0-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev binutils-dev wcslib-dev && \
+ sudo apt-get autoremove -y
+
diff --git a/LCS/Common/include/Common/Version.h b/LCS/Common/include/Common/Version.h
index 3e9697c0c402d1f67411dbee067ff166c5edbf2b..3cd389e1ec4677505d62b882abd34c606a03813d 100644
--- a/LCS/Common/include/Common/Version.h
+++ b/LCS/Common/include/Common/Version.h
@@ -63,6 +63,7 @@ namespace LOFAR {
// Construct a Version object from the relevant package info.
Version (const std::string& packageName,
+ const std::string& branch,
const std::string& version,
const std::string& confVersion,
const std::string& revision,
@@ -75,6 +76,9 @@ namespace LOFAR {
// Get the package name.
const std::string& packageName() const
{ return itsPackageName; }
+ // Get the LOFAR branch version (relative to repository root)
+ const std::string& branch() const
+ { return itsBranch; }
// Get the package version (as in repository)
const std::string& version() const
{ return itsVersion; }
@@ -195,6 +199,7 @@ namespace LOFAR {
//# Data members.
std::string itsPackageName;
+ std::string itsBranch;
std::string itsVersion;
std::string itsConfVersion;
std::string itsRevision;
diff --git a/LCS/Common/src/Version.cc b/LCS/Common/src/Version.cc
index fd9dd09dc1995236f2df3c9668fcbab1abe10e37..a64c78bf84f081d6beb26800f9cf066bccddab82 100644
--- a/LCS/Common/src/Version.cc
+++ b/LCS/Common/src/Version.cc
@@ -29,6 +29,7 @@
namespace LOFAR {
Version::Version (const std::string& packageName,
+ const std::string& branch,
const std::string& version,
const std::string& confVersion,
const std::string& revision,
@@ -38,6 +39,7 @@ namespace LOFAR {
const std::string& buildUser,
const std::string& buildMachine)
: itsPackageName (packageName),
+ itsBranch (branch),
itsVersion (version),
itsConfVersion (confVersion),
itsRevision (revision),
@@ -129,6 +131,7 @@ namespace LOFAR {
os << " (in CMakeLists.txt: " << confVersion() << ')';
}
os << std::endl;
+ os << indent << " branch = " << branch() << std::endl;
os << indent << " overall revision = " << revision();
if (!sameRev) os << " (note: used packages have different revision)";
os << std::endl;