Task #9048: Added Docker package for lofar-base/lofar-outputproc/lofar-pipeline docker builds

.gitattributes

@@ -1613,6 +1613,7 @@ CEP/Pipeline/recipes/sip/nodes/setupparmdb.py eol=lf
 CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py eol=lf
 CEP/Pipeline/recipes/sip/nodes/vdsmaker.py eol=lf
 CEP/Pipeline/recipes/sip/pipeline.cfg.in eol=lf
+CEP/Pipeline/recipes/sip/pipeline.cfg.thead01.cep4 -text
 CEP/Pipeline/recipes/sip/plugins/PipelineStep_addMapfile.py -text
 CEP/Pipeline/recipes/sip/plugins/PipelineStep_changeMapfile.py -text
 CEP/Pipeline/recipes/sip/plugins/PipelineStep_combineParsets.py -text
@@ -2344,6 +2345,13 @@ CMake/variants/variants.node503 -text
 CMake/variants/variants.node521 -text
 CMake/variants/variants.phi -text
 CMake/variants/variants.sharkbay -text
+Docker/docker-build-all.sh -text
+Docker/lofar-base/Dockerfile.tmpl -text
+Docker/lofar-base/bashrc -text
+Docker/lofar-base/chuser.sh -text
+Docker/lofar-base/subversion_servers -text
+Docker/lofar-outputproc/Dockerfile.tmpl -text
+Docker/lofar-pipeline/Dockerfile.tmpl -text
 JAVA/GUI/Plotter/dist/lib/sgt.jar -text svneol=unset#unset
 JAVA/GUI/Plotter/doc/Plotter.EAP -text
 JAVA/GUI/Plotter/doc/javadoc/resources/inherit.gif -text

CEP/Pipeline/framework/lofarpipe/support/remotecommand.py

@@ -78,6 +78,8 @@ def run_remote_command(config, logger, host, command, env, arguments = None):
     except:
         method = None
 
+    logger.info("********************** Remote method is %s" % method)
+
     if method == "paramiko":
         try:
             key_filename = config.get('remote', 'key_filename')
@@ -94,13 +96,17 @@ def run_remote_command(config, logger, host, command, env, arguments = None):
         return run_via_mpiexec_cep(logger, command, arguments, host)
     elif method == "slurm_srun_cep3":
         return run_via_slurm_srun_cep3(logger, command, arguments, host)
+    elif method == "custom_cmdline":
+        return run_via_custom_cmdline(logger, host, command, env, arguments, config)
     else:
         return run_via_ssh(logger, host, command, env, arguments)
 
 def run_via_slurm_srun_cep3(logger, command, arguments, host):
+    logger.debug("Dispatching command to %s with srun" % host)
     for arg in arguments:
         command = command + " " + str(arg)
     commandstring = ["srun","-N 1","-n 1","-w",host, "/bin/sh", "-c", "hostname && " + command]
+    #commandstring = ["srun","-N 1","--cpu_bind=map_cpu:none","-w",host, "/bin/sh", "-c", "hostname && " + command]
     # we have a bug that crashes jobs when too many get startet at the same time
     # temporary NOT 100% reliable workaround
     #from random import randint
@@ -179,6 +185,53 @@ def run_via_ssh(logger, host, command, environment, arguments):
     process.kill = lambda : os.kill(process.pid, signal.SIGKILL)
     return process
 
+def run_via_custom_cmdline(logger, host, command, environment, arguments, config):
+    """
+    Dispatch a remote command via a customisable command line
+
+    We return a Popen object pointing at the running executable, to which we add a
+    kill method for shutting down the connection if required.
+
+    The command line is taken from the "remote.cmdline" configuration option,
+    with the following strings replaced:
+
+      {host}         := host to execute command on
+      {command}      := bash command line to be executed
+      {uid}          := uid of the calling user
+
+      {image}        := docker.image configuration option
+      {slurm_job_id} := the SLURM job id to allocate resources in
+
+    """
+    commandArray = ["%s=%s" % (key, value) for key, value in environment.items()]
+    commandArray.append(command)
+    commandArray.extend(re.escape(str(arg)) for arg in arguments)
+    commandStr = " ".join(commandArray)
+
+    try:
+        image = config.get('docker', 'image')
+    except:
+        image = "lofar"
+
+    # Construct the full command line, except for {command}, as that itself
+    # can contain spaces which we don't want to split on.
+    full_command_line = config.get('remote', 'cmdline').format(
+      uid          = os.geteuid(),
+      slurm_job_id = os.environ.get("SLURM_JOB_ID"),
+      docker_image = image,
+      host         = host,
+      command      = "{command}"
+    ).split(' ')
+
+    # Fill in {command} somewhere
+    full_command_line = [x.format(command = commandStr) for x in full_command_line]
+
+    logger.debug("Dispatching command to %s with custom_cmdline: %s" % (host, full_command_line))
+
+    process = spawn_process(full_command_line, logger)
+    process.kill = lambda : os.kill(process.pid, signal.SIGKILL)
+    return process
+
 def run_via_paramiko(logger, host, command, environment, arguments, key_filename):
     """
     Dispatch a remote command via paramiko.
@@ -270,8 +323,7 @@ class ComputeJob(object):
                     "PYTHONPATH": os.environ.get('PYTHONPATH'),
                     "LD_LIBRARY_PATH": os.environ.get('LD_LIBRARY_PATH'),
                     "LOFARROOT" : os.environ.get('LOFARROOT'),
-                    "QUEUE_PREFIX" : os.environ.get('QUEUE_PREFIX',''),
-                    "LOFAR_OBSID" : os.environ.get('LOFAR_OBSID','')
+                    "QUEUE_PREFIX" : os.environ.get('QUEUE_PREFIX','')
                 },
                 arguments = [id, jobhost, jobport]
             )

CEP/Pipeline/recipes/sip/pipeline.cfg.thead01.cep4

+# This pipeline.cfg was used on the CEP4 test system to show how Docker and SLURM
+# can be used to distribute and run jobs.
+#
+# Is called as follows (inside a SLURM job allocation, f.e. "salloc -N 2"). Replace directory names
+# where needed:
+#
+# # Create this on every node
+# CONFIGDIR=/globalhome/mol/regression_test
+# HOSTS=(`scontrol show hostnames $SLURM_JOB_NODELIST`)
+#
+# # $CONFIGDIR is local, and ocntains both the working_dir (for input/output data) and the config files
+# # /share is a shared disk for the vds files, map files, logs.
+#
+# docker run --rm -e SLURM_JOB_ID=${SLURM_JOB_ID} -e LUSER=${UID} -v $HOME/.ssh:/home/lofar/.ssh:ro -v $CONFIGDIR:$CONFIGDIR -v /shared:/shared --net=host lofar-patched /bin/bash -c "\"$EXTRA_CMDS;python $CONFIGDIR/regression_test_runner.py preprocessing_pipeline --pipelinecfg $CONFIGDIR/pipeline.cfg --testdata $CONFIGDIR/data --computehost1 ${HOSTS[0]} --computehost2 ${HOSTS[1]} --workdir $CONFIGDIR/working_dir --rundir $CONFIGDIR/rundir\""
+
+
+[DEFAULT]
+lofarroot = /opt/lofar
+casaroot = /opt/casacore
+pyraproot = /opt/python-casacore/pyrap
+hdf5root = 
+wcsroot = /opt/wcslib
+pythonpath = /opt/lofar/lib/python2.7/site-packages
+# runtime dir is a global FS (nfs, lustre) to exchange small files (parsets, vds, map files, etc)
+runtime_directory = /shared/mol/regression_test/rundir
+recipe_directories = [%(pythonpath)s/lofarpipe/recipes]
+# working dir is the local dir in which input/output dataproducts reside
+working_directory = /globalhome/mol/regression_test/working_dir
+task_files = [%(lofarroot)s/share/pipeline/tasks.cfg]
+
+[layout]
+job_directory = %(runtime_directory)s/%(job_name)s
+
+[cluster]
+clusterdesc = %(lofarroot)s/share/local.clusterdesc
+
+[deploy]
+engine_ppath = %(pythonpath)s:%(pyraproot)s/lib:/opt/cep/pythonlibs/lib/python/site-packages
+engine_lpath = %(lofarroot)s/lib:%(casaroot)s/lib:%(pyraproot)s/lib:%(hdf5root)s/lib:%(wcsroot)s/lib
+
+[logging]
+log_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/pipeline.log
+xml_stat_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/statistics.xml
+
+[feedback]
+# Method of providing feedback to LOFAR.
+# Valid options:
+#    messagebus    Send feedback and status using LCS/MessageBus
+#    none          Do NOT send feedback and status
+method = none
+
+[docker]
+image = lofar-patched
+
+[remote]
+#method = slurm_srun_cep3 
+#method = ssh_docker
+method = custom_cmdline
+max_per_node = 1 
+
+# We take the following path to start a remote container:
+#
+#  [container] --SSH--> [host] --SRUN--> [worker node] --DOCKER--> [container]
+#
+# This path is needed because running SRUN from within the container needs a lot of cluster-specific infra
+# (SLURM config files, Munge keys, correct Munge user ID, munged).
+#
+# Throughout this path, we maintain:
+#   * userid, which is set to the user that started the master script container (-e LUSER=`id -u`)
+#   * environment (sudo -p, docker -e K=V, etc)
+#   * pseudo-tty to prevent buffering logs (ssh -tt, docker -t)
+
+#
+# host -> worker node:       srun -w {host} -N 1 -n 1 --jobid={slurm_job_id}
+#                            Needs the specific host, because the test system does not have a global file system.
+#
+# worker node -> container:  docker run -t --rm -e LUSER={uid} -w g -v /home/mol/.ssh:/home/lofar/.ssh:ro -v /globalhome/mol/regression_test:/globalhome/mol/regression_test -v /shared:/shared --net=host {docker_image}
+#                            Starts the container on the worker node, with pretty much the same parameters as the master container:
+#
+#                            --rm: don't linger resources when the container exits
+#                            -e LUSER=(uid): set the user to run as (the calling user)
+#                            -h {host}: set container hostname (for easier debugging) (doesnt work yet, using --net=host instead)
+#                            -v:   map the directories for input/output data, shared storage (parsets, vds files, etc)
+#
+#                            /bin/bash -c
+#
+#                            Required because the pipeline framework needs some bash functionality in the commands it starts.
+#cmdline = ssh -n -tt -x localhost srun -w {host} -N 1 -n 1 --jobid={slurm_job_id} docker run -t --rm -e LUSER={uid} -w g -v /home/mol/.ssh:/home/lofar/.ssh:ro -v /globalhome/mol/regression_test:/globalhome/mol/regression_test -v /shared:/shared --net=host {docker_image} /bin/bash -c
+cmdline = ssh -n -tt -x localhost srun -w {host} -N 1 -n 1 --jobid={slurm_job_id} docker run -t --rm -e LUSER={uid} -v %(runtime_directory)s:%(runtime_directory)s -v %(working_directory)s:%(working_directory)s --net=host {docker_image} /bin/bash -c "\"{command}\""

CMake/LofarPackageList.cmake

@@ -16,6 +16,7 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
   set(LOFAR_PACKAGE_LIST_INCLUDED TRUE)
   set(Calibration_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration)
   set(DP3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3)
+  set(Docker_SOURCE_DIR ${CMAKE_SOURCE_DIR}/Docker)
   set(GSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/GSM)
   set(Imager_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Imager)
   set(LMWCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/LMWCommon)

CMake/UpdatePackageVersion.cmake.in

@@ -77,7 +77,11 @@ if(_result EQUAL 0)
   endif(_root MATCHES ".+")
   if(_rvers MATCHES "^/trunk/")
     set(_rvers "trunk")
+    set(_branch "trunk")
   elseif(_rvers MATCHES "^/tags/" OR _rvers MATCHES "^/branches/")
+    # branch = directory right under tags/ or branches/
+    string(REGEX REPLACE "^/([^/]*/[^/]*)/.*$" "\\1" _branch "${_rvers}")
+
     # We have a task branch (for a bug) or a release branch.
     # Remove all but version and replace _ in version by .
     string(REGEX REPLACE "^/[^/]*/([^/]*)/.*$" "\\1" _rvers "${_rvers}")
@@ -121,6 +125,8 @@ file(WRITE "${_file}.tmp"
   "    // Get the info for this package.\n"
   "    static Version getInfo();\n"
   "\n"
+  "    // Get the branch that was checked out (relative to repository root).\n"
+  "    static std::string getBranch();\n"
   "    // Get the version number of the package (as given in repository).\n"
   "    static std::string getVersion();\n"
   "    // Get the svn version number of the package according to CMakeLists.txt.\n"
@@ -168,12 +174,15 @@ file(APPEND "${_file}.tmp"
   "  Version ${_pkg}Version::getInfo()\n"
   "  {\n"
   "    return Version (\"@PACKAGE_NAME@\",\n"
+  "                    getBranch(),\n"
   "                    getVersion(), getConfVersion(),\n"
   "                    getRevision(), getPackageRevision(),\n"
   "                    getNrChangedFiles(),\n"
   "                    getBuildTime(), getBuildUser(), getBuildMachine());\n"
   "  }\n"
   "\n"
+  "  std::string ${_pkg}Version::getBranch()\n"
+  "    { return \"${_branch}\"; }\n"
   "  std::string ${_pkg}Version::getVersion()\n"
   "    { return \"${_rvers}\"; }\n"
   "  std::string ${_pkg}Version::getConfVersion()\n"

CMakeLists.txt

@@ -18,6 +18,7 @@ include(LofarGeneral)
 ## ---------------------------------------------------------------------------
 include(LofarPackage)
 if(NOT DEFINED BUILD_PACKAGES)
+  lofar_add_package(Docker)
   lofar_add_package(LCS)
   lofar_add_package(CEP)
   lofar_add_package(RTCP)

Docker/CMakeLists.txt

+# $Id$
+
+lofar_package(Docker 1.0 DEPENDS Common)
+
+include(LofarPackageVersion)
+
+# Build version info
+set(docker_LIB_SRCS
+  Package__Version.cc
+)
+
+lofar_add_library(docker ${docker_LIB_SRCS})
+
+lofar_add_bin_program(versiondocker versiondocker.cc)
+
+#
+# For given directories, we generate the Dockerfile
+# by parsing their Dockerfile.tmpl through "docker-template".
+#
+# "docker-template" is a script that fills in variables with
+# respect to the build info (branch name, build time, etc)
+#
+
+# Directories with Dockerfile.tmpl to parse
+set(DOCKER_TEMPLATE_DIRS
+  lofar-base
+  lofar-pipeline
+  lofar-outputproc)
+
+# Note: "docker-template" only works as long as the sources are still around,
+# since it uses svn to query information from them.
+lofar_add_bin_scripts(docker-template)
+lofar_add_sbin_scripts(docker-build-all.sh)
+
+# Convert Dockerfile.tmpl -> Dockerfile in ${DOCKER_TEMPLATE_DIRS}
+foreach(_dir ${DOCKER_TEMPLATE_DIRS})
+  # _src -> _dst
+  set(_src ${CMAKE_CURRENT_SOURCE_DIR}/${_dir}/Dockerfile.tmpl)
+  set(_dst ${CMAKE_CURRENT_BINARY_DIR}/${_dir}_Dockerfile)
+
+  # add generating command, and (any) target to force the generation
+  # when "all" is build.
+  add_custom_command(
+    OUTPUT ${_dst}
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/docker-template < ${_src} > ${_dst}
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/docker-template ${_src} ${CMAKE_CURRENT_BINARY_DIR}/versiondocker
+  )
+  add_custom_target(${_dir}_Dockerfile_target ALL DEPENDS ${_dst})
+
+  # install resulting Dockerfile
+  install(FILES
+    ${_dst}
+    DESTINATION share/docker/${_dir}
+    RENAME Dockerfile
+  )
+endforeach()
+
+# Install everything else
+install(DIRECTORY
+  lofar-base
+  lofar-pipeline
+  lofar-outputproc
+  DESTINATION share/docker
+  USE_SOURCE_PERMISSIONS
+  PATTERN Dockerfile.tmpl EXCLUDE)

Docker/docker-build-all.sh

+#!/bin/bash
+
+TAG=`echo '${LOFAR_TAG}' | docker-template`
+
+function build {
+  IMAGE=$1
+  docker build -t ${IMAGE}:${TAG} ${IMAGE}
+}
+
+cd ${LOFARROOT}/share/docker
+
+build lofar-base && \
+build lofar-pipeline && \
+build lofar-outputproc
+

Docker/docker-template

+#!/bin/bash 
+#
+# Template engine for LOFAR
+#
+#   Reads stdin, replaces variables, writes stdout.
+#
+#   Variables replaced:
+#       ${LOFAR_BRANCH_NAME}  = Name of this branch, relative to root
+#       ${LOFAR_BRANCH_URL}   = Full subversion URL for this branch
+#       ${LOFAR_TAG}          = Tag for a Docker image
+#       ${LOFAR_REVISION}     = SVN revision number of this checkout (or HEAD if trunk)
+#       ${NOW}          = now in UTC (format: 2016-01-01 10:11:12)
+#       ${BUILD_UID}    = uid of building user (=caller)
+#
+
+# ----- LOFAR_BRANCH_NAME = branches/LOFAR-Task1234 -----
+# ----- LOFAR_BRANCH_NAME = trunk -----
+# ----- LOFAR_BRANCH_NAME = tags/LOFAR-Release-2_15_1 -----
+# ----- LOFAR_BRANCH_NAME = UNKNOWN -----
+
+# Make sure we obtain info about the project source!
+#PATH=$PATH:.
+VERSION_INFO=`(versiondocker || ./versiondocker) 2>/dev/null` # in cmake, executable is in .
+
+# Extract branch name w.r.t. repository root, e.g. branches/LOFAR-Task1234
+export LOFAR_BRANCH_NAME=`echo "$VERSION_INFO" | perl -ne 'print "$1" if /branch += +(.+)/;'`
+
+# ----- LOFAR_BRANCH_URL = https://svn.astron.nl/LOFAR/branches/LOFAR-Task1234 -----
+# ----- LOFAR_BRANCH_URL = https://svn.astron.nl/LOFAR/trunk -----
+# ----- LOFAR_BRANCH_URL = https://svn.astron.nl/LOFAR/tags/LOFAR-Release-2_15_1 -----
+
+export LOFAR_BRANCH_URL="https://svn.astron.nl/LOFAR/${LOFAR_BRANCH_NAME}"
+
+# ----- LOFAR_TAG = 1234 -----
+# ----- LOFAR_TAG = trunk -----
+# ----- LOFAR_TAG = 2_15_1 -----
+
+case "${LOFAR_BRANCH_NAME}" in
+  trunk)      export LOFAR_TAG=trunk ;;
+  tags/*)     export LOFAR_TAG=${LOFAR_BRANCH_NAME##tags/LOFAR-Release-} ;;
+  branches/*) export LOFAR_TAG=${LOFAR_BRANCH_NAME##branches/LOFAR*Task} ;;
+  *)          export LOFAR_TAG=latest ;;
+esac
+
+# ----- LOFAR_REVISION = 12345 -----
+
+export LOFAR_REVISION=`echo "$VERSION_INFO" | perl -ne 'print "$1" if /overall revision += +([0-9]+)/;'`
+
+# ----- NOW = 2016-01-01 10:11:12 -----
+
+export NOW="`date -u +'%F %T'`"
+
+# ----- BUILD_UID = 1001 ----
+
+export BUILD_UID=`id -u`
+
+# ----- Process input -----
+
+# Insert our knowledge when processing stdin -> stdout
+envsubst '$LOFAR_BRANCH_NAME $LOFAR_BRANCH_URL $LOFAR_TAG $LOFAR_REVISION $NOW $BUILD_UID'
+

Docker/lofar-base/Dockerfile.tmpl

+#
+# base
+#
+FROM ubuntu:14.04
+
+#
+# common-environment
+#
+ENV USER=lofar
+ENV INSTALLDIR=/opt
+
+#
+# environment
+#
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHON_VERSION=2.7
+
+#
+# versions
+#
+ENV CASACORE_VERSION=2.0.3 \
+    CASAREST_VERSION=1.4.1 \
+    PYTHON_CASACORE_VERSION=2.0.1 \
+    BOOST_VERSION=1.54
+
+#
+# set-uid
+#
+ENV UID=${BUILD_UID}
+
+#
+# set-build-options
+#
+ENV J=4
+
+#
+# Base and runtime dependencies
+#
+#RUN sed -i 's/archive.ubuntu.com/osmirror.rug.nl/' /etc/apt/sources.list 
+RUN apt-get update && apt-get upgrade -y
+RUN apt-get install -y sudo python2.7 libpython2.7 && \
+    apt-get install -y libblas3 liblapacke python-numpy libcfitsio3 libwcs4 libfftw3-bin libhdf5-7 libboost-python${BOOST_VERSION}.0
+
+#
+# setup-account
+#
+RUN (getent group sudo &>/dev/null || groupadd sudo) && \
+    echo "useradd -m ${USERADD_FLAGS} ${USER}" && \
+    useradd -m -u ${UID} ${USER} && \
+    usermod -a -G sudo ${USER} && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    sed -i 's/requiretty/!requiretty/g' /etc/sudoers
+
+#
+# setup install dir
+#
+RUN mkdir -p ${INSTALLDIR} && chown ${USER}.${USER} ${INSTALLDIR}
+
+USER ${USER}
+
+#
+# *******************
+#   Casacore
+# *******************
+#
+RUN sudo apt-get install -y wget git cmake g++ gfortran flex bison libblas-dev liblapacke-dev libfftw3-dev libhdf5-dev libboost-python-dev libcfitsio3-dev wcslib-dev && \
+    mkdir -p ${INSTALLDIR}/casacore/build && \
+    mkdir -p ${INSTALLDIR}/casacore/data && \
+    cd ${INSTALLDIR}/casacore && git clone https://github.com/casacore/casacore.git src && \
+    if [ "${CASACORE_VERSION}" != "latest" ]; then cd ${INSTALLDIR}/casacore/src && git checkout tags/v${CASACORE_VERSION}; fi && \
+    cd ${INSTALLDIR}/casacore/data && wget --retry-connrefused ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
+    cd ${INSTALLDIR}/casacore/data && tar xf WSRT_Measures.ztar  && rm -f WSRT_Measures.ztar && \
+    cd ${INSTALLDIR}/casacore/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/casacore/ -DDATA_DIR=${INSTALLDIR}/casacore/data -DBUILD_PYTHON=True -DUSE_OPENMP=True -DUSE_FFTW3=TRUE ../src/ && \
+    cd ${INSTALLDIR}/casacore/build && make -j ${J} && \
+    cd ${INSTALLDIR}/casacore/build && make install && \
+    bash -c "strip ${INSTALLDIR}/casacore/{lib,bin}/* || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/casacore/{build,src}" && \
+    sudo apt-get purge -y wget git cmake g++ gfortran flex bison libblas-dev liblapacke-dev libfftw3-dev libhdf5-dev libboost-python-dev libcfitsio3-dev wcslib-dev && \
+    sudo apt-get autoremove -y
+
+#
+# *******************
+#   Casarest
+# *******************
+#
+RUN sudo apt-get install -y git cmake g++ gfortran libboost-system-dev libboost-thread-dev libhdf5-dev libcfitsio3-dev wcslib-dev && \
+    mkdir -p ${INSTALLDIR}/casarest/build && \
+    cd ${INSTALLDIR}/casarest && git clone https://github.com/casacore/casarest.git src && \
+    if [ "${CASAREST_VERSION}" != "latest" ]; then cd ${INSTALLDIR}/casarest/src && git checkout tags/v${CASAREST_VERSION}; fi && \
+    cd ${INSTALLDIR}/casarest/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/casarest -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore ../src/ && \
+    cd ${INSTALLDIR}/casarest/build && make -j ${J} && \
+    cd ${INSTALLDIR}/casarest/build && make install && \
+    bash -c "strip ${INSTALLDIR}/casarest/{lib,bin}/* || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/casarest/{build,src}" && \
+    sudo apt-get purge -y git cmake g++ gfortran libboost-system-dev libboost-thread-dev libhdf5-dev libcfitsio3-dev wcslib-dev && \
+    sudo apt-get autoremove -y
+
+#
+# *******************
+#   Pyrap
+# *******************
+#
+RUN sudo apt-get install -y git make g++ python-setuptools libboost-python-dev libcfitsio3-dev wcslib-dev && \
+    mkdir ${INSTALLDIR}/python-casacore && \
+    cd ${INSTALLDIR}/python-casacore && git clone https://github.com/casacore/python-casacore && \
+    if [ "$PYTHON_CASACORE_VERSION" != "latest" ]; then cd ${INSTALLDIR}/python-casacore/python-casacore && git checkout tags/v${PYTHON_CASACORE_VERSION}; fi && \
+    cd ${INSTALLDIR}/python-casacore/python-casacore && ./setup.py build_ext -I${INSTALLDIR}/casacore/include/ -L${INSTALLDIR}/casacore/lib/ && \
+    mkdir -p ${INSTALLDIR}/python-casacore/lib/python${PYTHON_VERSION}/site-packages/ && \
+    mkdir -p ${INSTALLDIR}/python-casacore/lib64/python${PYTHON_VERSION}/site-packages/ && \
+    export PYTHONPATH=${INSTALLDIR}/python-casacore/lib/python${PYTHON_VERSION}/site-packages:${INSTALLDIR}/python-casacore/lib64/python${PYTHON_VERSION}/site-packages:$PYTHONPATH && cd ${INSTALLDIR}/python-casacore/python-casacore && ./setup.py install --prefix=${INSTALLDIR}/python-casacore/ && \
+    bash -c "find ${INSTALLDIR}/python-casacore/lib -name '*.so' | xargs strip || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/python-casacore/python-casacore" && \
+    sudo apt-get purge -y git make g++ python-setuptools libboost-python-dev libcfitsio3-dev wcslib-dev && \
+    sudo apt-get autoremove -y
+
+#
+# *******************
+#   QPID client
+# *******************
+#
+
+# Allow auto-checkout from lofar repo
+RUN mkdir /home/${USER}/.subversion
+COPY subversion_servers /home/${USER}/.subversion/servers
+
+# Run-time dependencies
+# QPID daemon legacy store would require: libaio1 libdb5.1++
+RUN sudo apt-get install -y sasl2-bin libuuid1 libnss3 libnspr4 xqilla libboost-program-options${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0
+
+# Install
+# QPID daemon legacy store would require: libaio-dev libdb5.1++-dev
+RUN sudo apt-get install -y subversion swig ruby ruby-dev python-dev libsasl2-dev pkg-config cmake libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential debhelper libsslcommon2-dev libxqilla-dev python-setuptools libboost-program-options${BOOST_VERSION}-dev libboost-filesystem${BOOST_VERSION}-dev && \
+    mkdir /opt/qpid && \
+    svn --non-interactive -q --username lofar-guest --password lofar-guest co ${LOFAR_BRANCH_URL}/LCS/MessageBus/qpid/ /opt/qpid; \
+    /opt/qpid/local/sbin/build_qpid && \
+    bash -c "strip /opt/qpid/{bin,lib}/* || true" && \
+    bash -c "rm -rf ~/sources" && \
+    sudo apt-get purge -y subversion swig ruby ruby-dev python-dev libsasl2-dev pkg-config cmake libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential debhelper libsslcommon2-dev libxqilla-dev python-setuptools libboost-program-options${BOOST_VERSION}-dev libboost-filesystem${BOOST_VERSION}-dev && \
+    sudo apt-get autoremove -y
+#
+# *******************
+#   DAL
+# *******************
+#
+RUN sudo apt-get install -y git cmake g++ swig python-dev libhdf5-dev && \
+    mkdir ${INSTALLDIR}/DAL && \
+    cd ${INSTALLDIR}/DAL && git clone https://github.com/nextgen-astrodata/DAL.git src && \
+    mkdir ${INSTALLDIR}/DAL/build && cd ${INSTALLDIR}/DAL/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/DAL ../src && \
+    make -j ${J} && \
+    make install && \
+    bash -c "find ${INSTALLDIR}/DAL/lib -name '*.so' | xargs strip || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/DAL/{src,build}" && \
+    sudo apt-get purge -y git cmake g++ swig python-dev libhdf5-dev && \
+    sudo apt-get autoremove -y
+
+#
+# config
+#
+COPY bashrc /home/${USER}/.bashrc
+
+#
+# entry
+#
+COPY chuser.sh /usr/local/bin/chuser.sh
+WORKDIR /home/${USER}
+ENTRYPOINT ["sudo","-E","/usr/local/bin/chuser.sh"]
+

Docker/lofar-base/bashrc

+#!/bin/bash
+
+# lofar
+[ -r ${INSTALLDIR}/lofar/lofarinit.sh ] && source ${INSTALLDIR}/lofar/lofarinit.sh
+
+# qpid
+source ${INSTALLDIR}/qpid/.profile
+
+# python-casacore
+export PYTHONPATH=${PYTHONPATH}:${INSTALLDIR}/python-casacore/lib/python2.7/site-packages
+
+# casacore
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${INSTALLDIR}/casacore/lib

Docker/lofar-base/chuser.sh

+#!/usr/bin/env bash
+
+# Fetch the user name used in this container
+export USER=${SUDO_USER}
+
+if [ -n "${LUSER}" ]; then
+  if [ -z "${LGROUP}" ]; then
+    LGROUP=${LUSER}
+  fi
+
+  OLDID=`id -u ${USER}`
+
+  # Replace USER by LUSER:LGROUP
+  sed -i -e "s/${USER}:x:[0-9]\+:[0-9]\+/${USER}:x:${LUSER}:${LGROUP}/g" /etc/passwd
+  sed -i -e "s/${USER}:x:[0-9]\+:/${USER}:x:${LGROUP}:/g" /etc/group
+
+  # Set ownership of home dir to new user
+  chown --from=${OLDID} -R ${LUSER}:${LGROUP} ${HOME}
+fi
+
+# Switch to the updated user
+export HOME=/home/${USER}
+touch -a $HOME/.bashrc
+sudo -u ${USER} -E -s /bin/bash -c "source $HOME/.bashrc;$*"

Docker/lofar-base/subversion_servers

+[groups]
+astron = svn.astron.nl
+
+[astron]
+store-passwords = yes
+store-plaintext-passwords = yes

Docker/lofar-outputproc/Dockerfile.tmpl

+#
+# base
+#
+FROM lofar-base:${LOFAR_TAG}
+
+#
+# *******************
+#   LOFAR
+# *******************
+#
+
+# Run-time dependencies
+RUN sudo apt-get install -y liblog4cplus-1.0-4 libxml2 libboost-thread${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0 libboost-date-time${BOOST_VERSION}.0 libpng12-0 libsigc++-2.0-dev libxml++2.6-2 libboost-regex${BOOST_VERSION}.0
+
+# Tell image build information
+ENV LOFAR_BRANCH=${LOFAR_BRANCH_NAME} \
+    LOFAR_REVISION=${LOFAR_REVISION}
+
+# Install
+RUN sudo apt-get install -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libboost-regex${BOOST_VERSION} binutils-dev && \
+    mkdir -p ${INSTALLDIR}/lofar/build/gnu_opt libcfitsio3-dev wcslib-dev && \
+    cd ${INSTALLDIR}/lofar && \
+    svn --non-interactive -q --username lofar-guest --password lofar-guest co -r ${LOFAR_REVISION} -N ${LOFAR_BRANCH_URL} src; \
+    svn --non-interactive -q up src/CMake && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && cmake -DBUILD_PACKAGES=Online_OutputProc -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/lofar/ -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DLOG4CPLUS_ROOT_DIR=${INSTALLDIR}/log4cplus/ -DQPID_ROOT_DIR=/opt/qpid/ -DDAL_ROOT_DIR=${INSTALLDIR}/DAL -DUSE_OPENMP=True ${INSTALLDIR}/lofar/src/ && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && sed -i '29,31d' include/ApplCommon/PosixTime.h && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && make -j ${J} && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && make install && \
+    bash -c "strip ${INSTALLDIR}/lofar/{bin,sbin,lib64}/* || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/lofar/{build,src}" && \
+    sudo apt-get purge -y subversion cmake g++ gfortran bison flex autogen liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python${BOOST_VERSION}-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev binutils-dev libcfitsio3-dev wcslib-dev && \
+    sudo apt-get autoremove -y
+

Docker/lofar-pipeline/Dockerfile.tmpl

+#
+# base
+#
+FROM lofar-base:${LOFAR_TAG}
+
+ENV AOFLAGGER_VERSION=2.7.1
+
+# Run-time dependencies
+RUN sudo apt-get install -y python-xmlrunner python-scipy liblog4cplus-1.0-4 libxml2 libboost-thread${BOOST_VERSION}.0 libboost-filesystem${BOOST_VERSION}.0 libboost-date-time${BOOST_VERSION}.0 libboost-signals${BOOST_VERSION}.0 libpng12-0 libsigc++-2.0-dev libxml++2.6-2 libgsl0ldbl openssh-client libboost-regex${BOOST_VERSION}.0 && \
+    sudo apt-get -y install python-pip python-dev && \
+    sudo pip install pyfits pywcs python-monetdb && \
+    sudo apt-get -y purge python-pip python-dev && \
+    sudo apt-get -y autoremove
+
+#
+# *******************
+#   AOFlagger
+# *******************
+#
+
+RUN sudo apt-get install -y wget cmake g++ libxml++2.6-dev libpng12-dev libfftw3-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-signals${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libcfitsio3-dev && \
+    mkdir -p ${INSTALLDIR}/aoflagger/build && \
+    bash -c "cd ${INSTALLDIR}/aoflagger && wget --retry-connrefused http://downloads.sourceforge.net/project/aoflagger/aoflagger-${AOFLAGGER_VERSION%%.?}.0/aoflagger-${AOFLAGGER_VERSION}.tar.bz2" && \
+    cd ${INSTALLDIR}/aoflagger && tar xf aoflagger-${AOFLAGGER_VERSION}.tar.bz2 && \
+    cd ${INSTALLDIR}/aoflagger/build && cmake -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/aoflagger ../aoflagger-${AOFLAGGER_VERSION} && \
+    cd ${INSTALLDIR}/aoflagger/build && make -j ${J} && \
+    cd ${INSTALLDIR}/aoflagger/build && make install && \
+    bash -c "strip ${INSTALLDIR}/aoflagger/{lib,bin}/* || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/aoflagger/{build,aoflagger-${AOFLAGGER_VERSION}}" && \
+    bash -c "rm -rf ${INSTALLDIR}/aoflagger/aoflagger-${AOFLAGGER_VERSION}.tar.bz2" && \
+    sudo apt-get -y purge wget cmake g++ libxml++2.6-dev libpng12-dev libfftw3-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-signals${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libcfitsio3-dev && \
+    sudo apt-get -y autoremove
+
+#
+# *******************
+#   LOFAR
+# *******************
+#
+
+# Tell image build information
+ENV LOFAR_BRANCH=${LOFAR_BRANCH_NAME} \
+    LOFAR_REVISION=${LOFAR_REVISION}
+
+# Install
+RUN sudo apt-get install -y subversion cmake g++ gfortran bison flex liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python-dev python-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libgsl0-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libboost-regex${BOOST_VERSION} binutils-dev libcfitsio3-dev wcslib-dev && \
+    mkdir -p ${INSTALLDIR}/lofar/build/gnu_opt && \
+    cd ${INSTALLDIR}/lofar && \
+    svn --non-interactive -q --username lofar-guest --password lofar-guest co -r ${LOFAR_REVISION} -N ${LOFAR_BRANCH_URL} src; \
+    svn --non-interactive -q up src/CMake && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && cmake -DBUILD_PACKAGES=Offline -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/lofar/ -DCASAREST_ROOT_DIR=${INSTALLDIR}/casarest/ -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore/ -DAOFLAGGER_ROOT_DIR=${INSTALLDIR}/aoflagger/ -DLOG4CPLUS_ROOT_DIR=${INSTALLDIR}/log4cplus/ -DQPID_ROOT_DIR=/opt/qpid/ -DUSE_OPENMP=True ${INSTALLDIR}/lofar/src/ && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && sed -i '29,31d' include/ApplCommon/PosixTime.h && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && make -j ${J} && \
+    cd ${INSTALLDIR}/lofar/build/gnu_opt && make install && \
+    bash -c "strip ${INSTALLDIR}/lofar/{bin,sbin,lib64}/* || true" && \
+    bash -c "rm -rf ${INSTALLDIR}/lofar/{build,src}" && \
+    sudo apt-get purge -y subversion cmake g++ gfortran bison flex liblog4cplus-dev libhdf5-dev libblitz0-dev libboost-dev libboost-python-dev python-dev libxml2-dev pkg-config libpng12-dev libfftw3-dev libunittest++-dev libxml++2.6-dev libgsl0-dev libboost-filesystem${BOOST_VERSION}-dev libboost-date-time${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev binutils-dev wcslib-dev && \
+    sudo apt-get autoremove -y
+

LCS/Common/include/Common/Version.h

@@ -63,6 +63,7 @@ namespace LOFAR {
 
     // Construct a Version object from the relevant package info.
     Version (const std::string& packageName,
+	     const std::string& branch,
 	     const std::string& version,
 	     const std::string& confVersion,
 	     const std::string& revision,
@@ -75,6 +76,9 @@ namespace LOFAR {
     // Get the package name.
     const std::string& packageName() const
       { return itsPackageName; }
+    // Get the LOFAR branch version (relative to repository root)
+    const std::string& branch() const
+      { return itsBranch; }
     // Get the package version (as in repository)
     const std::string& version() const
       { return itsVersion; }
@@ -195,6 +199,7 @@ namespace LOFAR {
 
     //# Data members.
     std::string itsPackageName;
+    std::string itsBranch;
     std::string itsVersion;
     std::string itsConfVersion;
     std::string itsRevision;

LCS/Common/src/Version.cc

@@ -29,6 +29,7 @@
 namespace LOFAR {
 
   Version::Version (const std::string& packageName,
+		    const std::string& branch,
 		    const std::string& version,
 		    const std::string& confVersion,
 		    const std::string& revision,
@@ -38,6 +39,7 @@ namespace LOFAR {
 		    const std::string& buildUser,
 		    const std::string& buildMachine)
     : itsPackageName     (packageName),
+      itsBranch          (branch),
       itsVersion         (version),
       itsConfVersion     (confVersion),
       itsRevision        (revision),
@@ -129,6 +131,7 @@ namespace LOFAR {
       os << " (in CMakeLists.txt: " << confVersion() << ')';
     }
     os << std::endl;
+    os << indent << " branch = " << branch() << std::endl;
     os << indent << " overall revision  = " << revision();
     if (!sameRev) os << " (note: used packages have different revision)";
     os << std::endl;