From 3abdce5ac923528d3f1e1684db1c84a410c2959c Mon Sep 17 00:00:00 2001
From: Adriaan Renting <renting@astron.nl>
Date: Fri, 15 Feb 2019 18:27:56 +0000
Subject: [PATCH] SW-310: Added a docker template lofar-test to build a docker
that can build the lofar code and run the various ctest tests.
---
.gitattributes | 8 +
Docker/lofar-test/Dockerfile.tmpl | 214 ++++++++++++++++++
Docker/lofar-test/bashrc | 18 ++
Docker/lofar-test/bashrc.d/00-casacore | 4 +
Docker/lofar-test/bashrc.d/00-qpid | 4 +
Docker/lofar-test/bashrc.d/01-python-casacore | 2 +
Docker/lofar-test/bashrc.d/50-lofar | 3 +
Docker/lofar-test/casarc | 2 +
Docker/lofar-test/chuser.sh | 22 ++
9 files changed, 277 insertions(+)
create mode 100644 Docker/lofar-test/Dockerfile.tmpl
create mode 100644 Docker/lofar-test/bashrc
create mode 100644 Docker/lofar-test/bashrc.d/00-casacore
create mode 100644 Docker/lofar-test/bashrc.d/00-qpid
create mode 100644 Docker/lofar-test/bashrc.d/01-python-casacore
create mode 100644 Docker/lofar-test/bashrc.d/50-lofar
create mode 100644 Docker/lofar-test/casarc
create mode 100755 Docker/lofar-test/chuser.sh
diff --git a/.gitattributes b/.gitattributes
index 747556a0753..e210c82b6d4 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1347,6 +1347,14 @@ Docker/lofar-pulp/sudoers -text
Docker/lofar-tbbwriter/Dockerfile -text
Docker/lofar-tbbwriter/bashrc -text
Docker/lofar-tbbwriter/chuser.sh -text
+Docker/lofar-test/Dockerfile.tmpl -text
+Docker/lofar-test/bashrc -text
+Docker/lofar-test/bashrc.d/00-casacore -text
+Docker/lofar-test/bashrc.d/00-qpid -text
+Docker/lofar-test/bashrc.d/01-python-casacore -text
+Docker/lofar-test/bashrc.d/50-lofar -text
+Docker/lofar-test/casarc -text
+Docker/lofar-test/chuser.sh -text
Docker/lofar-triggerservices/Dockerfile -text
Docker/lofar-triggerservices/README.md -text
Docker/lofar-triggerservices/ansible/bld.yaml -text
diff --git a/Docker/lofar-test/Dockerfile.tmpl b/Docker/lofar-test/Dockerfile.tmpl
new file mode 100644
index 00000000000..c4c13701424
--- /dev/null
+++ b/Docker/lofar-test/Dockerfile.tmpl
@@ -0,0 +1,214 @@
+#
+# base
+#
+FROM ubuntu:18.04
+
+# Make sure that Bash is the default shell. Otherwise
+# the shell's string substitutions won't work.
+SHELL ["/bin/bash", "-c"]
+
+# Allow to specify the LOFAR branch at build time of the image.
+# Use master as the default if nothing is specified.
+ARG LOFAR_VERSION=latest
+ENV LOFAR_VERSION=${LOFAR_VERSION}
+
+# LOFAR_TAG is an environment variable that gets used
+# (at least) by the lofar-pipeline image config file
+# CEP/Pipeline/recipes/sip/pipeline.cfg.CEP4.
+# It determines which version of the lofar-pipeline
+# images gets executed on the nodes.
+ENV LOFAR_TAG=${LOFAR_VERSION}
+
+# Tell image build information.
+ARG LOFAR_BUILDVARIANT=gnucxx11_optarch
+ENV LOFAR_BUILDVARIANT=${LOFAR_BUILDVARIANT}
+
+# Allow to specify the wanted CXX
+# Default is the new one.
+ARG CXX_ABI=1
+ENV CXX_ABI=${CXX_ABI}
+
+
+#
+# common-environment
+#
+ENV INSTALLDIR=/opt
+
+#
+# environment
+#
+ENV DEBIAN_FRONTEND=noninteractive \
+ PYTHON_VERSION=2.7
+
+#
+# versions
+#
+ENV CASACORE_VERSION=2.2.0 \
+ CASAREST_VERSION=latest \
+ PYTHON_CASACORE_VERSION=2.1.2 \
+ BOOST_VERSION=1.62 \
+ LIBHDF5_VERSION=100 \
+ READLINE_VERSION=7 \
+ NCURSES_VERSION=5
+
+#
+# set-build-options
+# Allow to specify the number of cpus as --build-arg.
+#
+ARG J=6
+ENV J=${J} \
+ CXX_FLAGS="--std=c++11 -W -Wall -Woverloaded-virtual -Wno-unknown-pragmas -D_GLIBCXX_USE_CXX11_ABI=1 -O2 -march=native"
+
+#
+# Base and runtime dependencies
+#
+#RUN sed -i 's/archive.ubuntu.com/osmirror.rug.nl/' /etc/apt/sources.list
+
+RUN apt-get update && apt-get upgrade -y && \
+ apt-get install -y apt-utils aptitude bash-completion mc most htop nano sudo vim python2.7 libpython2.7 libboost-python${BOOST_VERSION} libreadline${READLINE_VERSION} libncurses${NCURSES_VERSION} libopenblas-base libcfitsio-bin libwcs5 libfftw3-bin libhdf5-${LIBHDF5_VERSION} libhdf5-dev && \
+ apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get -y autoremove --purge
+
+# Install numpy
+RUN export BUILD_PACKAGES="python-setuptools python-pip" && \
+ apt-get install -y ${BUILD_PACKAGES} && \
+ pip install numpy && \
+ apt-get purge -y ${BUILD_PACKAGES} && \
+ apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get autoremove -y --purge
+
+#
+# open security holes (allow smooth user switching, allow sudo)
+#
+RUN echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+ sed -i 's/requiretty/!requiretty/g' /etc/sudoers
+
+#
+# setup install dir
+#
+RUN mkdir -p ${INSTALLDIR}
+
+#
+# *******************
+# Casacore
+# *******************
+#
+RUN export BUILD_PACKAGES="wget git cmake g++ gfortran flex bison libreadline-dev libncurses-dev libopenblas-dev libfftw3-dev libboost-python${BOOST_VERSION}-dev libcfitsio-dev wcslib-dev" && \
+ apt-get install -y ${BUILD_PACKAGES} && \
+ mkdir -p ${INSTALLDIR}/casacore/build && \
+ mkdir -p ${INSTALLDIR}/casacore/data && \
+ cd ${INSTALLDIR}/casacore && git clone https://github.com/casacore/casacore.git src && \
+ if [ "${CASACORE_VERSION}" != "latest" ]; then cd ${INSTALLDIR}/casacore/src && git checkout tags/v${CASACORE_VERSION}; fi && \
+ cd ${INSTALLDIR}/casacore/data && wget --retry-connrefused ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar && \
+ cd ${INSTALLDIR}/casacore/data && tar xf WSRT_Measures.ztar && rm -f WSRT_Measures.ztar && \
+ cd ${INSTALLDIR}/casacore/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/casacore/ -DDATA_DIR=${INSTALLDIR}/casacore/data -DBUILD_PYTHON=True -DENABLE_TABLELOCKING=OFF -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="${CXX_FLAGS} -fsigned-char -DNDEBUG" ../src/ && \
+ cd ${INSTALLDIR}/casacore/build && make -j ${J} && \
+ cd ${INSTALLDIR}/casacore/build && make install && \
+ bash -c "strip ${INSTALLDIR}/casacore/{lib,bin}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/casacore/{build,src}" && \
+ apt-get purge -y ${BUILD_PACKAGES} && \
+ apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get autoremove -y --purge
+
+# Install and enable custom casarc
+COPY ["casarc", "${INSTALLDIR}/"]
+ENV CASARCFILES=${INSTALLDIR}/casarc
+
+#
+# *******************
+# Casarest
+# *******************
+#
+# Run-time dependencies
+RUN apt-get install -y libboost-system${BOOST_VERSION} libboost-thread${BOOST_VERSION}
+
+# Install
+RUN export BUILD_PACKAGES="git cmake g++ gfortran libboost-system${BOOST_VERSION}-dev libboost-thread${BOOST_VERSION}-dev libcfitsio-dev wcslib-dev libopenblas-dev" && \
+ apt-get install -y ${BUILD_PACKAGES} && \
+ mkdir -p ${INSTALLDIR}/casarest/build && \
+ cd ${INSTALLDIR}/casarest && git clone https://github.com/casacore/casarest.git src && \
+ if [ "${CASAREST_VERSION}" != "latest" ]; then cd ${INSTALLDIR}/casarest/src && git checkout tags/v${CASAREST_VERSION}; fi && \
+ cd ${INSTALLDIR}/casarest/build && cmake -DCMAKE_INSTALL_PREFIX=${INSTALLDIR}/casarest -DCASACORE_ROOT_DIR=${INSTALLDIR}/casacore -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="${CXX_FLAGS} -DNDEBUG" ../src/ && \
+ cd ${INSTALLDIR}/casarest/build && make -j ${J} && \
+ cd ${INSTALLDIR}/casarest/build && make install && \
+ bash -c "strip ${INSTALLDIR}/casarest/{lib,bin}/* || true" && \
+ bash -c "rm -rf ${INSTALLDIR}/casarest/{build,src}" && \
+ apt-get purge -y ${BUILD_PACKAGES} && \
+ apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get autoremove -y --purge
+
+#
+# *******************
+# Pyrap
+# *******************
+#
+RUN export BUILD_PACKAGES="git make g++ python-setuptools libboost-python${BOOST_VERSION}-dev libcfitsio-dev wcslib-dev" && \
+ apt-get install -y ${BUILD_PACKAGES} && \
+ mkdir ${INSTALLDIR}/python-casacore && \
+ cd ${INSTALLDIR}/python-casacore && git clone https://github.com/casacore/python-casacore && \
+ if [ "$PYTHON_CASACORE_VERSION" != "latest" ]; then cd ${INSTALLDIR}/python-casacore/python-casacore && git checkout tags/v${PYTHON_CASACORE_VERSION}; fi && \
+ cd ${INSTALLDIR}/python-casacore/python-casacore && ./setup.py build_ext -I${INSTALLDIR}/casacore/include/ -L${INSTALLDIR}/casacore/lib/ && \
+ mkdir -p ${INSTALLDIR}/python-casacore/lib/python${PYTHON_VERSION}/site-packages/ && \
+ mkdir -p ${INSTALLDIR}/python-casacore/lib64/python${PYTHON_VERSION}/site-packages/ && \
+ export PYTHONPATH=${INSTALLDIR}/python-casacore/lib/python${PYTHON_VERSION}/site-packages:${INSTALLDIR}/python-casacore/lib64/python${PYTHON_VERSION}/site-packages:$PYTHONPATH && cd ${INSTALLDIR}/python-casacore/python-casacore && ./setup.py install --prefix=${INSTALLDIR}/python-casacore/ && \
+ bash -c "rm -rf ${INSTALLDIR}/python-casacore/python-casacore" && \
+ apt-get purge -y ${BUILD_PACKAGES} && \
+ apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get autoremove -y --purge
+
+
+#
+# *******************
+# QPID client
+# *******************
+#
+# Run-time dependencies
+# QPID daemon legacy store would require: libaio1 libdb5.1++
+RUN apt-get install -y sasl2-bin libuuid1 libnss3 libnspr4 xqilla libssl1.1 libssl1.0.0 libboost-program-options${BOOST_VERSION} libboost-filesystem${BOOST_VERSION}
+
+# Install
+# QPID daemon legacy store would require: libaio-dev libdb5.1++-dev
+RUN export BUILD_PACKAGES="git rsync swig ruby ruby-dev python-dev python-setuptools libsasl2-dev pkg-config cmake libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential g++ debhelper libssl-dev libxqilla-dev libboost-program-options${BOOST_VERSION}-dev libboost-filesystem${BOOST_VERSION}-dev" && \
+ apt-get install -y ${BUILD_PACKAGES} && \
+ mkdir ${INSTALLDIR}/qpid && \
+ git clone --branch ${LOFAR_VERSION//latest/master} https://git.astron.nl/LOFAR /tmp/LOFAR && \
+ rsync --archive /tmp/LOFAR/LCS/MessageBus/qpid/ ${INSTALLDIR}/qpid/ && \
+ rm -rf /tmp/LOFAR && \
+ bash -c "HOME=/tmp ${INSTALLDIR}/qpid/local/sbin/build_qpid" && \
+ bash -c "strip ${INSTALLDIR}/qpid/{bin,lib}/* || true" && \
+ bash -c "rm -rf /tmp/sources" && \
+ apt-get purge -y ${BUILD_PACKAGES} && \
+ apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get autoremove -y --purge
+
+#RUN echo Europe/Amsterdam >/etc/timezone && \
+# dpkg-reconfigure -f noninteractive tzdata
+#RUN export BUILD_PACKAGES="tzdata git rsync swig ruby ruby-dev python-dev python-setuptools python-mock python-psycopg2 python-testing.postgresql python-mysql.connector python-testing.mysqld python-django python-djangorestframework python-ldap python-dateutil python-coverage libxml++2.6-dev libunittest++-dev python-lxml python-pip liblog4cplus-dev libsasl2-dev pkg-config make g++ cmake gfortran libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential g++ debhelper libssl-dev libxqilla-dev libboost-all-dev libcfitsio-dev wcslib-dev" && \
+# apt-get install -y ${BUILD_PACKAGES}
+RUN export BUILD_PACKAGES="tzdata git rsync swig ruby ruby-dev python-dev python-setuptools python-mock python-psycopg2 python-testing.postgresql python-mysql.connector python-django python-djangorestframework python-ldap libxml++2.6-dev libunittest++-dev python-lxml python-pip liblog4cplus-dev libsasl2-dev pkg-config make g++ cmake gfortran libtool uuid-dev libxerces-c-dev libnss3-dev libnspr4-dev help2man fakeroot build-essential g++ debhelper libssl-dev libxqilla-dev libboost-all-dev libcfitsio-dev wcslib-dev" && \
+ apt-get install -y ${BUILD_PACKAGES}
+
+#RUN apt-get install -y python-testing.mysqld
+
+
+RUN pip install xmljson
+RUN pip install djangorestframework-xml
+RUN useradd renting
+#
+# entry
+#
+COPY ["bashrc", "${INSTALLDIR}/"]
+COPY ["bashrc.d", "${INSTALLDIR}/bashrc.d/"]
+COPY ["chuser.sh", "/usr/local/bin"]
+# Make sure that all files are readable by u,g,o and that
+# the ENTRYPOINT script is also executable.
+RUN chmod -R a+rx /usr/local/bin && \
+ find /opt/ ! -perm -a+r -exec chmod a+r {} +
+ENTRYPOINT ["/usr/local/bin/chuser.sh"]
+
diff --git a/Docker/lofar-test/bashrc b/Docker/lofar-test/bashrc
new file mode 100644
index 00000000000..2f0b31b34ba
--- /dev/null
+++ b/Docker/lofar-test/bashrc
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Read all profiles
+shopt -s nullglob
+for rc in ${INSTALLDIR}/bashrc.d/*; do
+ source $rc
+done
+
+bind '"\e[A": history-search-backward'
+bind '"\e[B": history-search-forward'
+
+# If id command returns zero, you have root access.
+if [ $(id -u) -eq 0 ];
+then # you are root, set red colour prompt
+ PS1="\\[$(tput setaf 1)\\]\\u@\\h:\\w #\\[$(tput sgr0)\\]"
+else # normal
+ PS1="\\u@\\h \\d >"
+fi
diff --git a/Docker/lofar-test/bashrc.d/00-casacore b/Docker/lofar-test/bashrc.d/00-casacore
new file mode 100644
index 00000000000..f3396ddfec1
--- /dev/null
+++ b/Docker/lofar-test/bashrc.d/00-casacore
@@ -0,0 +1,4 @@
+#!/bin/bash
+export PATH=${PATH}:${INSTALLDIR}/casacore/bin
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${INSTALLDIR}/casacore/lib
+export CASACORE_ROOT_DIR=${INSTALLDIR}/casacore/include/casacore/
diff --git a/Docker/lofar-test/bashrc.d/00-qpid b/Docker/lofar-test/bashrc.d/00-qpid
new file mode 100644
index 00000000000..970a66787d4
--- /dev/null
+++ b/Docker/lofar-test/bashrc.d/00-qpid
@@ -0,0 +1,4 @@
+#!/bin/bash
+source ${INSTALLDIR}/qpid/.profile
+export QPID_LIBRARY=${INSTALLDIR}/qpid/lib
+export QPID_INCLUDE_DIR=${INSTALLDIR}/qpid/include
diff --git a/Docker/lofar-test/bashrc.d/01-python-casacore b/Docker/lofar-test/bashrc.d/01-python-casacore
new file mode 100644
index 00000000000..d58385ad518
--- /dev/null
+++ b/Docker/lofar-test/bashrc.d/01-python-casacore
@@ -0,0 +1,2 @@
+#!/bin/bash
+export PYTHONPATH=${PYTHONPATH}:${INSTALLDIR}/python-casacore/lib/python2.7/site-packages
diff --git a/Docker/lofar-test/bashrc.d/50-lofar b/Docker/lofar-test/bashrc.d/50-lofar
new file mode 100644
index 00000000000..10ba02c20fd
--- /dev/null
+++ b/Docker/lofar-test/bashrc.d/50-lofar
@@ -0,0 +1,3 @@
+#!/bin/bash
+[ -r ${INSTALLDIR}/lofar/lofarinit.sh ] && source ${INSTALLDIR}/lofar/lofarinit.sh
+export PATH PYTHONPATH LD_LIBRARY_PATH LOFARROOT
diff --git a/Docker/lofar-test/casarc b/Docker/lofar-test/casarc
new file mode 100644
index 00000000000..f988729b2f1
--- /dev/null
+++ b/Docker/lofar-test/casarc
@@ -0,0 +1,2 @@
+# Write MeasurementSets using fewer files, and 4MB blocks
+tables.storage.type = multifile
diff --git a/Docker/lofar-test/chuser.sh b/Docker/lofar-test/chuser.sh
new file mode 100755
index 00000000000..4ba563587e5
--- /dev/null
+++ b/Docker/lofar-test/chuser.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Configure user
+if [ -z "${USER}" ]; then
+ export USER=${UID}
+fi
+
+# Create home directory
+if [ -z "${HOME}" ]; then
+ export HOME=/home/${USER}
+ mkdir -p $HOME && cd $HOME
+fi
+
+# Set the environment
+[ -e /opt/bashrc ] && source /opt/bashrc
+
+# Run the requested command
+if [ -z "$*" ]; then
+ exec /bin/bash
+else
+ exec "$@"
+fi
--
GitLab