SW-394: Add a bunch of Docker images to build individual packages

485ddb05 · Jörn Künsemöller · 41d82ec3 · 485ddb05 · 485ddb05 · 485ddb05
Commit 485ddb05 authored 5 years ago by Jörn Künsemöller
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
-image: centos:7
-before_script:
- - echo "Installing dependencies (this should go in a custom Docker image for CI to avoid reinstalling dependencies for each stage)..."
- - yum -y groupinstall 'Development Tools'
- - yum -y install epel-release
- - yum -y install cmake python3 python3-devel python3-pip log4cplus-devel postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools
- - pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql
 stages:
  - build
  - test
-build_TriggerServices:
+#
+# BUILD STAGE
+#
+build_CEP:
+  stage: build
+  image: ci_cep:latest
+  script:
+    - PACKAGE=CEP
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/casacore/install/ -DCASAREST_ROOT_DIR=/casarest/install/ ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_Docker:
+  stage: build
+  image: ci_docker:latest
+  script:
+    - PACKAGE=Docker
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_LCS:
+  stage: build
+  image: ci_lcs:latest
+  script:
+    - PACKAGE=LCS
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    #- docker run -d -v ~/wincc:/opt/wincc wincc:lates # todo: set correct path and wincc image
+    - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/casacore/install/ -DWINCC_ROOT_DIR=/wincc/ ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_LCU:
  stage: build
+  image: ci_lcu:latest
  script:
-    - echo "Building TriggerServices..."
+    - PACKAGE=LCU
-    - mkdir -p build/gnucxx11_opt 
+    - VARIANT=gnucxx11_opt
-    - cd build/gnucxx11_opt
+    - echo "Building $PACKAGE..."
-    - cmake -DBUILD_PACKAGES=TriggerServices ../.. 
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE ../..
    - make
  artifacts:
    paths:
      - build/
-test:
+build_LTA:
+  stage: build
+  image: ci_lta:latest
+  script:
+    - PACKAGE=LTA
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_MAC:
+  stage: build
+  image: ci_mac:latest
+  script:
+    - PACKAGE=MAC
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_QA:
+  stage: build
+  image: ci_qa:latest
+  script:
+    - PACKAGE=QA
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_RTCP:
+  stage: build
+  image: ci_rtcp:latest
+  script:
+    - PACKAGE=SAS
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/casacore/install/ -DDAL_ROOT_DIR=/DAL/install/ -DUSE_MPI=True -DUSE_OPENMP=True ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+build_SAS:
+  stage: build
+  image: ci_sas:latest
+  script:
+    - PACKAGE=SAS
+    - VARIANT=gnucxx11_opt
+    - echo "Building $PACKAGE..."
+    - mkdir -p build/$VARIANT
+    - cd build/$VARIANT
+    - cmake -DBUILD_PACKAGES=$PACKAGE ../..
+    - make
+  artifacts:
+    paths:
+      - build/
+# (part of SAS):
+# build_TriggerServices:
+#   stage: build
+#   image: ci_triggerservices:latest
+#   script:
+#     - PACKAGE=TriggerServices
+#     - VARIANT=gnucxx11_opt
+#     - echo "Building $PACKAGE..."
+#     - mkdir -p build/$VARIANT
+#     - cd build/$VARIANT
+#     - cmake -DBUILD_PACKAGES=$PACKAGE ../..
+#     - make
+#
+#   artifacts:
+#     paths:
+#       - build/
+#
+# TEST STAGE
+#
+# TODO: I guess we have to run cmake before each of these so ctest tests the correct package...
+test_CEP:
+  stage: test
+  image: ci_cep:latest
+  script:
+    - PACKAGE=CEP
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_Docker:
+  stage: test
+  image: ci_docker:latest
+  script:
+    - PACKAGE=Docker
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_LCS:
+  stage: test
+  image: ci_lcs:latest
+  script:
+    - PACKAGE=LCS
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_LCU:
+  stage: test
+  image: ci_lcu:latest
+  script:
+    - PACKAGE=LCU
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_LTA:
+  stage: test
+  image: ci_lta:latest
+  script:
+    - PACKAGE=LTA
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_MAC:
+  stage: test
+  image: ci_mac:latest
+  script:
+    - PACKAGE=MAC
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_QA:
+  stage: test
+  image: ci_qa:latest
+  script:
+    - PACKAGE=QA
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_RTCP:
+  stage: test
+  image: ci_rtcp:latest
+  script:
+    - PACKAGE=RTCP
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
+    - ctest
+test_SAS:
  stage: test
+  image: ci_sas:latest
  script:
-    - echo "Testing TriggerServices..."
+    - PACKAGE=SAS
-    - cd build/gnucxx11_opt
+    - VARIANT=gnucxx11_opt
+    - echo "Testing $PACKAGE..."
+    - cd build/$VARIANT
    - ctest
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_base
+++ b/Docker/lofar-ci/Dockerfile_ci_base
+#
+# This base image is just some beasic dev tools on top of CentOS 7
+#
+# base
+#
+FROM centos:7
+RUN yum -y groupinstall 'Development Tools' && \
+    yum -y install epel-release && \
+    yum -y install cmake log4cplus-devel python3 python3-devel python3-pip
--- a/Docker/lofar-ci/Dockerfile_ci_cep
+++ b/Docker/lofar-ci/Dockerfile_ci_cep
+#
+# This builds an image with all dependencies for the CEP package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for CEP..." && \
+    yum -y install boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel
+    pip3 install numpy
+RUN echo "Installing Casacore..." && \
+    git clone https://github.com/casacore/casacore && \
+    mkdir /casacore/build/ && \
+    cd /casacore/build/ && \
+    cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
+    make && \
+    make install
+RUN echo "Installing Casarest..." && \
+    cd / && git clone https://github.com/casacore/casarest.git && \
+    mkdir -p /casarest/build/ && \
+    cd /casarest/build/ && \
+    cmake -DCMAKE_INSTALL_PREFIX=../install -DCASACORE_ROOT_DIR=/casacore/install -DCMAKE_BUILD_TYPE=Release .. && \
+    make && \
+    make install
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_docker
+++ b/Docker/lofar-ci/Dockerfile_ci_docker
+#
+# This builds an image with all dependencies for the QA package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for Docker..." && \
+    yum -y install boost-python36-devel readline-devel
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_lcs
+++ b/Docker/lofar-ci/Dockerfile_ci_lcs
+#
+# This builds an image with all dependencies for the LCS package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for LCS..." && \
+    yum -y install  boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools unittest-cpp-devel readline-devel && \
+    pip3 install numpy kombu requests
+RUN echo "Installing Casacore..." && \
+    git clone https://github.com/casacore/casacore && \
+    mkdir /casacore/build/ && \
+    cd /casacore/build/ && \
+    cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
+    make && \
+    make install
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_lcu
+++ b/Docker/lofar-ci/Dockerfile_ci_lcu
+#
+# This builds an image with all dependencies for the LCU package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for LCU..." && \
+    yum -y install postgresql-devel && \
+    pip3 install psycopg2 testing.postgresql lxml mock numpy
--- a/Docker/lofar-ci/Dockerfile_ci_lta
+++ b/Docker/lofar-ci/Dockerfile_ci_lta
+#
+# This builds an image with all dependencies for the LTA package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for LTA..." && \
+    yum -y install postgresql-devel && \
+    pip3 install kombu requests pysimplesoap mysql-connector psycopg2 flask
--- a/Docker/lofar-ci/Dockerfile_ci_mac
+++ b/Docker/lofar-ci/Dockerfile_ci_mac
+#
+# This builds an image with all dependencies for the LCU package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for LCU..." && \
+    yum -y install readline-devel boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel autogen && \
+    pip3 install psycopg2 testing.postgresql lxml mock numpy
+RUN echo "Installing Casacore..." && \
+    git clone https://github.com/casacore/casacore && \
+    mkdir /casacore/build/ && \
+    cd /casacore/build/ && \
+    cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
+    make && \
+    make install
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_qa
+++ b/Docker/lofar-ci/Dockerfile_ci_qa
+#
+# This builds an image with all dependencies for the QA package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for QA..." && \
+    yum -y install boost-python36-devel readline-devel postgresql-devel && \
+    pip3 install kombu requests psycopg2 testing.postgresql
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_rtcp
+++ b/Docker/lofar-ci/Dockerfile_ci_rtcp
+#
+# This builds an image with all dependencies for the RTCP package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for RTCP..." && \
+    yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \
+    yum -y install autogen  boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel ncurses-devel readline-devel fftw-devel libpqxx-devel unittest-cpp-devel numactl-devel openmpi-devel cuda && \
+    pip3 install numpy
+RUN echo "Installing Casacore..." && \
+    git clone https://github.com/casacore/casacore && \
+    mkdir /casacore/build/ && \
+    cd /casacore/build/ && \
+    cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
+    make && \
+    make install
+RUN echo "Installing DAL..." && \
+    git clone https://github.com/nextgen-astrodata/DAL.git && \
+    mkdir /DAL/build && \
+    cd /DAL/build/ && \
+    cmake -DCMAKE_INSTALL_PREFIX=../install -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 -DPYTHON_LIBRARY:FILEPATH=/usr/lib64/libpython3.6m.so -DPYTHON_INCLUDE_DIR=/usr/include/python3.6m/ .. && \
+    make && \
+    make install
--- a/Docker/lofar-ci/Dockerfile_ci_sas
+++ b/Docker/lofar-ci/Dockerfile_ci_sas
+#
+# This builds an image with all dependencies for the SAS package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for SAS..." && \
+    yum -y install postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools libpqxx-devel java-devel qt-devel autogen boost-python36-devel && \
+    pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql
--- a/Docker/lofar-ci/Dockerfile_ci_triggerservices
+++ b/Docker/lofar-ci/Dockerfile_ci_triggerservices
+#
+# This builds an image with all dependencies for the TriggerServices package based on the ci_base image
+#
+# base
+#
+FROM ci_base:latest
+RUN echo "Installing packages for TriggerServices..." && \
+    yum -y install postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools && \
+    pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql