diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05bb6a7c688e609163fe51886dbf427c7f94eb02..6c409fe9ac0dcff2bcc5d58fc9b35c609039df61 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,286 @@ -image: centos:7 -before_script: - - echo "Installing dependencies (this should go in a custom Docker image for CI to avoid reinstalling dependencies for each stage)..." - - yum -y groupinstall 'Development Tools' - - yum -y install epel-release - - yum -y install cmake python3 python3-devel python3-pip log4cplus-devel postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools - - pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql stages: - build - test -build_TriggerServices: +# +# BUILD STAGE +# + +build_CEP: + stage: build + image: ci_cep:latest + script: + - PACKAGE=CEP + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/casacore/install/ -DCASAREST_ROOT_DIR=/casarest/install/ ../.. + - make + + artifacts: + paths: + - build/ + + +build_Docker: + stage: build + image: ci_docker:latest + script: + - PACKAGE=Docker + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE ../.. + - make + + artifacts: + paths: + - build/ + + +build_LCS: + stage: build + image: ci_lcs:latest + script: + - PACKAGE=LCS + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + #- docker run -d -v ~/wincc:/opt/wincc wincc:lates # todo: set correct path and wincc image + - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/casacore/install/ -DWINCC_ROOT_DIR=/wincc/ ../.. + - make + + artifacts: + paths: + - build/ + + +build_LCU: stage: build + image: ci_lcu:latest script: - - echo "Building TriggerServices..." - - mkdir -p build/gnucxx11_opt - - cd build/gnucxx11_opt - - cmake -DBUILD_PACKAGES=TriggerServices ../.. + - PACKAGE=LCU + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE ../.. - make artifacts: paths: - build/ -test: + +build_LTA: + stage: build + image: ci_lta:latest + script: + - PACKAGE=LTA + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE ../.. + - make + + artifacts: + paths: + - build/ + + +build_MAC: + stage: build + image: ci_mac:latest + script: + - PACKAGE=MAC + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE ../.. + - make + + artifacts: + paths: + - build/ + + +build_QA: + stage: build + image: ci_qa:latest + script: + - PACKAGE=QA + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE ../.. + - make + + artifacts: + paths: + - build/ + + +build_RTCP: + stage: build + image: ci_rtcp:latest + script: + - PACKAGE=SAS + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/casacore/install/ -DDAL_ROOT_DIR=/DAL/install/ -DUSE_MPI=True -DUSE_OPENMP=True ../.. + - make + + artifacts: + paths: + - build/ + + +build_SAS: + stage: build + image: ci_sas:latest + script: + - PACKAGE=SAS + - VARIANT=gnucxx11_opt + - echo "Building $PACKAGE..." + - mkdir -p build/$VARIANT + - cd build/$VARIANT + - cmake -DBUILD_PACKAGES=$PACKAGE ../.. + - make + + artifacts: + paths: + - build/ + + +# (part of SAS): +# build_TriggerServices: +# stage: build +# image: ci_triggerservices:latest +# script: +# - PACKAGE=TriggerServices +# - VARIANT=gnucxx11_opt +# - echo "Building $PACKAGE..." +# - mkdir -p build/$VARIANT +# - cd build/$VARIANT +# - cmake -DBUILD_PACKAGES=$PACKAGE ../.. +# - make +# +# artifacts: +# paths: +# - build/ + + + +# +# TEST STAGE +# +# TODO: I guess we have to run cmake before each of these so ctest tests the correct package... + + +test_CEP: + stage: test + image: ci_cep:latest + script: + - PACKAGE=CEP + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_Docker: + stage: test + image: ci_docker:latest + script: + - PACKAGE=Docker + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_LCS: + stage: test + image: ci_lcs:latest + script: + - PACKAGE=LCS + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_LCU: + stage: test + image: ci_lcu:latest + script: + - PACKAGE=LCU + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_LTA: + stage: test + image: ci_lta:latest + script: + - PACKAGE=LTA + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_MAC: + stage: test + image: ci_mac:latest + script: + - PACKAGE=MAC + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_QA: + stage: test + image: ci_qa:latest + script: + - PACKAGE=QA + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_RTCP: + stage: test + image: ci_rtcp:latest + script: + - PACKAGE=RTCP + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT + - ctest + + +test_SAS: stage: test + image: ci_sas:latest script: - - echo "Testing TriggerServices..." - - cd build/gnucxx11_opt + - PACKAGE=SAS + - VARIANT=gnucxx11_opt + - echo "Testing $PACKAGE..." + - cd build/$VARIANT - ctest \ No newline at end of file diff --git a/Docker/lofar-ci/Dockerfile_ci_base b/Docker/lofar-ci/Dockerfile_ci_base new file mode 100644 index 0000000000000000000000000000000000000000..90c2c4e624ad7b29e974f3c53ca2b54c98f99a96 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_base @@ -0,0 +1,12 @@ +# +# This base image is just some beasic dev tools on top of CentOS 7 +# +# base +# +FROM centos:7 + +RUN yum -y groupinstall 'Development Tools' && \ + yum -y install epel-release && \ + yum -y install cmake log4cplus-devel python3 python3-devel python3-pip + + diff --git a/Docker/lofar-ci/Dockerfile_ci_cep b/Docker/lofar-ci/Dockerfile_ci_cep new file mode 100644 index 0000000000000000000000000000000000000000..f1eef91f76cdedccee5cf95f39b48c3210479506 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_cep @@ -0,0 +1,26 @@ +# +# This builds an image with all dependencies for the CEP package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for CEP..." && \ + yum -y install boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel + pip3 install numpy + +RUN echo "Installing Casacore..." && \ + git clone https://github.com/casacore/casacore && \ + mkdir /casacore/build/ && \ + cd /casacore/build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \ + make && \ + make install + +RUN echo "Installing Casarest..." && \ + cd / && git clone https://github.com/casacore/casarest.git && \ + mkdir -p /casarest/build/ && \ + cd /casarest/build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=../install -DCASACORE_ROOT_DIR=/casacore/install -DCMAKE_BUILD_TYPE=Release .. && \ + make && \ + make install \ No newline at end of file diff --git a/Docker/lofar-ci/Dockerfile_ci_docker b/Docker/lofar-ci/Dockerfile_ci_docker new file mode 100644 index 0000000000000000000000000000000000000000..cdc49787e22b26ed4e7b427da3edcef3044dd3b4 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_docker @@ -0,0 +1,9 @@ +# +# This builds an image with all dependencies for the QA package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for Docker..." && \ + yum -y install boost-python36-devel readline-devel \ No newline at end of file diff --git a/Docker/lofar-ci/Dockerfile_ci_lcs b/Docker/lofar-ci/Dockerfile_ci_lcs new file mode 100644 index 0000000000000000000000000000000000000000..8ce4f417cf91cc664af433e3668f9f880ba135cc --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_lcs @@ -0,0 +1,18 @@ +# +# This builds an image with all dependencies for the LCS package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for LCS..." && \ + yum -y install boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools unittest-cpp-devel readline-devel && \ + pip3 install numpy kombu requests + +RUN echo "Installing Casacore..." && \ + git clone https://github.com/casacore/casacore && \ + mkdir /casacore/build/ && \ + cd /casacore/build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \ + make && \ + make install \ No newline at end of file diff --git a/Docker/lofar-ci/Dockerfile_ci_lcu b/Docker/lofar-ci/Dockerfile_ci_lcu new file mode 100644 index 0000000000000000000000000000000000000000..9d4d56e32602c0a24c5fb7b69395b0dfce8ca352 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_lcu @@ -0,0 +1,10 @@ +# +# This builds an image with all dependencies for the LCU package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for LCU..." && \ + yum -y install postgresql-devel && \ + pip3 install psycopg2 testing.postgresql lxml mock numpy diff --git a/Docker/lofar-ci/Dockerfile_ci_lta b/Docker/lofar-ci/Dockerfile_ci_lta new file mode 100644 index 0000000000000000000000000000000000000000..1efc46400bf6d0a6c722c088bda7a8a687353047 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_lta @@ -0,0 +1,10 @@ +# +# This builds an image with all dependencies for the LTA package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for LTA..." && \ + yum -y install postgresql-devel && \ + pip3 install kombu requests pysimplesoap mysql-connector psycopg2 flask diff --git a/Docker/lofar-ci/Dockerfile_ci_mac b/Docker/lofar-ci/Dockerfile_ci_mac new file mode 100644 index 0000000000000000000000000000000000000000..5a3836b4c95c22d2b9d711907b7159c866d1a84a --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_mac @@ -0,0 +1,18 @@ +# +# This builds an image with all dependencies for the LCU package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for LCU..." && \ + yum -y install readline-devel boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel autogen && \ + pip3 install psycopg2 testing.postgresql lxml mock numpy + +RUN echo "Installing Casacore..." && \ + git clone https://github.com/casacore/casacore && \ + mkdir /casacore/build/ && \ + cd /casacore/build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \ + make && \ + make install \ No newline at end of file diff --git a/Docker/lofar-ci/Dockerfile_ci_qa b/Docker/lofar-ci/Dockerfile_ci_qa new file mode 100644 index 0000000000000000000000000000000000000000..6b6e0e2494943775e51a594b95906afcac5d0e22 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_qa @@ -0,0 +1,10 @@ +# +# This builds an image with all dependencies for the QA package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for QA..." && \ + yum -y install boost-python36-devel readline-devel postgresql-devel && \ + pip3 install kombu requests psycopg2 testing.postgresql \ No newline at end of file diff --git a/Docker/lofar-ci/Dockerfile_ci_rtcp b/Docker/lofar-ci/Dockerfile_ci_rtcp new file mode 100644 index 0000000000000000000000000000000000000000..dcbd5b2a893ba4095b9323c8279849ef2505e947 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_rtcp @@ -0,0 +1,30 @@ +# +# This builds an image with all dependencies for the RTCP package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for RTCP..." && \ + yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo && \ + yum -y install autogen boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel ncurses-devel readline-devel fftw-devel libpqxx-devel unittest-cpp-devel numactl-devel openmpi-devel cuda && \ + pip3 install numpy + +RUN echo "Installing Casacore..." && \ + git clone https://github.com/casacore/casacore && \ + mkdir /casacore/build/ && \ + cd /casacore/build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=../install -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \ + make && \ + make install + +RUN echo "Installing DAL..." && \ + git clone https://github.com/nextgen-astrodata/DAL.git && \ + mkdir /DAL/build && \ + cd /DAL/build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=../install -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 -DPYTHON_LIBRARY:FILEPATH=/usr/lib64/libpython3.6m.so -DPYTHON_INCLUDE_DIR=/usr/include/python3.6m/ .. && \ + make && \ + make install + + + diff --git a/Docker/lofar-ci/Dockerfile_ci_sas b/Docker/lofar-ci/Dockerfile_ci_sas new file mode 100644 index 0000000000000000000000000000000000000000..f4b70027c4b5d250480d75a7bb69d60a19850f76 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_sas @@ -0,0 +1,10 @@ +# +# This builds an image with all dependencies for the SAS package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for SAS..." && \ + yum -y install postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools libpqxx-devel java-devel qt-devel autogen boost-python36-devel && \ + pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql diff --git a/Docker/lofar-ci/Dockerfile_ci_triggerservices b/Docker/lofar-ci/Dockerfile_ci_triggerservices new file mode 100644 index 0000000000000000000000000000000000000000..88a3a0e3ba4c039248739b92cc1fce26f5800a16 --- /dev/null +++ b/Docker/lofar-ci/Dockerfile_ci_triggerservices @@ -0,0 +1,10 @@ +# +# This builds an image with all dependencies for the TriggerServices package based on the ci_base image +# +# base +# +FROM ci_base:latest + +RUN echo "Installing packages for TriggerServices..." && \ + yum -y install postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools && \ + pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql