Merge branch 'TMSS-176' into 'master'

Resolve TMSS-176 Closes TMSS-176 See merge request !157

Merge branch 'TMSS-176' into 'master'
c7bb17d7 · Jörn Künsemöller · aea01681 · 2ff9f2a1 · c7bb17d7 · c7bb17d7
Commit c7bb17d7 authored 5 years ago by Jörn Künsemöller
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 stages:
+  - prepare-base
  - prepare
  - build
  - unit_test
@@ -7,26 +8,30 @@ stages:
  - deploy
 #
-# PREPARE STAGE
+# PREPARE BASE STAGE
 #
+prepare_ci_base_docker_image:
+  stage: prepare-base
+  script:
+    - docker build -t ci_base:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_base .
+#
+# PREPARE STAGE
+#
 prepare_ci_sas_docker_image:
  stage: prepare
  script:
-    - docker build -t ci_base -f  Docker/lofar-ci/Dockerfile_ci_base .
+    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_sas:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_sas .
-    - docker build -t ci_sas -f  Docker/lofar-ci/Dockerfile_ci_sas .
 prepare_ci_lta_docker_image:
  stage: prepare
  script:
-    - docker build -t ci_base -f  Docker/lofar-ci/Dockerfile_ci_base .
+    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lta:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_lta .
-    - docker build -t ci_lta -f  Docker/lofar-ci/Dockerfile_ci_lta .
 prepare_ci_mac_docker_image:
  stage: prepare
  script:
-    - docker build -t ci_base -f  Docker/lofar-ci/Dockerfile_ci_base .
+    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_mac:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_mac .
-    - docker build -t ci_mac -f  Docker/lofar-ci/Dockerfile_ci_mac .
 #
 # BUILD STAGE
@@ -34,7 +39,7 @@ prepare_ci_mac_docker_image:
 build_TMSS:
  stage: build
-  image: ci_sas:latest
+  image: ci_sas:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=TMSS
    - echo "Building $PACKAGE..."
@@ -52,7 +57,7 @@ build_TMSS:
 build_RAServices:
  stage: build
-  image: ci_sas:latest
+  image: ci_sas:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=RAServices
    - echo "Building $PACKAGE..."
@@ -70,7 +75,7 @@ build_RAServices:
 build_LTAIngest:
  stage: build
-  image: ci_lta:latest
+  image: ci_lta:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=LTAIngest
    - echo "Building $PACKAGE..."
@@ -88,15 +93,19 @@ build_LTAIngest:
 build_MCU_MAC:
  stage: build
-  image: ci_mac:latest
+  image: ci_mac:$CI_COMMIT_SHORT_SHA
  script:
-    - PACKAGE=MainCU
+    - PACKAGE=MCU_MAC
    - echo "Building $PACKAGE..."
    - mkdir -p build/gnucxx11_opt
    - cd build/gnucxx11_opt
-    - cmake -DBUILD_PACKAGES=$PACKAGE -DUSE_LOG4CPLUS=false -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.14/ -DBLITZ_ROOT_DIR=/opt/blitz/ ../..
+    - cmake -DBUILD_PACKAGES=$PACKAGE -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
    - make -j 12
-    - make install
+    - make DESTDIR=${CI_BUILDS_DIR}/install install
+    - cd ${CI_BUILDS_DIR}/install/opt/lofar
+    - tar --ignore-failed-read --exclude=include -czf MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
+    - curl --insecure --upload-file MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar -u upload:upload https://support.astron.nl/nexus/content/repositories/branches/nl/astron/lofar/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.x86_64.ztar
  dependencies:
    - prepare_ci_mac_docker_image
  artifacts:
@@ -110,7 +119,7 @@ build_MCU_MAC:
 unit_test_TMSS:
  stage: unit_test
-  image: ci_sas:latest
+  image: ci_sas:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=TMSS
    - echo "Testing $PACKAGE..."
@@ -126,7 +135,7 @@ unit_test_TMSS:
 unit_test_RAServices:
  stage: unit_test
-  image: ci_sas:latest
+  image: ci_sas:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=RAServices
    - echo "Testing $PACKAGE..."
@@ -148,7 +157,7 @@ unit_test_RAServices:
 unit_test_LTAIngest:
  stage: unit_test
-  image: ci_lta:latest
+  image: ci_lta:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=LTAIngest
    - echo "Testing $PACKAGE..."
@@ -170,9 +179,10 @@ unit_test_LTAIngest:
 unit_test_MCU_MAC:
  stage: unit_test
-  image: ci_mac:latest
+  image: ci_mac:$CI_COMMIT_SHORT_SHA
  script:
-    - PACKAGE=MainCu
+    - echo "Starting Qpid server..." && qpidd &
+    - PACKAGE=MCU_MAC
    - echo "Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_INTEGRATION_TESTS=true ctest
@@ -200,7 +210,7 @@ dockerize_TMSS:
  script:
    - cd build/gnucxx11_opt
    - ls *
-    - docker build -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss .
+    - docker build --build-arg SAS_VERSION=$CI_COMMIT_SHORT_SHA -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss .
    - cd ../..
    - cd SAS/TMSS/test/oidc/docker-test-mozilla-django-oidc
    - docker build -t tmss_testprovider:$CI_COMMIT_SHORT_SHA -f dockerfiles/oidc_testprovider .
@@ -222,7 +232,7 @@ dockerize_TMSS:
 integration_test_TMSS:
  stage: integration_test
-  image: ci_sas:latest
+  image: ci_sas:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=TMSS
    - echo "Integration Testing $PACKAGE..."
@@ -244,7 +254,7 @@ integration_test_TMSS:
 integration_test_RAServices:
  stage: integration_test
-  image: ci_sas:latest
+  image: ci_sas:$CI_COMMIT_SHORT_SHA
  services:
    - rabbitmq:latest
  variables:
@@ -266,7 +276,7 @@ integration_test_RAServices:
 integration_test_LTAIngest:
  stage: integration_test
-  image: ci_lta:latest
+  image: ci_lta:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=LTAIngest
    - echo "Integration Testing $PACKAGE..."
@@ -338,3 +348,19 @@ deploy-tmss-ua:
  when: manual
  only:
    - "master"
+deploy-MCU_MAC-test:
+  stage: deploy
+  before_script:
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan mcu199.control.lofar >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - ssh lofarsys@mcu199.control.lofar "MAC_install -b ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA} -v ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  dependencies:
+    - unit_test_MCU_MAC
+  when: manual
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_lta
+++ b/Docker/lofar-ci/Dockerfile_ci_lta
@@ -3,7 +3,8 @@
 #
 # base
 #
-FROM ci_base:latest
+ARG BASE_VERSION=latest
+FROM ci_base:$BASE_VERSION
 RUN echo "Installing packages for LTA..."

--- a/Docker/lofar-ci/Dockerfile_ci_mac
+++ b/Docker/lofar-ci/Dockerfile_ci_mac
@@ -3,21 +3,39 @@
 #
 # base
 #
-FROM ci_base:latest
+# !!! Note: Since Docker apparently does not allow Dockerfile-relative paths in a COPY and Gitlab CI works relative
+# !!!       to the repo root, this image has to be build with that context, e.g.:
+# !!!       docker build -f Docker/lofar-ci/Dockerfile_ci_mac -t ci_mac:latest .
+#
+ARG BASE_VERSION=latest
+FROM ci_base:$BASE_VERSION
 RUN echo "Installing packages for MAC..." && \
    yum -y install readline-devel boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel autogen postgresql-devel cmake3 libpqxx-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools unittest-cpp-devel jsoncpp-devel jsoncpp libcurl-devel libcurl && \
    pip3 install psycopg2 testing.postgresql lxml mock numpy kombu requests python-dateutil fabric
+RUN echo "Installing WinCC3.14 build and Demo App from Nexus repo..." && \
+    cd /tmp && \
+    wget https://support.astron.nl/nexus/content/repositories/snapshots/nl/astron/lofar/wincc/3_16/WinCC_OA_3.16-base-rhel-0-14.x86_64.rpm && \
+    wget https://support.astron.nl/nexus/content/repositories/snapshots/nl/astron/lofar/wincc/3_16/WinCC_OA_3.16-specialfunctions-rhel-0-14.x86_64.rpm && \
+    wget https://support.astron.nl/nexus/content/repositories/snapshots/nl/astron/lofar/wincc/3_16/WinCC_OA_3.16-applications-rhel-0-14.x86_64.rpm && \
+    wget https://support.astron.nl/nexus/content/repositories/snapshots/nl/astron/lofar/wincc/3_16/WinCC_OA_3.16-api-rhel-0-14.x86_64.rpm && \
+    yum install -y WinCC_OA_3.16-base-rhel-0-14.x86_64.rpm && \
+    yum install -y WinCC_OA_3.16-specialfunctions-rhel-0-14.x86_64.rpm && \
+    yum install -y WinCC_OA_3.16-applications-rhel-0-14.x86_64.rpm && \
+    yum install -y WinCC_OA_3.16-api-rhel-0-14.x86_64.rpm
+COPY Docker/lofar-ci/pvssInst.conf /etc/opt/pvss/pvssInst.conf
 USER lofarsys
-#RUN echo "Installing Casacore..." && \
+RUN echo "Installing Casacore..." && \
-#    git clone https://github.com/casacore/casacore && \
+    mkdir -p /opt/3rdparty_sources/ && cd /opt/3rdparty_sources/ && \
-#    mkdir /casacore/build/ && \
+    git clone --branch v3.2.0 https://github.com/casacore/casacore.git casacore && \
-#    cd /casacore/build/ && \
+    mkdir casacore/build/ && cd casacore/build/ && \
-#    cmake -DCMAKE_INSTALL_PREFIX=/opt/casacore -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
+    cmake -DCMAKE_INSTALL_PREFIX=/opt/casacore -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
-#    make -j 8 && \
+    make -j 8 && \
-#    make install
+    make install
 RUN echo "Installing Blitz++" && \
    mkdir -p /opt/3rdparty_sources/ && cd /opt/3rdparty_sources/ && \
@@ -27,11 +45,7 @@ RUN echo "Installing Blitz++" && \
    make -j 8 lib && \
    make install
-RUN echo "Installing WinCC3.14 from nexus ALTA repo..." && \
+ENV LD_LIBRARY_PATH /opt/blitz/lib64/:$LD_LIBRARY_PATH
-    cd /tmp && \
-    wget https://support.astron.nl/nexus/content/repositories/snapshots/nl/alta/buildWinCC314api.tar.gz && \
-    tar -xvf buildWinCC314api.tar.gz && \
-    cd opt && \
-    mv WinCC_OA /opt/
-ENV LD_LIBRARY_PATH /opt/WinCC_OA/3.14/bin:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /opt/WinCC_OA/3.16/bin:$LD_LIBRARY_PATH
+ENV PATH /opt/WinCC_OA/3.14/bin/:$PATH
\ No newline at end of file
--- a/Docker/lofar-ci/Dockerfile_ci_sas
+++ b/Docker/lofar-ci/Dockerfile_ci_sas
@@ -3,7 +3,8 @@
 #
 # base
 #
-FROM ci_base:latest
+ARG BASE_VERSION=latest
+FROM ci_base:$BASE_VERSION
 RUN echo "Installing packages for SAS..." && \
    yum install -y log4cplus log4cplus-devel python3 python3-libs python3-devel boost readline-devel boost-devel binutils-devel boost-python36 boost-python36-devel gettext which openldap-devel npm nodejs git java-11-openjdk python-twisted-core

--- a/Docker/lofar-ci/pvssInst.conf
+++ b/Docker/lofar-ci/pvssInst.conf
+[Software\ETM\PVSS II\3.16]
+currentProject = "DemoApplication_3.16"
+InstallationDir = "/opt/WinCC_OA/3.16"
+[Software\ETM\PVSS II\Configs\AdvS7base_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/AdvS7base_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/AdvS7base_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/AdvS7base_3.16"
+[Software\ETM\PVSS II\Configs\AdvS7_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/AdvS7_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/AdvS7_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/AdvS7_3.16"
+[Software\ETM\PVSS II\Configs\BACnet_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/BACnet_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/BACnet_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/BACnet_3.16"
+[Software\ETM\PVSS II\Configs\BasicS7base_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/BasicS7base_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/BasicS7base_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/BasicS7base_3.16"
+[Software\ETM\PVSS II\Configs\BasicS7_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/BasicS7_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/BasicS7_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/BasicS7_3.16"
+[Software\ETM\PVSS II\Configs\CommCenter_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/CommCenter_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/CommCenter_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/CommCenter_3.16"
+[Software\ETM\PVSS II\Configs\DBLogger_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/DBLogger_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/DBLogger_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/DBLogger_3.16"
+[Software\ETM\PVSS II\Configs\Redundancy2x2_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/Redundancy2x2_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/Redundancy2x2_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/Redundancy2x2_3.16"
+[Software\ETM\PVSS II\Configs\Stdlib_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/Stdlib_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/Stdlib_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/Stdlib_3.16"
+[Software\ETM\PVSS II\Configs\MobileAppServer_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/MobileAppServer_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/MobileAppServer_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/MobileAppServer_3.16"
+[Software\ETM\PVSS II\Configs\SmartSCADA_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/SmartSCADA_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/SmartSCADA_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/SmartSCADA_3.16"
+[Software\ETM\PVSS II\Configs\TestFramework_3.16]
+notRunnable = "1"
+InstallationDate = "2020.03.11 09:56:57.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/TestFramework_3.16"
+pvss_path = ""
+PVSS_II = "/opt/WinCC_OA/3.16/TestFramework_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/TestFramework_3.16"
+[Software\ETM\PVSS II\Configs\DemoApplication_3.16]
+notRunnable = "0"
+InstallationDate = "2020.03.11 09:57:30.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/DemoApplication_3.16"
+pvss_path = "/opt/WinCC_OA/3.16"
+PVSS_II = "/opt/WinCC_OA/3.16/DemoApplication_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/DemoApplication_3.16"
+[Software\ETM\PVSS II\Configs\GettingStarted_3.16]
+notRunnable = "0"
+InstallationDate = "2020.03.11 09:57:30.000"
+InstallationUser = "root"
+InstallationVersion = "3.16"
+proj_path = "/opt/WinCC_OA/3.16/GettingStarted_3.16"
+pvss_path = "/opt/WinCC_OA/3.16"
+PVSS_II = "/opt/WinCC_OA/3.16/GettingStarted_3.16/config/config"
+InstallationDir = "/opt/WinCC_OA/3.16/GettingStarted_3.16"
\ No newline at end of file
--- a/SAS/TMSS/src/Dockerfile-tmss
+++ b/SAS/TMSS/src/Dockerfile-tmss
@@ -7,7 +7,8 @@
 #
 # docker build [-t image_name:tag] -f docker/Dockerfile-tmss .
 #
-FROM ci_sas:latest
+ARG SAS_VERSION=latest
+FROM ci_sas:$SAS_VERSION
 USER lofarsys
 RUN mkdir -p /opt/lofar