L2SS-451: Ignore submodule files in pipeline

Attempt to notify gitlab of dependencies

Depends-On: adae3f2d

.gitlab-ci.yml

@@ -2,6 +2,7 @@
 #              images is in place.
 image: artefact.skao.int/ska-tango-images-tango-itango:9.3.5
 variables:
+  GIT_SUBMODULE_STRATEGY: recursive
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   # The PBR dependency requires a set version, not actually used
   # Instead `util/lofar_git.py:get_version()` is used.
@@ -16,11 +17,12 @@ stages:
   - unit-tests
   - integration-tests
 newline_at_eof:
-      stage: linting
-      before_script:
-        - pip3 install -r devices/test-requirements.txt
-      script:
-        - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude .tox,.egg-info,docker
+  stage: linting
+  before_script:
+    - pip3 install -r devices/test-requirements.txt
+  script:
+#     TODO(Corne): Ignore shell files in submodules more cleanly
+    - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker
 python_linting:
   stage: linting
   script:
@@ -37,6 +39,7 @@ shellcheck:
     - sudo apt-get update
     - sudo apt-get install -y shellcheck
   script:
+#     TODO(Corne): Ignore shell files in submodules
     - shellcheck **/*.sh
 unit_test:
   stage: unit-tests
@@ -45,28 +48,33 @@ unit_test:
     - sudo apt-get install -y git
   script:
     - cd devices
-    - tox --recreate -e py37
-integration_test:
+    - tox -e py37
+integration_test_docker:
   stage: integration-tests
-  allow_failure: true
+  image: docker:latest
   tags:
     - privileged
   services:
-    - name: docker:20.10.8-dind
+    - name: docker:dind
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
-# Everything below does not work currently, we need a privileged container
-# that can run the dind service
   before_script:
-    - sudo apt update
-    - sudo apt install -y docker.io
-    - export USER=$(id | awk -F'=' '{print $2}' | awk -F'(' '{print $2}' | awk -F')' '{print $1}')
-    - echo $USER
-#    - sudo usermod -aG docker $USER
-    - sudo docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - apk add --update make bash docker-compose
+    - apk add --update bind-tools
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
   script:
-    - touch /home/$USER/.Xauthority
-    - source bootstrap/etc/lofar20rc.sh
-    - export HOSTNAME=$(cat /run/systemd/netif/leases/2 | grep ^ADDRESS= | awk -F'=' '{print $2}')
-    - echo $HOSTNAME
-    - sudo $CI_PROJECT_DIR/sbin/run_integration_test.sh
+    - touch /root/.Xauthority
+#    Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
+    - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh
+#    Hack HOSTNAME env variable into host.docker.internal, set in docker-compose
+    - export HOSTNAME=host.docker.internal
+#    - export HOSTNAME=$(hostname -i)
+#    - export HOSTNAME=$(cat /run/systemd/netif/leases/2 | grep ^ADDRESS= | awk -F'=' '{print $2}')
+#    source the lofarrc file and mask its non zero exit code
+    - . bootstrap/etc/lofar20rc.sh || true
+#    TANGO_HOST must be unset our databaseds will be unreachable
+    - unset TANGO_HOST
+#    Allow integration test to execute
+    - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh

bin/dump_ConfigDb.sh

 #!/bin/bash
 
-# writes the JSON dump to stdout
-docker exec -it dsconfig python -m dsconfig.dump
+# writes the JSON dump to stdout, Do not change -i into -it incompatible with gitlab ci!
+docker exec -i "${CONTAINER_NAME_PREFIX}"dsconfig python -m dsconfig.

bin/itango_console.sh

 #!/bin/bash
-exec docker exec -it itango itango3
+exec docker exec -it "${CONTAINER_NAME_PREFIX}"itango itango3

bin/itango_shell.sh

 #!/bin/bash
-exec docker exec -it itango /bin/bash
+exec docker exec -it "${CONTAINER_NAME_PREFIX}"itango /bin/bash

bin/start-DS.sh

-#!/bin/bash
-function help()
-{
-    why="${1}"
-    echo -e "*** Cannot start the Python device server.\\n${why}\\n\\n* The Python file for the device server must be the 1st parameter that is provided.\\n* The instance of this device server must be the 2nd parameter that is provided."
-    exit -1
-}
-
-# Check if the mandatory parameters are present:
-# ${1}:  device server's Python file
-# ${2}:  instance of the device server's executable in the configDB
-case ${#} in
-    0)
-        help "The device server's Python file and the instance are missing."
-        ;;
-    1)
-        help "The device server's instance is missing."
-        ;;
-    *)
-        deviceServer="${1}"
-        shift
-        instance="${1}"
-        shift
-        ;;
-esac
-
-# Find the path to the device server's Python file that is
-# relative to the /hosthome directory (in Docker the user's
-# mounted ${HOME}).
-# ATTENTION
-# This is assuming that the device server's Python file exists
-# on the Docker's host in the user's ${HOME} directory.
-runThis=$(basename "${deviceServer}")
-runThis=${runThis//.sh/.py}
-if [ -f "${runThis}" ]; then
-    myDir=${PWD}
-else
-    myDir=${PWD}/$(dirname "${deviceServer}")
-fi
-deviceServerPath=${myDir/${HOME}/\/hosthome}
-
-# Tango log lines start with a UNIX timestamp. Replace them with the UTC time.
-docker exec -it itango python3 "${deviceServerPath}/${runThis}" "${instance}" "${@}" | perl -ne 'use Time::Piece; s/^([0-9]+)/gmtime($1)->strftime("%F %T")/e; print;'

bootstrap/etc/lofar20rc.sh

@@ -16,6 +16,14 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then
   alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git"
 fi
 
+if [ ! -z ${CI_BUILD_ID+x} ]; then
+    export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}-
+elif [ ! -z ${CI_JOB_ID+x} ]; then
+    export CONTAINER_NAME_PREFIX=${CI_JOB_ID}-
+else
+    unset CONTAINER_NAME_PREFIX
+fi
+
 # This needs to be modified for a development environment.
 # In case you run multiple Docker networks on the same host in parallel, you need to specify a unique
 # network name for each of them.
@@ -26,7 +34,6 @@ export NETWORK_MODE=tangonet
 # Example:  export TANGO_HOST=station-xk25.astron.nl:10000
 export TANGO_HOST=$(hostname):10000
 
-
 #
 # NO MODIFICATION BEYOND THIS POINT!
 #

devices/integration_test/devices/test_device_recv.py

@@ -42,7 +42,7 @@ class TestDeviceRECV(base.IntegrationTestCase):
 
         d = DeviceProxy("LTS/RECV/1")
 
-        d.initialise()
+        d.Initialise()
 
         self.assertEqual(DevState.STANDBY, d.state())
 
@@ -51,7 +51,7 @@ class TestDeviceRECV(base.IntegrationTestCase):
 
         d = DeviceProxy("LTS/RECV/1")
 
-        d.initialise()
+        d.Initialise()
 
         d.on()
 

devices/integration_test/devices/test_device_sdp.py

@@ -38,12 +38,19 @@ class TestDeviceSDP(base.IntegrationTestCase):
 
         self.assertEqual(DevState.OFF, d.state())
 
+    def test_device_sdp_ping(self):
+        """Test if we can successfully ping the device server"""
+
+        d = DeviceProxy("LTS/SDP/1")
+
+        self.assertGreater(d.ping(), 0)
+
     def test_device_sdp_initialize(self):
         """Test if we can transition to standby"""
 
         d = DeviceProxy("LTS/SDP/1")
 
-        d.initialise()
+        d.Initialise()
 
         self.assertEqual(DevState.STANDBY, d.state())
 
@@ -52,7 +59,7 @@ class TestDeviceSDP(base.IntegrationTestCase):
 
         d = DeviceProxy("LTS/SDP/1")
 
-        d.initialise()
+        d.Initialise()
 
         d.on()
 

devices/integration_test/devices/test_tango_database.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+import time
+
+from tango import Database
+from tango._tango import DevState
+
+from integration_test import base
+
+
+class TestTangoDatabase(base.IntegrationTestCase):
+
+    def setUp(self):
+        """Intentionally recreate the device object in each test"""
+        super(TestTangoDatabase, self).setUp()
+
+    def test_database_servers(self):
+        """Connect to the database and find at least 3 servers
+
+        One for SDP, RECV and the databaseds itself.
+        """
+
+        d = Database()
+
+        # Ensure this value is close to actual amount of servers defined by
+        # integration_ConfigDb.json
+        self.assertGreater(len(d.get_server_list()), 16)

docker-compose/archiver.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   archiver-maria-db:
     image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/mariadb_hdbpp:2021-05-28
-    container_name: archiver-maria-db
+    container_name: ${CONTAINER_NAME_PREFIX}archiver-maria-db
     networks:
       - control
     ports:
@@ -23,7 +23,7 @@ services:
       image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-archiver:2021-05-28
       networks:
       - control
-      container_name: hdbpp-es
+      container_name: ${CONTAINER_NAME_PREFIX}hdbpp-es
       depends_on:
         - databaseds
         - dsconfig
@@ -42,7 +42,7 @@ services:
       image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-archiver:${TANGO_ARCHIVER_VERSION}
       networks:
       - control
-      container_name: hdbpp-cm
+      container_name: ${CONTAINER_NAME_PREFIX}hdbpp-cm
       depends_on:
         - databaseds
         - dsconfig
@@ -58,7 +58,7 @@ services:
 
   dsconfig:
     image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-dsconfig:${TANGO_DSCONFIG_VERSION}
-    container_name: dsconfig
+    container_name: ${CONTAINER_NAME_PREFIX}dsconfig
     networks:
       - control
     depends_on:

docker-compose/device-apsct.yml

@@ -26,6 +26,8 @@ services:
       - control
     ports:
       - "5709:5709" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-apspu.yml

@@ -26,6 +26,8 @@ services:
       - control
     ports:
       - "5710:5710" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-boot.yml

@@ -25,6 +25,8 @@ services:
       - control
     ports:
       - "5708:5708" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
       - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-docker.yml

@@ -26,6 +26,8 @@ services:
       - control
     ports:
       - "5705:5705" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
       - ..:/opt/lofar/tango:rw
       - /var/run/docker.sock:/var/run/docker.sock:rw # we want to control our sibling containers, NOT do docker-in-docker (dind)

docker-compose/device-observation_control.yml

@@ -25,6 +25,8 @@ services:
       - control
     ports:
       - "5703:5703" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-recv.yml

@@ -26,6 +26,8 @@ services:
       - control
     ports:
       - "5707:5707" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-sdp.yml

@@ -26,6 +26,8 @@ services:
       - control
     ports:
       - "5701:5701" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-sst.yml

@@ -29,6 +29,8 @@ services:
         - "5001:5001/udp" # port to receive SST UDP packets on
         - "5101:5101/tcp" # port to emit SST TCP packets on
         - "5702:5702" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-unb2.yml

@@ -26,6 +26,8 @@ services:
       - control
     ports:
       - "5704:5704" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment:

docker-compose/device-xst.yml

@@ -29,6 +29,8 @@ services:
         - "5002:5002/udp" # port to receive XST UDP packets on
         - "5102:5102/tcp" # port to emit XST TCP packets on
         - "5706:5706" # unique port for this DS
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
         - ..:/opt/lofar/tango:rw
     environment: