L2SS-951: Prevent excessive memory pressure in run_integration_test

docker-compose/integration-test.yml

@@ -30,7 +30,7 @@ services:
       - --timeout=30
       - --strict
       - --
-      - tox --recreate -e integration
+      - tox -e integration
     command:
 #     Allow for arguments to be passed that wil be put after the entrypoint
 #     tox is configured to take these arguments as integration test directory

sbin/run_integration_test.sh

@@ -32,7 +32,7 @@ function integration_test {
     echo "make restart ${restarts[@]} ..."
     make restart "${restarts[@]}"
   fi
-  sleep 5
+  sleep 10
   echo "make integration ${1} ..."
   make integration "${1}"
 }
@@ -64,6 +64,8 @@ fi
 
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 
+
+
 # Start the database server first
 make build databaseds dsconfig
 make start databaseds dsconfig
@@ -90,8 +92,17 @@ make build archiver-timescale hdbppts-cm hdbppts-es
 # shellcheck disable=SC2086
 make stop $DEVICES $SIMULATORS hdbppts-es hdbppts-cm archiver-timescale
 make stop device-docker # this one does not test well in docker-in-docker
+make stop elk
+
+# Run dummy integration test to install pytango in tox virtualenv without
+# the memory pressure of the ELK stack.
+integration_test dummy
+
 make start elk
 
+# Give elk time to start
+sleep 10
+
 # Update the dsconfig
 # Do not remove `bash`, otherwise statement ignored by gitlab ci shell!
 bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/LOFAR_ConfigDb.json
@@ -114,7 +125,7 @@ make start archiver-timescale
 
 # Give devices time to restart
 # TODO(Corne Lukken): Use a nicer more reliable mechanism
-sleep 60
+sleep 70
 
 # Give archiver-timescale time to start
 # shellcheck disable=SC2016

tangostationcontrol/tangostationcontrol/integration_test/README.md

@@ -3,6 +3,13 @@
 Integration tests are separated into multi modules. Each module requires a
 different state and configuration. These configurations are managed externally.
 
+To minimize runtime overhead and memory pressure it is best to execute the
+dummy integration test module before any other container is created. This will
+ensure  pytango is installed in the `.tox/integration` virtual environment
+before the system experiences the severe memory pressure of the ELK stack.
+
+Simply run `make integration dummy` from inside the `docker-compose` folder.
+
 In total the orchestration of integration tests is handled through four separate
 layers, each one calling the next:
 
@@ -17,7 +24,7 @@ value can be left empty for the `default` module:
 
 Individual tests can be invoked using arguments:
 
-`TEST_MODULE=default tox -e integration`
+`TEST_MODULE=default tox -e integration import.path.class.functionname`
 
 These arguments and modules can also be passed at the level of the Makefile
 instead of through tox directly:
@@ -48,7 +55,7 @@ cd tangostationcontrol
 # Single test to significantly reduce runtime
 tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
 source .tox/integration/bin/activate
-# Add import pdb; pdb.set_trace() somehwere
+# Add import pdb; pdb.set_trace() somewhere
 nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py
 python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
 ```
@@ -71,3 +78,10 @@ are running and are in the required state.
 ```shell
 sbin/run_integration_test.sh
 ```
+
+## Cleanup, recovery or starting over
+
+All docker content including images, containers, networks and volumes can
+be deleted using the following:
+
+`docker stop $(docker ps | tail -n+2 | awk '{NF=1}1' | awk '{printf("%s ",$0)} END { printf "\n" }'); docker system prune --all; docker volume prune`

tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+from tangostationcontrol.integration_test import base
+
+
+class TestDummy(base.IntegrationTestCase):
+    def test_dummy(self):
+        self.assertTrue(True)