From 271cd222a1b9d4e91cf57b2002c88536d87f478c Mon Sep 17 00:00:00 2001
From: lukken <lukken@astron.nl>
Date: Tue, 27 Sep 2022 13:56:04 +0000
Subject: [PATCH] L2SS-951: Prevent excessive memory pressure in
 run_integration_test

---
 docker-compose/integration-test.yml            |  2 +-
 sbin/run_integration_test.sh                   | 15 +++++++++++++--
 .../integration_test/README.md                 | 18 ++++++++++++++++--
 .../integration_test/dummy/__init__.py         |  0
 .../integration_test/dummy/test_dummy.py       | 15 +++++++++++++++
 5 files changed, 45 insertions(+), 5 deletions(-)
 create mode 100644 tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py
 create mode 100644 tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py

diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml
index 6e7e5407d..1a856078d 100644
--- a/docker-compose/integration-test.yml
+++ b/docker-compose/integration-test.yml
@@ -30,7 +30,7 @@ services:
       - --timeout=30
       - --strict
       - --
-      - tox --recreate -e integration
+      - tox -e integration
     command:
 #     Allow for arguments to be passed that wil be put after the entrypoint
 #     tox is configured to take these arguments as integration test directory
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index 3047557a3..50e488d50 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -32,7 +32,7 @@ function integration_test {
     echo "make restart ${restarts[@]} ..."
     make restart "${restarts[@]}"
   fi
-  sleep 5
+  sleep 10
   echo "make integration ${1} ..."
   make integration "${1}"
 }
@@ -64,6 +64,8 @@ fi
 
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 
+
+
 # Start the database server first
 make build databaseds dsconfig
 make start databaseds dsconfig
@@ -90,8 +92,17 @@ make build archiver-timescale hdbppts-cm hdbppts-es
 # shellcheck disable=SC2086
 make stop $DEVICES $SIMULATORS hdbppts-es hdbppts-cm archiver-timescale
 make stop device-docker # this one does not test well in docker-in-docker
+make stop elk
+
+# Run dummy integration test to install pytango in tox virtualenv without
+# the memory pressure of the ELK stack.
+integration_test dummy
+
 make start elk
 
+# Give elk time to start
+sleep 10
+
 # Update the dsconfig
 # Do not remove `bash`, otherwise statement ignored by gitlab ci shell!
 bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/LOFAR_ConfigDb.json
@@ -114,7 +125,7 @@ make start archiver-timescale
 
 # Give devices time to restart
 # TODO(Corne Lukken): Use a nicer more reliable mechanism
-sleep 60
+sleep 70
 
 # Give archiver-timescale time to start
 # shellcheck disable=SC2016
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/README.md b/tangostationcontrol/tangostationcontrol/integration_test/README.md
index d06aa9b50..6c2321ad6 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/README.md
+++ b/tangostationcontrol/tangostationcontrol/integration_test/README.md
@@ -3,6 +3,13 @@
 Integration tests are separated into multi modules. Each module requires a
 different state and configuration. These configurations are managed externally.
 
+To minimize runtime overhead and memory pressure it is best to execute the
+dummy integration test module before any other container is created. This will
+ensure  pytango is installed in the `.tox/integration` virtual environment
+before the system experiences the severe memory pressure of the ELK stack.
+
+Simply run `make integration dummy` from inside the `docker-compose` folder.
+
 In total the orchestration of integration tests is handled through four separate
 layers, each one calling the next:
 
@@ -17,7 +24,7 @@ value can be left empty for the `default` module:
 
 Individual tests can be invoked using arguments:
 
-`TEST_MODULE=default tox -e integration`
+`TEST_MODULE=default tox -e integration import.path.class.functionname`
 
 These arguments and modules can also be passed at the level of the Makefile
 instead of through tox directly:
@@ -48,7 +55,7 @@ cd tangostationcontrol
 # Single test to significantly reduce runtime
 tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
 source .tox/integration/bin/activate
-# Add import pdb; pdb.set_trace() somehwere
+# Add import pdb; pdb.set_trace() somewhere
 nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py
 python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
 ```
@@ -71,3 +78,10 @@ are running and are in the required state.
 ```shell
 sbin/run_integration_test.sh
 ```
+
+## Cleanup, recovery or starting over
+
+All docker content including images, containers, networks and volumes can
+be deleted using the following:
+
+`docker stop $(docker ps | tail -n+2 | awk '{NF=1}1' | awk '{printf("%s ",$0)} END { printf "\n" }'); docker system prune --all; docker volume prune`
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py b/tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py b/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py
new file mode 100644
index 000000000..61956173c
--- /dev/null
+++ b/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the LOFAR 2.0 Station Software
+#
+#
+#
+# Distributed under the terms of the APACHE license.
+# See LICENSE.txt for more info.
+
+from tangostationcontrol.integration_test import base
+
+
+class TestDummy(base.IntegrationTestCase):
+    def test_dummy(self):
+        self.assertTrue(True)
-- 
GitLab