From 271cd222a1b9d4e91cf57b2002c88536d87f478c Mon Sep 17 00:00:00 2001 From: lukken <lukken@astron.nl> Date: Tue, 27 Sep 2022 13:56:04 +0000 Subject: [PATCH] L2SS-951: Prevent excessive memory pressure in run_integration_test --- docker-compose/integration-test.yml | 2 +- sbin/run_integration_test.sh | 15 +++++++++++++-- .../integration_test/README.md | 18 ++++++++++++++++-- .../integration_test/dummy/__init__.py | 0 .../integration_test/dummy/test_dummy.py | 15 +++++++++++++++ 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py create mode 100644 tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index 6e7e5407d..1a856078d 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -30,7 +30,7 @@ services: - --timeout=30 - --strict - -- - - tox --recreate -e integration + - tox -e integration command: # Allow for arguments to be passed that wil be put after the entrypoint # tox is configured to take these arguments as integration test directory diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 3047557a3..50e488d50 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -32,7 +32,7 @@ function integration_test { echo "make restart ${restarts[@]} ..." make restart "${restarts[@]}" fi - sleep 5 + sleep 10 echo "make integration ${1} ..." make integration "${1}" } @@ -64,6 +64,8 @@ fi cd "$LOFAR20_DIR/docker-compose" || exit 1 + + # Start the database server first make build databaseds dsconfig make start databaseds dsconfig @@ -90,8 +92,17 @@ make build archiver-timescale hdbppts-cm hdbppts-es # shellcheck disable=SC2086 make stop $DEVICES $SIMULATORS hdbppts-es hdbppts-cm archiver-timescale make stop device-docker # this one does not test well in docker-in-docker +make stop elk + +# Run dummy integration test to install pytango in tox virtualenv without +# the memory pressure of the ELK stack. +integration_test dummy + make start elk +# Give elk time to start +sleep 10 + # Update the dsconfig # Do not remove `bash`, otherwise statement ignored by gitlab ci shell! bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/LOFAR_ConfigDb.json @@ -114,7 +125,7 @@ make start archiver-timescale # Give devices time to restart # TODO(Corne Lukken): Use a nicer more reliable mechanism -sleep 60 +sleep 70 # Give archiver-timescale time to start # shellcheck disable=SC2016 diff --git a/tangostationcontrol/tangostationcontrol/integration_test/README.md b/tangostationcontrol/tangostationcontrol/integration_test/README.md index d06aa9b50..6c2321ad6 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/README.md +++ b/tangostationcontrol/tangostationcontrol/integration_test/README.md @@ -3,6 +3,13 @@ Integration tests are separated into multi modules. Each module requires a different state and configuration. These configurations are managed externally. +To minimize runtime overhead and memory pressure it is best to execute the +dummy integration test module before any other container is created. This will +ensure pytango is installed in the `.tox/integration` virtual environment +before the system experiences the severe memory pressure of the ELK stack. + +Simply run `make integration dummy` from inside the `docker-compose` folder. + In total the orchestration of integration tests is handled through four separate layers, each one calling the next: @@ -17,7 +24,7 @@ value can be left empty for the `default` module: Individual tests can be invoked using arguments: -`TEST_MODULE=default tox -e integration` +`TEST_MODULE=default tox -e integration import.path.class.functionname` These arguments and modules can also be passed at the level of the Makefile instead of through tox directly: @@ -48,7 +55,7 @@ cd tangostationcontrol # Single test to significantly reduce runtime tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith source .tox/integration/bin/activate -# Add import pdb; pdb.set_trace() somehwere +# Add import pdb; pdb.set_trace() somewhere nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith ``` @@ -71,3 +78,10 @@ are running and are in the required state. ```shell sbin/run_integration_test.sh ``` + +## Cleanup, recovery or starting over + +All docker content including images, containers, networks and volumes can +be deleted using the following: + +`docker stop $(docker ps | tail -n+2 | awk '{NF=1}1' | awk '{printf("%s ",$0)} END { printf "\n" }'); docker system prune --all; docker volume prune` diff --git a/tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py b/tangostationcontrol/tangostationcontrol/integration_test/dummy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py b/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py new file mode 100644 index 000000000..61956173c --- /dev/null +++ b/tangostationcontrol/tangostationcontrol/integration_test/dummy/test_dummy.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the LOFAR 2.0 Station Software +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +from tangostationcontrol.integration_test import base + + +class TestDummy(base.IntegrationTestCase): + def test_dummy(self): + self.assertTrue(True) -- GitLab