diff --git a/docker-compose/.env b/docker-compose/.env index 53937727c24d398a5d82b24c31f205db50064163..eb952e41424380f649d797ec6f2fa6c8879f961c 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -19,3 +19,5 @@ PG_SUPERUSER_PASSWORD=password PG_HDB_PASSWORD=hdbpp MYSQL_ROOT_PASSWORD=secret MYSQL_PASSWORD=tango + +TEST_MODULE=default diff --git a/docker-compose/Makefile b/docker-compose/Makefile index 6b989f8a12a6ad02c780082e50e55ef1e279b864..b64d7ff70b917574e68fc574e205b82d43007f11 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -50,6 +50,8 @@ else ifeq (attach,$(firstword $(MAKECMDGOALS))) endif else ifeq (run,$(firstword $(MAKECMDGOALS))) RUN_TARGET = true +else ifeq (integration,$(firstword $(MAKECMDGOALS))) + INTEGRATION_TARGET = true endif ifdef SERVICE_TARGET @@ -64,6 +66,12 @@ else ifdef RUN_TARGET $(eval $(SERVICE):;@:) SERVICE_ARGS := $(wordlist 3, $(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) $(eval $(SERVICE_ARGS):;@:) +else ifdef INTEGRATION_TARGET + # Isolate second argument as integration module, the rest as arguments + INTEGRATION_MODULE := $(wordlist 2, 2, $(MAKECMDGOALS)) + $(eval $(INTEGRATION_MODULE):;@:) + INTEGRATION_ARGS := $(wordlist 3, $(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) + $(eval $(INTEGRATION_ARGS):;@:) endif # @@ -142,7 +150,7 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \ DOCKER_GID=$(DOCKER_GID) -.PHONY: up down minimal run start stop restart build build-nocache status clean pull help +.PHONY: up down minimal run integration start stop restart build build-nocache status clean pull help .DEFAULT_GOAL := help pull: ## pull the images from the Docker hub @@ -162,6 +170,9 @@ up: minimal ## start the base TANGO system and prepare requested services run: minimal ## run a service using arguments and delete it afterwards $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm $(SERVICE) $(SERVICE_ARGS) +integration: minimal ## run a service using arguments and delete it afterwards + TEST_MODULE=$(INTEGRATION_MODULE) $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm integration-test $(INTEGRATION_ARGS) + down: ## stop all services and tear down the system $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down ifneq ($(NETWORK_MODE),host) diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index e59ca9c92b8b3d1e20bef63b19f0ee2a0a969025..944f47a3580484430a40eaf9a0b94e57902f1df3 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -22,6 +22,7 @@ services: - ..:/opt/lofar/tango:rw environment: - TANGO_HOST=${TANGO_HOST} + - TEST_MODULE=${TEST_MODULE} working_dir: /opt/lofar/tango/tangostationcontrol entrypoint: - /usr/local/bin/wait-for-it.sh diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 1f5ab6280ed50d6d4f7c7a06cfab1b7629e48845..6d73ee8bcad618cb07d94ceab1336a9c8bbb6c4e 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -1,5 +1,54 @@ #!/bin/bash -e +# Usage function explains how parameters are parsed +function usage { + echo "./$(basename "$0") + no arguments, builds and configures all docker containers and starts each + stage of the integration test one after the other. Between each stage the + dsconfig is updated accordingly." + echo "" + echo "./$(basename "$0") -h + displays this help message" + echo "" + echo "./$(basename "$0") import-path.<class><function-name> + A specific test can be defined by specifying the entire an import path + and optionally extending this with a specific class and function. For + instance tangostationcontrol.integration_test.observations.test_archiver.TestArchiver.test_get_multimember_devices" + echo "" +} + +# Configure the config database, restart containers and run a specific +# integration module or even specific tests +# integration_test module restarted_containers config_files specific_test +function integration_test { + configs=($3) + restarts=($2) + for config in "${configs[@]}"; do + bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${config}" + done + if [ ! -z "${2+x}" ]; then + make restart ${restarts[*]} + fi + sleep 5 + make integration "${1}" +} + +# list of arguments expected in the input +optstring=":h" + +while getopts ${optstring} arg; do + case ${arg} in + h) + usage + exit 0 + ;; + ?) + echo "Invalid option: -${OPTARG}." + exit 2 + ;; + esac +done + if [ -z "$LOFAR20_DIR" ]; then # We assume we aren't in the PATH, so we can derive our path. # We need our parent directory. @@ -70,19 +119,8 @@ echo '/usr/local/bin/wait-for-it.sh archiver-timescale:5432 --strict --timeout=3 cd "$LOFAR20_DIR/docker-compose" || exit 1 make up integration-test -# Run the default integration tests -make run integration-test default - -# Configure integration test for recv_cluster module -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/integrations/recvcluster_ConfigDb.json -make restart device-recv device-tilebeam device-antennafield -sleep 5 +integration_test default -make run integration-test recv_cluster - -# Configure integration test for multi-observation devices -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/integrations/multiobs_ConfigDb.json -make restart archiver-timescale hdbppts-cm hdbppts-es -sleep 5 +integration_test recv_cluster "device-recv device-tilebeam device-antennafield" "${LOFAR20_DIR}/CDB/integrations/recvcluster_ConfigDb.json" -make run integration-test observations +integration_test observations "archiver-timescale hdbppts-cm hdbppts-es" "${LOFAR20_DIR}/CDB/integrations/multiobs_ConfigDb.json" diff --git a/tangostationcontrol/tangostationcontrol/integration_test/README.md b/tangostationcontrol/tangostationcontrol/integration_test/README.md index efae3d939d6d34c67fe4fee6485092962e4c4343..972e3a2a9074c2d2fc9994b460a615d37ccc2ca1 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/README.md +++ b/tangostationcontrol/tangostationcontrol/integration_test/README.md @@ -3,8 +3,55 @@ Integration tests are separated into multi modules. Each module requires a different state and configuration. These configurations are managed externally. -Invocation of individual modules is achieved through tox: -`tox -e integration MODULE_SUBFOLDER` +In total the orchestration of integration tests is handled through four separate +layers, each one calling the next: + +1. `sbin/run_integration_test.sh` +2. `docker-compose/Makefile` - integration target +3. `docker-compose/integration-test.yml` +4. `tangostrationcontrol/tox.ini` - integration job + +Invocation of individual modules is achieved through tox, this environment +value can be left empty for the `default` module: +`TEST_MODULE=modulefolder tox -e integration` + +Individual tests can be invoked using arguments: + +`TEST_MODULE=default tox -e integration` + +These arguments and modules can also be passed at the level of the Makefile +instead of through tox directly: + +```shell +make integration default import.path.class.functionname` +``` + +## Breakpoints & Debuggers with Integration Tests + +It can be extremely helpful to be able to mount a debugger during integration +tests. This can be achieved in two or three steps depending on the availability +of tango on the host + +1. docker exec into a container based on `lofar-device-base` +2. Navigate to the `tangostationcontrol` directory +3. Execute `tox -e integration` to generate the virtual environment +4. Activate the virtual environment `source .tox/integration/bin/activate` +5. Modify the source files and add `import pdb; pdb.set_trace()` where you want + breakpoints. +6. Execute the desired test using testtools `python -m testtools.run imort.class.path.functionname` + +A concrete example is shown below: + +```shell +docker exec -it device-sdp /bin/bash +cd tangostationcontrol +# Single test to significantly reduce runtime +tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith +source .tox/integration/bin/activate +# Add import pdb; pdb.set_trace() somehwere +nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py +python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith +``` ## Approach diff --git a/tangostationcontrol/tox.ini b/tangostationcontrol/tox.ini index 5c7ac29a49125ec8f8542372c6e1b99ebe369e6d..62b18988c5b4289ae1c12b3283d7e1b4b52fd8f7 100644 --- a/tangostationcontrol/tox.ini +++ b/tangostationcontrol/tox.ini @@ -29,7 +29,7 @@ commands = {envpython} -m stestr run {posargs} ; Warning running integration tests will make changes to your docker system! ; These tests should only be run by the integration-test docker container. passenv = TANGO_HOST -setenv = TESTS_DIR=./tangostationcontrol/integration_test/{posargs} +setenv = TESTS_DIR=./tangostationcontrol/integration_test/{env:TEST_MODULE:default} commands = {envpython} -m stestr run --serial {posargs}