Merge branch 'L2SS-803' into 'master'

L2SS-804: Modules using env for individual integration tests

Closes L2SS-804

See merge request !360

docker-compose/.env

@@ -19,3 +19,5 @@ PG_SUPERUSER_PASSWORD=password
 PG_HDB_PASSWORD=hdbpp
 MYSQL_ROOT_PASSWORD=secret
 MYSQL_PASSWORD=tango
+
+TEST_MODULE=default

docker-compose/Makefile

@@ -50,6 +50,8 @@ else ifeq (attach,$(firstword $(MAKECMDGOALS)))
     endif
 else ifeq (run,$(firstword $(MAKECMDGOALS)))
     RUN_TARGET = true
+else ifeq (integration,$(firstword $(MAKECMDGOALS)))
+    INTEGRATION_TARGET = true
 endif
 
 ifdef SERVICE_TARGET
@@ -64,6 +66,12 @@ else ifdef RUN_TARGET
     $(eval $(SERVICE):;@:)
     SERVICE_ARGS := $(wordlist 3, $(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
     $(eval $(SERVICE_ARGS):;@:)
+else ifdef INTEGRATION_TARGET
+    # Isolate second argument as integration module, the rest as arguments
+    INTEGRATION_MODULE := $(wordlist 2, 2, $(MAKECMDGOALS))
+     $(eval $(INTEGRATION_MODULE):;@:)
+    INTEGRATION_ARGS := $(wordlist 3, $(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+    $(eval $(INTEGRATION_ARGS):;@:)
 endif
 
 #
@@ -142,7 +150,7 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
     DOCKER_GID=$(DOCKER_GID)
 
 
-.PHONY: up down minimal run start stop restart build build-nocache status clean pull help
+.PHONY: up down minimal run integration start stop restart build build-nocache status clean pull help
 .DEFAULT_GOAL := help
 
 pull: ## pull the images from the Docker hub
@@ -162,6 +170,9 @@ up: minimal  ## start the base TANGO system and prepare requested services
 run: minimal  ## run a service using arguments and delete it afterwards
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm $(SERVICE) $(SERVICE_ARGS)
 
+integration: minimal  ## run a service using arguments and delete it afterwards
+	TEST_MODULE=$(INTEGRATION_MODULE) $(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) run --no-deps --rm integration-test $(INTEGRATION_ARGS)
+
 down:  ## stop all services and tear down the system
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) down
 ifneq ($(NETWORK_MODE),host)

docker-compose/integration-test.yml

@@ -22,6 +22,7 @@ services:
         - ..:/opt/lofar/tango:rw
     environment:
       - TANGO_HOST=${TANGO_HOST}
+      - TEST_MODULE=${TEST_MODULE}
     working_dir: /opt/lofar/tango/tangostationcontrol
     entrypoint:
       - /usr/local/bin/wait-for-it.sh

sbin/run_integration_test.sh

 #!/bin/bash -e
 
+# Usage function explains how parameters are parsed
+function usage {
+    echo "./$(basename "$0")
+      no arguments, builds and configures all docker containers and starts each
+      stage of the integration test one after the other. Between each stage the
+      dsconfig is updated accordingly."
+    echo ""
+    echo "./$(basename "$0") -h
+      displays this help message"
+    echo ""
+    echo "./$(basename "$0") import-path.<class><function-name>
+      A specific test can be defined by specifying the entire an import path
+      and optionally extending this with a specific class and function. For
+      instance tangostationcontrol.integration_test.observations.test_archiver.TestArchiver.test_get_multimember_devices"
+    echo ""
+}
+
+# Configure the config database, restart containers and run a specific
+# integration module or even specific tests
+# integration_test module restarted_containers config_files specific_test
+function integration_test {
+  IFS=" " read -r -a restarts <<< "${2}"
+  IFS=" " read -r -a configs <<< "${3}"
+  for config in "${configs[@]}"; do
+    bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${config}"
+  done
+  if [ ! -z "${2+x}" ]; then
+    make restart "${restarts[@]}"
+  fi
+  sleep 5
+  make integration "${1}"
+}
+
+# list of arguments expected in the input
+optstring=":h"
+
+while getopts ${optstring} arg; do
+  case ${arg} in
+    h)
+      usage
+      exit 0
+      ;;
+    ?)
+      echo "Invalid option: -${OPTARG}."
+      exit 2
+      ;;
+  esac
+done
+
 if [ -z "$LOFAR20_DIR" ]; then
     # We assume we aren't in the PATH, so we can derive our path.
     # We need our parent directory.
@@ -70,19 +119,8 @@ echo '/usr/local/bin/wait-for-it.sh archiver-timescale:5432 --strict --timeout=3
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 make up integration-test
 
-# Run the default integration tests
-make run integration-test default
+integration_test default
 
-# Configure integration test for recv_cluster module
-bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/integrations/recvcluster_ConfigDb.json
-make restart device-recv device-tilebeam device-antennafield
-sleep 5
-
-make run integration-test recv_cluster
-
-# Configure integration test for multi-observation devices
-bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/integrations/multiobs_ConfigDb.json
-make restart archiver-timescale hdbppts-cm hdbppts-es
-sleep 5
+integration_test recv_cluster "device-recv device-tilebeam device-antennafield" "${LOFAR20_DIR}/CDB/integrations/recvcluster_ConfigDb.json"
 
-make run integration-test observations
+integration_test observations "archiver-timescale hdbppts-cm hdbppts-es" "${LOFAR20_DIR}/CDB/integrations/multiobs_ConfigDb.json"

tangostationcontrol/tangostationcontrol/integration_test/README.md

@@ -3,8 +3,55 @@
 Integration tests are separated into multi modules. Each module requires a
 different state and configuration. These configurations are managed externally.
 
-Invocation of individual modules is achieved through tox:
-`tox -e integration MODULE_SUBFOLDER`
+In total the orchestration of integration tests is handled through four separate
+layers, each one calling the next:
+
+1. `sbin/run_integration_test.sh`
+2. `docker-compose/Makefile` - integration target
+3. `docker-compose/integration-test.yml`
+4. `tangostrationcontrol/tox.ini` - integration job
+
+Invocation of individual modules is achieved through tox, this environment
+value can be left empty for the `default` module:
+`TEST_MODULE=modulefolder tox -e integration`
+
+Individual tests can be invoked using arguments:
+
+`TEST_MODULE=default tox -e integration`
+
+These arguments and modules can also be passed at the level of the Makefile
+instead of through tox directly:
+
+```shell
+make integration default import.path.class.functionname`
+```
+
+## Breakpoints & Debuggers with Integration Tests
+
+It can be extremely helpful to be able to mount a debugger during integration
+tests. This can be achieved in two or three steps depending on the availability
+of tango on the host
+
+1. docker exec into a container based on `lofar-device-base`
+2. Navigate to the `tangostationcontrol` directory
+3. Execute `tox -e integration` to generate the virtual environment
+4. Activate the virtual environment `source .tox/integration/bin/activate`
+5. Modify the source files and add `import pdb; pdb.set_trace()` where you want
+   breakpoints.
+6. Execute the desired test using testtools `python -m testtools.run imort.class.path.functionname`
+
+A concrete example is shown below:
+
+```shell
+docker exec -it device-sdp /bin/bash
+cd tangostationcontrol
+# Single test to significantly reduce runtime
+tox -e integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
+source .tox/integration/bin/activate
+# Add import pdb; pdb.set_trace() somehwere
+nano integration tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.py
+python -m testtools.run tangostationcontrol.integration_test.default.devices.test_device_digitalbeam.TestDeviceDigitalBeam.test_pointing_to_zenith
+```
 
 ## Approach
 

tangostationcontrol/tox.ini

@@ -29,7 +29,7 @@ commands = {envpython} -m stestr run {posargs}
 ; Warning running integration tests will make changes to your docker system!
 ; These tests should only be run by the integration-test docker container.
 passenv = TANGO_HOST
-setenv = TESTS_DIR=./tangostationcontrol/integration_test/{posargs}
+setenv = TESTS_DIR=./tangostationcontrol/integration_test/{env:TEST_MODULE:default}
 commands =
     {envpython} -m stestr run --serial {posargs}