diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3217933d5c2bec9e6b1570400d2338c66d43185..85348759b1a70101d67231c05d3a6fa53e3d998d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ stages: - build - unit_test - integration_test + - regressions_test - dockerize - deploy-test - deploy-prod @@ -288,7 +289,7 @@ unit_test_SCU: - PACKAGE=SCU - echo "Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true services: - rabbitmq:latest @@ -313,7 +314,7 @@ unit_test_LTAIngest: - PACKAGE=LTAIngest - echo "Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true services: - rabbitmq:latest @@ -338,7 +339,7 @@ unit_test_LTAIngestTransfer: - PACKAGE=LTAIngestTransfer - echo "Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true services: - rabbitmq:latest @@ -364,7 +365,7 @@ unit_test_MCU_MAC: - PACKAGE=MCU_MAC - echo "Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true services: - rabbitmq:latest @@ -448,7 +449,7 @@ integration_test_SCU: - PACKAGE=SCU - echo "Integration Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true needs: - build_SCU @@ -468,7 +469,7 @@ integration_test_LTAIngest: - PACKAGE=LTAIngest - echo "Integration Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true services: - rabbitmq:latest @@ -494,7 +495,7 @@ integration_test_LTAIngestTransfer: - PACKAGE=LTAIngestTransfer - echo "Integration Testing $PACKAGE..." - cd build/gnucxx11_opt - - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true SKIP_REGRESSION_TESTS=true ctest interruptible: true services: - rabbitmq:latest @@ -513,6 +514,36 @@ integration_test_LTAIngestTransfer: rules: - if: '$FASTLANE == "false"' +# +# REGRESSION TEST STAGE +# + +regression_test_SCU: + stage: regression_test + image: ci_scu:$CI_COMMIT_SHORT_SHA + services: + - rabbitmq:latest + variables: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq. + script: + - PACKAGE=SCU + - echo "Regression Testing $PACKAGE..." + - cd build/gnucxx11_opt + - SKIP_PYTHON_COVERAGE=true SKIP_REGRESSION_TESTS=false SKIP_UNIT_TESTS=true SKIP_INTEGRATION_TESTS=true ctest + interruptible: true + needs: + - build_SCU + # - unit_test_SCU + artifacts: + name: regression-test-report + when: always + paths: + - build/gnucxx11_opt/Testing/Temporary/LastTest.log + rules: + - if: '$FASTLANE == "false"' + # # DEPLOY TEST STAGE # diff --git a/LCS/PyCommon/test_utils.py b/LCS/PyCommon/test_utils.py index e5e641ab7bfdba562528bc0e3c8dacfd00efe84f..408b4b8d78a424b74326f54b97fab067bcf5e111 100644 --- a/LCS/PyCommon/test_utils.py +++ b/LCS/PyCommon/test_utils.py @@ -43,6 +43,10 @@ def assertEqualXML(test, expected): msg = diff raise AssertionError(msg) +def skip_regression_tests() -> bool: + '''returns a boolen True of the environment var SKIP_REGRESSION_TESTS has been set to a 'true' value''' + return os.environ.get('SKIP_REGRESSION_TESTS', default='False').lower() in ['1', 'true', 'on'] + def skip_integration_tests() -> bool: '''returns a boolen True of the environment var SKIP_INTEGRATION_TESTS has been set to a 'true' value''' return os.environ.get('SKIP_INTEGRATION_TESTS', default='False').lower() in ['1', 'true', 'on'] @@ -61,7 +65,15 @@ def exit_with_skipped_code_if_skip_integration_tests(): if skip_integration_tests(): exit(3) +def exit_with_skipped_code_if_skip_regression_tests(): + '''exit with code 3 (SKIPPED) when the environment var SKIP_REGRESSION_TESTS has been set to a 'true' value''' + if skip_regression_tests(): + exit(3) + # decorators for selective tests +regression_test = unittest.skipIf(skip_regression_tests(), + 'Regression tests are disabled via env SKIP_REGRESSION_TESTS') + integration_test = unittest.skipIf(skip_integration_tests(), 'Integration tests are disabled via env SKIP_INTEGRATION_TESTS') diff --git a/SAS/TMSS/backend/test/t_observing_strategies_regression_test.py b/SAS/TMSS/backend/test/t_observing_strategies_regression_test.py index 2d85c48bf4257ea5b2d05fb7f89673591752602b..96bc895c3d7d6cdedf7a8e19e9d7768c13d88b4a 100755 --- a/SAS/TMSS/backend/test/t_observing_strategies_regression_test.py +++ b/SAS/TMSS/backend/test/t_observing_strategies_regression_test.py @@ -5,9 +5,9 @@ import logging logger = logging.getLogger('lofar.'+__name__) logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) -from lofar.common.test_utils import exit_with_skipped_code_if_skip_integration_tests, assertEqualXML +from lofar.common.test_utils import exit_with_skipped_code_if_skip_regression_tests, assertEqualXML -exit_with_skipped_code_if_skip_integration_tests() +exit_with_skipped_code_if_skip_regression_tests() import json import os