From c5f294b37a916eabd7b5773e32d62c0cf4e5bf60 Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Tue, 8 Jan 2019 17:59:12 +0000
Subject: [PATCH] OSB-37: end of day commit
---
.gitattributes | 6 +-
CMake/DjangoPostgres.cmake | 56 -----
LCU/Maintenance/DBInterface/CMakeLists.txt | 1 -
.../monitoringdb/serializers/rtsm.py | 3 +-
.../DBInterface/monitoringdb/urls.py | 11 +-
.../DBInterface/test/CMakeLists.txt | 3 +
.../test/controllers/CMakeLists.txt | 7 +
.../t_all_components_error_types.run | 5 +
.../t_all_components_error_types.sh | 3 +
.../test_all_components_error_types.py | 14 ++
.../test/controllers/test_data_generators.py | 210 ++++++++++++++++++
11 files changed, 255 insertions(+), 64 deletions(-)
delete mode 100644 CMake/DjangoPostgres.cmake
create mode 100644 LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt
create mode 100755 LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run
create mode 100755 LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh
create mode 100644 LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py
create mode 100644 LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py
diff --git a/.gitattributes b/.gitattributes
index 88ae44cd946..34b1482832a 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1265,7 +1265,6 @@ CEP/PyBDSM/test/tbdsm_process_image.in -text
CEP/PyBDSM/test/tbdsm_process_image.in_fits -text svneol=unset#image/x-fits
CEP/PyBDSM/test/tbdsm_process_image.py -text
CEP/doc/package.dox -text
-CMake/DjangoPostgres.cmake -text
CMake/FindALGLIB.cmake -text
CMake/FindAOFlagger.cmake -text
CMake/FindCUDADriver.cmake -text
@@ -1848,6 +1847,11 @@ LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text
LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text
LCU/Maintenance/DBInterface/test/CMakeLists.txt -text
LCU/Maintenance/DBInterface/test/__init__.py -text
+LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt -text
+LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run -text
+LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh -text
+LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py -text
+LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py -text
LCU/Maintenance/DBInterface/test/postgres_testrunner.py -text
LCU/Maintenance/DBInterface/test/t_rtsm_models.run -text
LCU/Maintenance/DBInterface/test/t_rtsm_models.sh -text
diff --git a/CMake/DjangoPostgres.cmake b/CMake/DjangoPostgres.cmake
deleted file mode 100644
index 5c0aff3f4c2..00000000000
--- a/CMake/DjangoPostgres.cmake
+++ /dev/null
@@ -1,56 +0,0 @@
-# - Setup the test infrastructure for the django postgres database.
-# Variables used by this module:
-# project_name - Name of the django project
-# test_directory - Directory where the tests are stored
-# Variables defined by this module:
-
-# Copyright (C) 2009
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-
-macro(add_django_project_test_dir project_name django_binary)
- message(STATUS "setting up the test infrastructure for ${project_name} ")
- find_python_module(testing.postgresql)
- find_python_module(ldap_test REQUIRED) # sudo pip3 install python-ldap-test
- find_python_module(ldap3 REQUIRED) # sudo pip3 install ldap3
-
- set(DJANGO_PROJECT ${project_name})
- # Parsing arguments
- set(one_value_arguments DB_USER DB_PASS)
- ## syntax is cmake_parse_arguments(prefix options one_value_arguments multi_value_arguments arguments_to_be_parsed)
- cmake_parse_arguments(DJANGO_PROJECT "" "${one_value_arguments}" "" ${ARGN} )
- if(DJANGO_PROJECT_DB_USER)
- set(DB_USER ${ADD_DJANGO_PROJECT_DB_USER})
-
- else(DJANGO_PROJECT_DB_USER)
- string(RANDOM DB_USER)
- message(STATUS "using ${DB_PASS} as test database user")
- endif(DJANGO_PROJECT_DB_USER)
-
- if(DJANGO_PROJECT_DB_PASS)
- set(DB_PASS ${ADD_DJANGO_PROJECT_DB_PASS})
- else(DJANGO_PROJECT_DB_PASS)
- string(RANDOM DB_PASS)
- message(STATUS "using ${DB_PASS} as test database password")
- endif(DJANGO_PROJECT_DB_PASS)
-
-
- message(STATUS "credentials used for the postgres database \tuser: " ${DB_USER} "\tpass: " ${DB_PASS})
- configure_file(${CMAKE_SOURCE_DIR}/CMake/testscripts/django_postgres.sh ${CMAKE_CURRENT_BINARY_DIR}/test_funcs.sh @ONLY)
-
-endmacro(add_django_project_test_dir)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/CMakeLists.txt b/LCU/Maintenance/DBInterface/CMakeLists.txt
index 1bb941cfd83..22ab80dd164 100644
--- a/LCU/Maintenance/DBInterface/CMakeLists.txt
+++ b/LCU/Maintenance/DBInterface/CMakeLists.txt
@@ -6,7 +6,6 @@ lofar_find_package(Python 3.0 REQUIRED)
lofar_package(DBInterface 1.0 DEPENDS PyCommon)
include(PythonInstall)
-include(DjangoPostgres)
include(FindPythonModule)
find_python_module(django REQUIRED)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index 23ab677706c..cd2a577bf40 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -48,7 +48,8 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
model = RTSMObservation
fields = '__all__'
- def compute_summary(self, RTSMObservation_entry):
+ @staticmethod
+ def compute_summary(RTSMObservation_entry):
summary_entries = RTSMObservation_entry.errors.defer('spectra'). \
filter(observation=RTSMObservation_entry).values('error_type', 'mode', 'rcu', 'start_frequency',
'stop_frequency'). \
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
index 67ff6ed7521..3286959e501 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -1,11 +1,11 @@
from django.conf.urls import url, include
from rest_framework import routers
+from rest_framework.documentation import include_docs_urls
+
from .views.controllers import *
-from .views.station_test_views import *
-from .views.rtsm_views import *
from .views.logs_view import *
-from rest_framework.documentation import include_docs_urls
-from django.urls import path
+from .views.rtsm_views import *
+from .views.station_test_views import *
log_router = routers.DefaultRouter()
log_router.register(r'action_log', ActionLogViewSet)
@@ -45,6 +45,7 @@ urlpatterns = [
url(r'^api/view/ctrl_stationtest_statistics', ControllerStationTestStatistics.as_view()),
url(r'^api/view/ctrl_list_component_error_types', ControllerAllComponentErrorTypes.as_view()),
url(r'^api/view/ctrl_station_component_errors', ControllerStationComponentErrors.as_view()),
- url(r'^api/view/ctrl_station_component_element_errors', ControllerStationComponentElementErrors.as_view()),
+ url(r'^api/view/ctrl_station_component_element_errors',
+ ControllerStationComponentElementErrors.as_view()),
url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
]
diff --git a/LCU/Maintenance/DBInterface/test/CMakeLists.txt b/LCU/Maintenance/DBInterface/test/CMakeLists.txt
index ad09e46b946..57751fe31fb 100644
--- a/LCU/Maintenance/DBInterface/test/CMakeLists.txt
+++ b/LCU/Maintenance/DBInterface/test/CMakeLists.txt
@@ -1,5 +1,6 @@
# $Id$
include(LofarCTest)
+find_python_module(coverage)
set(py_files postgres_testrunner.py
__init__.py
@@ -7,3 +8,5 @@ set(py_files postgres_testrunner.py
python_install(${py_files} DESTINATION lofar/maintenance/test)
lofar_add_test(t_rtsm_models)
+
+add_subdirectory(controllers)
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt b/LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt
new file mode 100644
index 00000000000..1f9417fb438
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt
@@ -0,0 +1,7 @@
+# $Id$
+include(LofarCTest)
+
+set(py_files test_all_components_error_types.py test_data_generators.py)
+python_install(${py_files} DESTINATION lofar/maintenance/test)
+
+lofar_add_test(t_all_components_error_types)
diff --git a/LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run b/LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run
new file mode 100755
index 00000000000..18af7b15809
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+coverage3 run --include=*controllers* -m lofar.maintenance.manage test lofar.maintenance.test.test_all_components_error_types --testrunner=lofar.maintenance.test.postgres_testrunner.PostgresqlTestRunner --settings lofar.maintenance.django_postgresql.test_settings
+coverage3 report
+
diff --git a/LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh b/LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh
new file mode 100755
index 00000000000..cde0479208d
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+./runctest.sh t_all_components_error_types
diff --git a/LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py b/LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py
new file mode 100644
index 00000000000..66d9fc11da0
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py
@@ -0,0 +1,14 @@
+from rest_framework.test import APIClient, APITestCase
+
+from test_data_generators import generate_random_test_data
+
+
+class AllComponentsErrorTypesTest(APITestCase):
+ def setUp(self):
+ self.expected_error_types = ['SN', 'RS', 'VF']
+ self.apiClient = APIClient()
+ generate_random_test_data(2, 3, 5, 0, 0, expected_error_types=self.expected_error_types)
+
+ def test_successfull_api_query(self):
+ response = self.apiClient.get('/api/view/ctrl_list_component_error_types')
+ self.assertEqual(set(response.data), set(self.expected_error_types))
diff --git a/LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py b/LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py
new file mode 100644
index 00000000000..6c2543aa780
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py
@@ -0,0 +1,210 @@
+import random
+from datetime import datetime
+
+from lofar.maintenance.monitoringdb.models.component import Component
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation, RTSMError
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
+from lofar.maintenance.monitoringdb.serializers.rtsm import RTSMObservationSerializer
+
+TEST_STATION_NAMES = ('CS001', 'CS002')
+TEST_STATION_TYPE = ('C', 'R', 'I')
+
+TEST_COMPONENT_IDS = [i for i in range(48)]
+TEST_COMPONENT_ERROR_TYPES = ('HIGH_NOISE', 'LOW_NOISE', 'DOWN')
+
+TEST_COMPONENT_TYPES = ('HBA', 'LBH')
+TEST_MODES = (1, 2, 3, 4, 5, 6, 7)
+
+TEST_START_DATETIME = datetime(2018, 5, 12, 0, 0, 0)
+TEST_END_DATETIME = datetime(2018, 5, 12, 0, 0, 0)
+
+
+def random_date_in_range(start, end):
+ """
+ Computes a random datetime between the specified interval
+ :param start: start date of the interval
+ :type start: datetime
+ :param end: end date of the interval
+ :type end: datetime
+ :return: a random datetime in the specified interval
+ :rtype: datetime
+ """
+ time = random.uniform(start.timestamp(), end.timestamp())
+ return datetime.fromtimestamp(time)
+
+
+def generate_test_component(sample_size, station, expected_component_ids=TEST_COMPONENT_IDS,
+ expected_component_types=TEST_COMPONENT_TYPES):
+ """
+ Generate a sample of [sample_size] components in a given station
+ :param expected_component_ids: the component ids that have to be used for the generation
+ :type expected_component_ids: tuple(int)
+ :param expected_component_types: the component types that have to be used for the generation
+ :type expected_component_types: tuple(str)
+ :param sample_size: sample size
+ :type sample_size: int
+ :param station: generated components' station
+ :type station: Station
+ :return: a tuple of component entities
+ :rtype: tuple(Component)
+ """
+ component_list = []
+ for i in range(sample_size):
+ test_component = Component()
+ test_component.component_id = random.choice(expected_component_ids)
+ test_component.station = station
+ test_component.type = random.choice(expected_component_types)
+ test_component.save()
+ component_list.append(test_component)
+ return tuple(component_list)
+
+
+def generate_test_component_errors(sample_size, station_test,
+ expected_component_error_types=TEST_COMPONENT_ERROR_TYPES,
+ expected_component_types=TEST_COMPONENT_TYPES):
+ """
+ Generate a sample of [sample size] component errors found during a station test
+ :param expected_component_types: the component types to be used for the generation
+ :param expected_component_error_types: the error types to be used for the generation
+ :type expected_component_error_types: tuple(str)
+ :param sample_size: size of the sample
+ :type sample_size: int
+ :param station_test: component errors' station test
+ :type station_test: StationTest
+ :return: a tuple of component error instances
+ """
+ component_error_list = []
+ for i in range(sample_size):
+ component_entity, = generate_test_component(1, station_test.station,
+ expected_component_types=expected_component_types)
+ component_error = ComponentError()
+ component_error.station_test = station_test
+ component_error.component = component_entity
+ component_error.type = random.choice(expected_component_error_types)
+ component_error.save()
+ component_error_list.append(component_error)
+ return tuple(component_error_list)
+
+
+def generate_test_station(sample_size):
+ """
+ Generate a sample of station to be used for testing
+ :param sample_size: size of the sample
+ :type sample_size: int
+
+ :return: a tuple of stations
+ :rtype: tuple(Station)
+ """
+ station_list = []
+ for i in range(sample_size):
+ station = Station()
+ station.name = random.choice(TEST_STATION_NAMES)
+ station.type = random.choice(TEST_STATION_TYPE)
+ station.location = ''
+ station.save()
+ station_list.append(station)
+ return station_list
+
+
+def generate_test_station_test(sample_size, station, start_datetime=TEST_START_DATETIME,
+ end_datetime=TEST_END_DATETIME):
+ """
+ Generate a sample of station to be used for testing
+ :param sample_size: size of the sample
+ :type sample_size: int
+ :param station: station test's station
+ :type station: Station
+ :param start_datetime: end date and time of the time interval in which the station test will be generated
+ :type start_datetime: datetime
+ :param end_datetime: end date and time of the time interval in which the station test will be generated
+ :return: a tuple of station tests
+ :rtype: tuple(StationTest)
+ """
+ station_test_list = []
+ for i in range(sample_size):
+ station_test = StationTest()
+ station_test.station = station
+ station_test.start_datetime = random_date_in_range(start_datetime, end_datetime)
+ station_test.end_datetime = random_date_in_range(station_test.start_datetime, end_datetime)
+ station_test.save()
+ station_test_list.append(station_test)
+ return tuple(station_test_list)
+
+
+def generate_test_rtsm_error(sample_size, rtsm_observation, expected_rcu_ids=TEST_COMPONENT_IDS,
+ expected_error_types=TEST_COMPONENT_ERROR_TYPES,
+ expected_mode=TEST_MODES,
+ start_datetime=TEST_START_DATETIME,
+ end_datetime=TEST_END_DATETIME):
+ """
+
+ :param sample_size:
+ :param rtsm_observation:
+ :type rtsm_observation: RTSMObservation
+ :param expected_rcu_ids:
+ :param expected_error_types:
+ :param expected_mode:
+ :param start_datetime:
+ :param end_datetime:
+ :return:
+ """
+ for i in range(sample_size):
+ rtsm_error = RTSMError()
+ rtsm_error.observation = rtsm_observation
+ rtsm_error.mode = random.choice(expected_mode)
+ rtsm_error.rcu = random.choice(expected_rcu_ids)
+ rtsm_error.start_frequency = 100
+ rtsm_error.stop_frequency = 200
+ rtsm_error.time = random_date_in_range(start_datetime, end_datetime)
+ rtsm_error.error_type = random.choice(expected_error_types)
+ rtsm_error.save()
+ RTSMObservationSerializer.compute_summary(rtsm_observation)
+
+
+def generate_rtsm_observation(sample_size, station, start_date=datetime(2018, 5, 12, 0, 0, 0),
+ end_date=datetime(2018, 5, 12, 1, 0, 0)):
+ """
+ Generate a sample of RTSMObservation objects
+ :param sample_size: size of the sample
+ :type sample_size: int
+ :param station: RTSM Observation's station
+ :type station: Station
+ :param start_date: end date and time of the time interval in which the station test will be generated
+ :type start_date: datetime
+ :param end_date: end date and time of the time interval in which the station test will be generated
+ :return: a tuple of RTSM Observations
+ :rtype: tuple(RTSMObservation)
+ """
+ rtsm_observation_list = []
+ for i in range(sample_size):
+ rtsm_observation = RTSMObservation()
+ rtsm_observation.station = station
+ rtsm_observation.observation_id = random.uniform(6000, 7000)
+ rtsm_observation.start_datetime = random_date_in_range(start_date, end_date)
+ rtsm_observation.end_datetime = random_date_in_range(rtsm_observation.start_datetime,
+ end_date)
+ rtsm_observation.save()
+ rtsm_observation_list.append(rtsm_observation)
+ return rtsm_observation_list
+
+
+def generate_random_test_data(number_of_station,
+ number_of_station_tests,
+ number_of_component_errors,
+ number_of_observations,
+ number_of_rtsm_errors,
+ expected_error_types=TEST_COMPONENT_ERROR_TYPES,
+ expected_component_types=TEST_COMPONENT_TYPES):
+ """
+ Generates and insert in the test database the data to test
+ the view response
+ """
+ stations = generate_test_station(number_of_station)
+ for station in stations:
+ station_tests = generate_test_station_test(number_of_station_tests, station)
+ for station_test in station_tests:
+ generate_test_component_errors(number_of_component_errors, station_test,
+ expected_component_error_types=expected_error_types,
+ expected_component_types=expected_component_types)
--
GitLab