OSB-37: end of day commit

.gitattributes

@@ -1265,7 +1265,6 @@ CEP/PyBDSM/test/tbdsm_process_image.in -text
 CEP/PyBDSM/test/tbdsm_process_image.in_fits -text svneol=unset#image/x-fits
 CEP/PyBDSM/test/tbdsm_process_image.py -text
 CEP/doc/package.dox -text
-CMake/DjangoPostgres.cmake -text
 CMake/FindALGLIB.cmake -text
 CMake/FindAOFlagger.cmake -text
 CMake/FindCUDADriver.cmake -text
@@ -1848,6 +1847,11 @@ LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text
 LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text
 LCU/Maintenance/DBInterface/test/CMakeLists.txt -text
 LCU/Maintenance/DBInterface/test/__init__.py -text
+LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt -text
+LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run -text
+LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh -text
+LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py -text
+LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py -text
 LCU/Maintenance/DBInterface/test/postgres_testrunner.py -text
 LCU/Maintenance/DBInterface/test/t_rtsm_models.run -text
 LCU/Maintenance/DBInterface/test/t_rtsm_models.sh -text

CMake/DjangoPostgres.cmake

-# - Setup the test infrastructure for the django postgres database.
-# Variables used by this module:
-#  project_name     - Name of the django project
-#  test_directory   - Directory where the tests are stored
-# Variables defined by this module:
-
-# Copyright (C) 2009
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-
-macro(add_django_project_test_dir project_name django_binary)
-    message(STATUS "setting up the test infrastructure for ${project_name} ")
-    find_python_module(testing.postgresql)
-    find_python_module(ldap_test REQUIRED)            # sudo pip3 install python-ldap-test
-    find_python_module(ldap3 REQUIRED)            # sudo pip3 install ldap3
-
-    set(DJANGO_PROJECT ${project_name})
-    # Parsing arguments
-    set(one_value_arguments DB_USER DB_PASS)
-    ## syntax is cmake_parse_arguments(prefix options one_value_arguments multi_value_arguments arguments_to_be_parsed)
-    cmake_parse_arguments(DJANGO_PROJECT "" "${one_value_arguments}" "" ${ARGN} )
-    if(DJANGO_PROJECT_DB_USER)
-        set(DB_USER ${ADD_DJANGO_PROJECT_DB_USER})
-
-    else(DJANGO_PROJECT_DB_USER)
-        string(RANDOM DB_USER)
-        message(STATUS "using ${DB_PASS} as test database user")
-    endif(DJANGO_PROJECT_DB_USER)
-
-    if(DJANGO_PROJECT_DB_PASS)
-        set(DB_PASS ${ADD_DJANGO_PROJECT_DB_PASS})
-    else(DJANGO_PROJECT_DB_PASS)
-        string(RANDOM DB_PASS)
-        message(STATUS "using ${DB_PASS} as test database password")
-    endif(DJANGO_PROJECT_DB_PASS)
-
-
-    message(STATUS "credentials used for the postgres database \tuser: " ${DB_USER} "\tpass: " ${DB_PASS})
-    configure_file(${CMAKE_SOURCE_DIR}/CMake/testscripts/django_postgres.sh ${CMAKE_CURRENT_BINARY_DIR}/test_funcs.sh @ONLY)
-
-endmacro(add_django_project_test_dir)
\ No newline at end of file

LCU/Maintenance/DBInterface/CMakeLists.txt

@@ -6,7 +6,6 @@ lofar_find_package(Python 3.0 REQUIRED)
 lofar_package(DBInterface 1.0 DEPENDS PyCommon)
 
 include(PythonInstall)
-include(DjangoPostgres)
 
 include(FindPythonModule)
 find_python_module(django REQUIRED)

LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py

@@ -48,7 +48,8 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
         model = RTSMObservation
         fields = '__all__'
 
-    def compute_summary(self, RTSMObservation_entry):
+    @staticmethod
+    def compute_summary(RTSMObservation_entry):
         summary_entries = RTSMObservation_entry.errors.defer('spectra'). \
             filter(observation=RTSMObservation_entry).values('error_type', 'mode', 'rcu', 'start_frequency',
                                                              'stop_frequency'). \

LCU/Maintenance/DBInterface/monitoringdb/urls.py

 from django.conf.urls import url, include
 from rest_framework import routers
+from rest_framework.documentation import include_docs_urls
+
 from .views.controllers import *
-from .views.station_test_views import *
-from .views.rtsm_views import *
 from .views.logs_view import *
-from rest_framework.documentation import include_docs_urls
-from django.urls import path
+from .views.rtsm_views import *
+from .views.station_test_views import *
 
 log_router = routers.DefaultRouter()
 log_router.register(r'action_log', ActionLogViewSet)
@@ -45,6 +45,7 @@ urlpatterns = [
     url(r'^api/view/ctrl_stationtest_statistics', ControllerStationTestStatistics.as_view()),
     url(r'^api/view/ctrl_list_component_error_types', ControllerAllComponentErrorTypes.as_view()),
     url(r'^api/view/ctrl_station_component_errors', ControllerStationComponentErrors.as_view()),
-    url(r'^api/view/ctrl_station_component_element_errors', ControllerStationComponentElementErrors.as_view()),
+    url(r'^api/view/ctrl_station_component_element_errors',
+        ControllerStationComponentElementErrors.as_view()),
     url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
 ]

LCU/Maintenance/DBInterface/test/CMakeLists.txt

 # $Id$
 include(LofarCTest)
+find_python_module(coverage)
 
 set(py_files postgres_testrunner.py
              __init__.py
@@ -7,3 +8,5 @@ set(py_files postgres_testrunner.py
 python_install(${py_files} DESTINATION lofar/maintenance/test)
 
 lofar_add_test(t_rtsm_models)
+
+add_subdirectory(controllers)
\ No newline at end of file

LCU/Maintenance/DBInterface/test/controllers/CMakeLists.txt

+# $Id$
+include(LofarCTest)
+
+set(py_files test_all_components_error_types.py test_data_generators.py)
+python_install(${py_files} DESTINATION lofar/maintenance/test)
+
+lofar_add_test(t_all_components_error_types)

LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.run

+#!/usr/bin/env bash
+
+coverage3 run --include=*controllers* -m lofar.maintenance.manage test lofar.maintenance.test.test_all_components_error_types --testrunner=lofar.maintenance.test.postgres_testrunner.PostgresqlTestRunner  --settings lofar.maintenance.django_postgresql.test_settings
+coverage3 report
+

LCU/Maintenance/DBInterface/test/controllers/t_all_components_error_types.sh

+#!/usr/bin/env bash
+
+./runctest.sh t_all_components_error_types

LCU/Maintenance/DBInterface/test/controllers/test_all_components_error_types.py

+from rest_framework.test import APIClient, APITestCase
+
+from test_data_generators import generate_random_test_data
+
+
+class AllComponentsErrorTypesTest(APITestCase):
+    def setUp(self):
+        self.expected_error_types = ['SN', 'RS', 'VF']
+        self.apiClient = APIClient()
+        generate_random_test_data(2, 3, 5, 0, 0, expected_error_types=self.expected_error_types)
+
+    def test_successfull_api_query(self):
+        response = self.apiClient.get('/api/view/ctrl_list_component_error_types')
+        self.assertEqual(set(response.data), set(self.expected_error_types))

LCU/Maintenance/DBInterface/test/controllers/test_data_generators.py

+import random
+from datetime import datetime
+
+from lofar.maintenance.monitoringdb.models.component import Component
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation, RTSMError
+from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.station_test import StationTest
+from lofar.maintenance.monitoringdb.serializers.rtsm import RTSMObservationSerializer
+
+TEST_STATION_NAMES = ('CS001', 'CS002')
+TEST_STATION_TYPE = ('C', 'R', 'I')
+
+TEST_COMPONENT_IDS = [i for i in range(48)]
+TEST_COMPONENT_ERROR_TYPES = ('HIGH_NOISE', 'LOW_NOISE', 'DOWN')
+
+TEST_COMPONENT_TYPES = ('HBA', 'LBH')
+TEST_MODES = (1, 2, 3, 4, 5, 6, 7)
+
+TEST_START_DATETIME = datetime(2018, 5, 12, 0, 0, 0)
+TEST_END_DATETIME = datetime(2018, 5, 12, 0, 0, 0)
+
+
+def random_date_in_range(start, end):
+    """
+    Computes a random datetime between the specified interval
+    :param start: start date of the interval
+    :type start: datetime
+    :param end: end date of the interval
+    :type end: datetime
+    :return: a random datetime in the specified interval
+    :rtype: datetime
+    """
+    time = random.uniform(start.timestamp(), end.timestamp())
+    return datetime.fromtimestamp(time)
+
+
+def generate_test_component(sample_size, station, expected_component_ids=TEST_COMPONENT_IDS,
+                            expected_component_types=TEST_COMPONENT_TYPES):
+    """
+    Generate a sample of [sample_size] components in a given station
+    :param expected_component_ids: the component ids that have to be used for the generation
+    :type expected_component_ids: tuple(int)
+    :param expected_component_types: the component types that have to be used for the generation
+    :type expected_component_types: tuple(str)
+    :param sample_size: sample size
+    :type sample_size: int
+    :param station: generated components' station
+    :type station: Station
+    :return: a tuple of component entities
+    :rtype: tuple(Component)
+    """
+    component_list = []
+    for i in range(sample_size):
+        test_component = Component()
+        test_component.component_id = random.choice(expected_component_ids)
+        test_component.station = station
+        test_component.type = random.choice(expected_component_types)
+        test_component.save()
+        component_list.append(test_component)
+    return tuple(component_list)
+
+
+def generate_test_component_errors(sample_size, station_test,
+                                   expected_component_error_types=TEST_COMPONENT_ERROR_TYPES,
+                                   expected_component_types=TEST_COMPONENT_TYPES):
+    """
+    Generate a sample of [sample size] component errors found during a station test
+    :param expected_component_types: the component types to be used for the generation
+    :param expected_component_error_types: the error types to be used for the generation
+    :type expected_component_error_types: tuple(str)
+    :param sample_size: size of the sample
+    :type sample_size: int
+    :param station_test: component errors' station test
+    :type station_test: StationTest
+    :return: a tuple of component error instances
+    """
+    component_error_list = []
+    for i in range(sample_size):
+        component_entity, = generate_test_component(1, station_test.station,
+                                                    expected_component_types=expected_component_types)
+        component_error = ComponentError()
+        component_error.station_test = station_test
+        component_error.component = component_entity
+        component_error.type = random.choice(expected_component_error_types)
+        component_error.save()
+        component_error_list.append(component_error)
+    return tuple(component_error_list)
+
+
+def generate_test_station(sample_size):
+    """
+    Generate a sample of station to be used for testing
+    :param sample_size: size of the sample
+    :type sample_size: int
+
+    :return: a tuple of stations
+    :rtype: tuple(Station)
+    """
+    station_list = []
+    for i in range(sample_size):
+        station = Station()
+        station.name = random.choice(TEST_STATION_NAMES)
+        station.type = random.choice(TEST_STATION_TYPE)
+        station.location = ''
+        station.save()
+        station_list.append(station)
+    return station_list
+
+
+def generate_test_station_test(sample_size, station, start_datetime=TEST_START_DATETIME,
+                               end_datetime=TEST_END_DATETIME):
+    """
+    Generate a sample of station to be used for testing
+    :param sample_size: size of the sample
+    :type sample_size: int
+    :param station: station test's station
+    :type station: Station
+    :param start_datetime: end date and time of the time interval in which the station test will be generated
+    :type start_datetime: datetime
+    :param end_datetime: end date and time of the time interval in which the station test will be generated
+    :return: a tuple of station tests
+    :rtype: tuple(StationTest)
+    """
+    station_test_list = []
+    for i in range(sample_size):
+        station_test = StationTest()
+        station_test.station = station
+        station_test.start_datetime = random_date_in_range(start_datetime, end_datetime)
+        station_test.end_datetime = random_date_in_range(station_test.start_datetime, end_datetime)
+        station_test.save()
+        station_test_list.append(station_test)
+    return tuple(station_test_list)
+
+
+def generate_test_rtsm_error(sample_size, rtsm_observation, expected_rcu_ids=TEST_COMPONENT_IDS,
+                             expected_error_types=TEST_COMPONENT_ERROR_TYPES,
+                             expected_mode=TEST_MODES,
+                             start_datetime=TEST_START_DATETIME,
+                             end_datetime=TEST_END_DATETIME):
+    """
+
+    :param sample_size:
+    :param rtsm_observation:
+    :type rtsm_observation: RTSMObservation
+    :param expected_rcu_ids:
+    :param expected_error_types:
+    :param expected_mode:
+    :param start_datetime:
+    :param end_datetime:
+    :return:
+    """
+    for i in range(sample_size):
+        rtsm_error = RTSMError()
+        rtsm_error.observation = rtsm_observation
+        rtsm_error.mode = random.choice(expected_mode)
+        rtsm_error.rcu = random.choice(expected_rcu_ids)
+        rtsm_error.start_frequency = 100
+        rtsm_error.stop_frequency = 200
+        rtsm_error.time = random_date_in_range(start_datetime, end_datetime)
+        rtsm_error.error_type = random.choice(expected_error_types)
+        rtsm_error.save()
+        RTSMObservationSerializer.compute_summary(rtsm_observation)
+
+
+def generate_rtsm_observation(sample_size, station, start_date=datetime(2018, 5, 12, 0, 0, 0),
+                              end_date=datetime(2018, 5, 12, 1, 0, 0)):
+    """
+    Generate a sample of RTSMObservation objects
+    :param sample_size: size of the sample
+    :type sample_size: int
+    :param station: RTSM Observation's station
+    :type station: Station
+    :param start_date: end date and time of the time interval in which the station test will be generated
+    :type start_date: datetime
+    :param end_date: end date and time of the time interval in which the station test will be generated
+    :return: a tuple of RTSM Observations
+    :rtype: tuple(RTSMObservation)
+    """
+    rtsm_observation_list = []
+    for i in range(sample_size):
+        rtsm_observation = RTSMObservation()
+        rtsm_observation.station = station
+        rtsm_observation.observation_id = random.uniform(6000, 7000)
+        rtsm_observation.start_datetime = random_date_in_range(start_date, end_date)
+        rtsm_observation.end_datetime = random_date_in_range(rtsm_observation.start_datetime,
+                                                             end_date)
+        rtsm_observation.save()
+        rtsm_observation_list.append(rtsm_observation)
+    return rtsm_observation_list
+
+
+def generate_random_test_data(number_of_station,
+                              number_of_station_tests,
+                              number_of_component_errors,
+                              number_of_observations,
+                              number_of_rtsm_errors,
+                              expected_error_types=TEST_COMPONENT_ERROR_TYPES,
+                              expected_component_types=TEST_COMPONENT_TYPES):
+    """
+    Generates and insert in the test database the data to test
+    the view response
+    """
+    stations = generate_test_station(number_of_station)
+    for station in stations:
+        station_tests = generate_test_station_test(number_of_station_tests, station)
+        for station_test in station_tests:
+            generate_test_component_errors(number_of_component_errors, station_test,
+                                           expected_component_error_types=expected_error_types,
+                                           expected_component_types=expected_component_types)