OSB-28: merge with master branch and various updates and fixes

.gitattributes

@@ -1875,16 +1875,24 @@ LCU/Maintenance/MDB_WebView/maintenancedb_view/src/themes/lofar.scss -text
 LCU/Maintenance/MDB_tools/CMakeLists.txt -text
 LCU/Maintenance/MDB_tools/bin/mdb_loader.py -text
 LCU/Maintenance/MDB_tools/bin/probe_mdb.py -text
+LCU/Maintenance/MDB_tools/bin/station_tests_watchdog.py -text
 LCU/Maintenance/MDB_tools/cli/__init__.py -text
 LCU/Maintenance/MDB_tools/cli/mdb_loader.py -text
 LCU/Maintenance/MDB_tools/cli/probe_mdb.py -text
+LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py -text
 LCU/Maintenance/MDB_tools/deploy.sh -text
 LCU/Maintenance/MDB_tools/fabfile.py -text
+LCU/Maintenance/MDB_tools/lib/CMakeLists.txt -text
+LCU/Maintenance/MDB_tools/lib/__init__.py -text
+LCU/Maintenance/MDB_tools/lib/client.py -text
 LCU/Maintenance/MDB_tools/requirements.txt -text
 LCU/Maintenance/MDB_tools/test/output_testing.py -text
 LCU/Maintenance/MDB_tools/test/python-coverage.sh eol=lf
-LCU/Maintenance/MDB_tools/test/t_mdb_loader.in.test_rtsm.data -text
-LCU/Maintenance/MDB_tools/test/t_mdb_loader.in.test_stationtest.data -text
+LCU/Maintenance/MDB_tools/test/t_client.in.test_rtsm.data -text
+LCU/Maintenance/MDB_tools/test/t_client.in.test_stationtest.data -text
+LCU/Maintenance/MDB_tools/test/t_client.py -text
+LCU/Maintenance/MDB_tools/test/t_client.run -text
+LCU/Maintenance/MDB_tools/test/t_client.sh -text
 LCU/Maintenance/MDB_tools/test/t_mdb_loader.py -text
 LCU/Maintenance/MDB_tools/test/t_mdb_loader.run -text
 LCU/Maintenance/MDB_tools/test/t_mdb_loader.sh -text
@@ -1899,6 +1907,12 @@ LCU/Maintenance/MDB_tools/test/t_probe_mdb.in.stationtest_rest_output.json -text
 LCU/Maintenance/MDB_tools/test/t_probe_mdb.py -text
 LCU/Maintenance/MDB_tools/test/t_probe_mdb.run -text
 LCU/Maintenance/MDB_tools/test/t_probe_mdb.sh -text
+LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog.py -text
+LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog.run -text
+LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog.sh -text
+LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.py -text
+LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.run -text
+LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.sh -text
 LCU/Maintenance/__init__.py -text
 LCU/PPSTune/CMakeLists.txt -text
 LCU/PPSTune/MANIFEST.in -text

LCU/Maintenance/DBInterface/monitoringdb/models/log.py

@@ -2,7 +2,7 @@ import django.db.models as models
 
 
 class ActionLog(models.Model):
-    entry_id = models.IntegerField()
+    entry_id = models.IntegerField(help_text='database id of the concerned object')
     model_name = models.CharField(max_length=100)
     who = models.CharField(max_length=100)
     when = models.DateTimeField(auto_now_add=True)

LCU/Maintenance/DBInterface/monitoringdb/models/station.py

@@ -3,6 +3,6 @@ from .fixed_types import STATION_TYPES
 
 
 class Station(models.Model):
-    location = models.CharField(max_length=50, null=True, blank=True)
-    name = models.CharField(max_length=10)
-    type = models.CharField(max_length=1, choices=STATION_TYPES)
+    location = models.CharField(max_length=50, null=True, blank=True, help_text='where the station is located')
+    name = models.CharField(max_length=10, help_text='name of the station')
+    type = models.CharField(max_length=1, choices=STATION_TYPES, help_text='station type one of [Core[C], Remote[R], International[I]]')

LCU/Maintenance/DBInterface/monitoringdb/urls.py

@@ -33,14 +33,16 @@ rtsm_router.register(r'', RTSMObservationViewSet)
 
 
 urlpatterns = [
+
+
     url(r'^api/stationtests/', include(station_test_router.urls)),
     url(r'^api/rtsm/', include(rtsm_router.urls)),
+    url(r'^api/log/', include(log_router.urls)),
 
     url(r'^api/api-auth', include('rest_framework.urls', namespace='rest_framework')),
     url(r'^api/stationtests/raw/insert', insert_raw_station_test),
     url(r'^api/rtsm/raw/insert', insert_raw_rtsm_test),
-
-    url(r'^api/log/', include(log_router.urls)),
+    url(r'^api/view/ctrl_stationoverview', ControllerStationOverview.as_view()),
 
     url(r'^api/docs', include_docs_urls(title='Monitoring DB API'))
 ]

LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py

 from rest_framework.viewsets import ReadOnlyModelViewSet
+from rest_framework.schemas import SchemaGenerator
 from lofar.maintenance.monitoringdb.serializers.log import ActionLogSerializer, ActionLog
 
 
 class ActionLogViewSet(ReadOnlyModelViewSet):
+    """
+    Action event log line
+
+    """
     queryset = ActionLog.objects.all()
     serializer_class = ActionLogSerializer
     filter_fields = '__all__'

LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py

@@ -77,13 +77,13 @@ def insert_raw_rtsm_test(request):
                 logger.exception("raw station info malformed %s: %s",
                                  request.data['content'], e)
                 logger.info('station test malformed skipping insertion.')
-                return Response(status=status.HTTP_200_OK, data='skipping insertion RTSM malformed')
+                return Response(status=status.HTTP_406_NOT_ACCEPTABLE, data='skipping insertion RTSM malformed')
             except Exception as e:
                 logger.exception("exception occurred while parsing raw station info %s: %s",
                                  request.data['content'], e)
                 return Response(exception=True,
                                 data="the post message is not correct." +
-                                     " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
+                                     " It has to be of the form {content:[RAWSTRING], station_name:[STATION_NAME]}: %s. Request provided %s" % (
                                      e, request),
                                 status=status.HTTP_400_BAD_REQUEST)
         else:

LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py

 from django_filters import rest_framework as filters
 from rest_framework.viewsets import ReadOnlyModelViewSet
+from rest_framework.views import APIView
+import coreapi, coreschema
+from rest_framework.schemas import ManualSchema
+
 
 from .common import *
 from ..models.component import Component
@@ -9,78 +13,101 @@ from ..models.element_error import ElementError
 from ..models.station import Station
 from ..models.station_test import StationTest
 from ..serializers.component import ComponentSerializer
+from collections import OrderedDict
 from ..serializers.component_error import ComponentErrorSerializer
 from ..serializers.element import ElementSerializer
 from ..serializers.element_error import ElementErrorSerializer
 from ..serializers.station import StationSerializer
 from ..serializers.station_tests import StationTestSerializer
 from ..station_test_raw_parser import parse_raw_station_test
-
+from django.db.models import Count
 logger = logging.getLogger('views')
 
 
-class ComponentViewSet(ReadOnlyModelViewSet):
-    serializer_class = ComponentSerializer
-    queryset = Component.objects.all()
-    filter_fields = '__all__'
-
-
 class StationTestFilterSet(filters.FilterSet):
     station_name = filters.CharFilter(field_name='station__name',
-                                      lookup_expr='contains')
+                                      lookup_expr='contains', help_text='name of the station')
     station_type = filters.CharFilter(field_name='station__type',
-                                      lookup_expr='contains')
+                                      lookup_expr='contains',
+                                      help_text='selects the station type:' +
+                                                'one of [Core[C], Remote[R], International[I]]')
 
-    from_date = filters.DateFilter(field_name='start_datetime', lookup_expr='gt')
-    to_date = filters.DateFilter(field_name='end_datetime', lookup_expr='lt')
+    from_date = filters.DateFilter(field_name='start_datetime', lookup_expr='gt',
+                                   help_text='select station tests from date time')
+    to_date = filters.DateFilter(field_name='end_datetime', lookup_expr='lt',
+                                 help_text='select station tests until date time')
 
 
 class ComponentErrorFilterSet(filters.FilterSet):
-    component_id = filters.NumberFilter(field_name='component', lookup_expr='component_id')
-    component_type = filters.CharFilter(field_name='component', lookup_expr='type')
+    component_id = filters.NumberFilter(field_name='component', lookup_expr='component_id',
+                                        help_text='select by component id')
+    component_type = filters.CharFilter(field_name='component', lookup_expr='type',
+                                        help_text='select by component type')
 
     station_name = filters.CharFilter(field_name='station_test__station',
-                                      lookup_expr='name__contains')
+                                      lookup_expr='name__contains',
+                                      help_text='station name with name like')
     station_type = filters.CharFilter(field_name='station_test__station',
-                                      lookup_expr='type__contains')
+                                      lookup_expr='type__contains',
+                                      help_text='station type like')
 
-    from_date = filters.DateFilter(field_name='station_test', lookup_expr='start_datetime__gt')
-    to_date = filters.DateFilter(field_name='station_test', lookup_expr='end_datetime__lt')
+    from_date = filters.DateFilter(field_name='station_test', lookup_expr='start_datetime__gt',
+                                   help_text='select component errors from date time')
+    to_date = filters.DateFilter(field_name='station_test', lookup_expr='end_datetime__lt',
+                                 help_text='select component errors until date time')
     type = filters.CharFilter(field_name='type',
-                              lookup_expr='contains')
+                              lookup_expr='contains',
+                              help_text='component error type')
 
 
 class ElementErrorFilterSet(filters.FilterSet):
-    component_id = filters.NumberFilter(field_name='component_error__component', lookup_expr='component_id')
-    element_id = filters.NumberFilter(field_name='element', lookup_expr='element_id')
+    component_id = filters.NumberFilter(field_name='component_error__component',
+                                        lookup_expr='component_id',
+                                        help_text='id of the parent component')
+    element_id = filters.NumberFilter(field_name='element', lookup_expr='element_id',
+                                      help_text='element id')
     station_name = filters.CharFilter(field_name='component_error__station_test__station',
-                                      lookup_expr='name__contains')
+                                      lookup_expr='name__contains',
+                                      help_text='name of the station')
     station_type = filters.CharFilter(field_name='component_error__station_test__station',
-                                      lookup_expr='type__contains')
+                                      lookup_expr='type__contains',
+                                      help_text='station type')
 
-    from_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='start_datetime__gt')
-    to_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='end_datetime__lt')
+    from_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='start_datetime__gt',
+                                   help_text='select element errors from date time')
+    to_date = filters.DateFilter(field_name='component_error__station_test', lookup_expr='end_datetime__lt',
+                                   help_text='select element errors until date time')
 
     type = filters.CharFilter(field_name='type',
-                              lookup_expr='contains')
-
-
-class ComponentErrorViewSet(ReadOnlyModelViewSet):
-    serializer_class = ComponentErrorSerializer
-    queryset = ComponentError.objects.all().order_by('-station_test__start_datetime',
-                                                     'type',
-                                                     'component__component_id',
-                                                     'component__station__name')
-    filter_fields = '__all__'
-    filterset_class = ComponentErrorFilterSet
+                              lookup_expr='contains', help_text='element error type')
 
 
 class StationViewSet(ReadOnlyModelViewSet):
+    """
+    retrieve:
+    retrieve a specific station from the database
+
+    list:
+    list all the stations present in the database
+    """
     serializer_class = StationSerializer
     queryset = Station.objects.all()
     filter_fields = '__all__'
 
 
+class ComponentViewSet(ReadOnlyModelViewSet):
+
+    serializer_class = ComponentSerializer
+    queryset = Component.objects.all()
+    filter_fields = '__all__'
+
+
+class ElementViewSet(ReadOnlyModelViewSet):
+    serializer_class = ElementSerializer
+    queryset = Element.objects.all()
+    filter_fields = '__all__'
+
+
 class StationTestViewSet(ReadOnlyModelViewSet):
     serializer_class = StationTestSerializer
     queryset = StationTest.objects.all().order_by('-start_datetime', 'station__name')
@@ -88,6 +115,16 @@ class StationTestViewSet(ReadOnlyModelViewSet):
     filterset_class = StationTestFilterSet
 
 
+class ComponentErrorViewSet(ReadOnlyModelViewSet):
+    serializer_class = ComponentErrorSerializer
+    queryset = ComponentError.objects.all().order_by('-station_test__start_datetime',
+                                                     'type',
+                                                     'component__component_id',
+                                                     'component__station__name')
+    filter_fields = '__all__'
+    filterset_class = ComponentErrorFilterSet
+
+
 class ElementErrorViewSet(ReadOnlyModelViewSet):
     serializer_class = ElementErrorSerializer
     queryset = ElementError.objects.all().order_by('-component_error__station_test__start_datetime',
@@ -98,22 +135,13 @@ class ElementErrorViewSet(ReadOnlyModelViewSet):
     filterset_class = ElementErrorFilterSet
 
 
-class ElementViewSet(ReadOnlyModelViewSet):
-    serializer_class = ElementSerializer
-    queryset = Element.objects.all()
-    filter_fields = '__all__'
-
-
 @api_view(['POST'])
 def insert_raw_station_test(request):
     """
     This function is meant to parse a request of the form
-    {
-    "content": "[STATION TEST RAW TEXT]"
-    }
+    { content: [STATION TEST RAW TEXT]   }
     parse the content field and create all the station_test entity related into the database
-    :param request: HTTP request
-    :return:
+
     """
     if request.method == 'POST':
 
@@ -146,3 +174,86 @@ def insert_raw_station_test(request):
         logger.info('Request processed correctly. Processed station test ids: %s', station_test_ids)
         return Response(status=status.HTTP_200_OK,
                         data='Station tests inserted with ids {} \n'.format(station_test_ids))
+
+
+class ControllerStationOverview(APIView):
+    """
+    Overview of the latest tests performed on the stations
+    """
+
+    DEFAULT_STATION_GROUP = 'ALL'
+    DEFAULT_ONLY_ERRORS = False
+    DEFAULT_N_STATION_TESTS = 4
+    DEFAULT_N_RTSM = 4
+
+    queryset = StationTest.objects.all()
+    schema = ManualSchema(fields=[
+        coreapi.Field(
+            "station_group",
+            required=False,
+            location='query',
+            schema=coreschema.Enum(['C', 'R', 'I', 'ALL'], description=
+                                     'Station group to select for choices are [C|R|I|ALL]',
+                                   )
+        ),
+        coreapi.Field(
+            "n_station_tests",
+            required=False,
+            location='query',
+            schema=coreschema.Integer(description='number of station tests to select',
+                                      minimum=1)
+        ),
+        coreapi.Field(
+            "n_rtsm",
+            required=False,
+            location='query',
+            schema=coreschema.Integer(description='number of station tests to select',
+                                      minimum=1)
+        ),
+        coreapi.Field(
+            "errors_only",
+            required=False,
+            location='query',
+            schema=coreschema.Boolean(description=
+                                      'displays or not only the station with more than one error')
+        )
+    ]
+    )
+
+
+    def get(self, request, format=None):
+
+        print(request.validate())
+
+        errors_only = request.query_params.get('errors_only', self.DEFAULT_ONLY_ERRORS)
+        print(request.query_params)
+        station_group = request.query_params.get('station_group', self.DEFAULT_STATION_GROUP)
+        n_station_tests = int(request.query_params.get('n_station_tests', self.DEFAULT_N_STATION_TESTS))
+        n_rtsm = request.query_params.get('n_rtsm', self.DEFAULT_N_RTSM)
+
+        if station_group is not 'ALL':
+            station_entities = Station.objects.all()
+
+        # Since django preferes a ordered dict over a dict we make it happy... for now
+        response_payload = OrderedDict()
+        for station_entity in station_entities:
+
+            station_test_list = StationTest.objects.filter(
+                station__name=station_entity.name).order_by('-end_datetime')[:n_station_tests-1]
+            response_payload[station_entity.name] = list()
+            for station_test in station_test_list:
+                station_test_payload = OrderedDict()
+                component_errors = station_test.component_errors
+
+                station_test_payload['total_component_errors'] = station_test.component_errors.count()
+                station_test_payload['start_datetime'] = station_test.start_datetime
+                station_test_payload['end_datetime'] = station_test.end_datetime
+                station_test_payload['checks'] = station_test.checks
+                component_errors_summary = component_errors.\
+                    values('component__type', 'type').annotate(
+                    total=Count('type')).order_by('-total')
+                station_test_payload['STUFF'] = [(item['component__type'], item['type'], item['total']) for item in component_errors_summary]
+
+                response_payload[station_entity.name].append(station_test_payload)
+
+        return Response(status=status.HTTP_200_OK, data=response_payload)
\ No newline at end of file

LCU/Maintenance/MDB_WebView/maintenancedb_view/src/pages/DetailsPage.js

 import React, { Component } from 'react';
 import Header from '../components/header.js'
 
+var data = [
+  'cs001': 2,
+  'cs002': 2,
+  'cs003': 2,
+]
 class DetailsPage extends Component {
+
   render() {
+    var list = data.map((item) => <div>{item}</div>);
+
     return (
       <div>
         <Header active_page={this.props.location} />
-        <div>Details Overview</div>
+        <div>Details Overview!</div>
+        <div>{list}</div>
       </div>
     );
   }

LCU/Maintenance/MDB_tools/CMakeLists.txt

@@ -8,18 +8,21 @@ include(PythonInstall)
 include(FindPythonModule)
 
 #Required packages
-find_python_module(requests)
-find_python_module(beautifultable)
-find_python_module(blessings)
+find_python_module(beautifultable REQUIRED)
+find_python_module(blessings REQUIRED)
+find_python_module(inotify REQUIRED)
 
 set(_py_files
   cli/__init__.py
   cli/mdb_loader.py
-  cli/probe_mdb.py)
+  cli/probe_mdb.py
+  cli/station_tests_watchdog.py)
 
 set(_bin_files
   bin/mdb_loader.py
-  bin/probe_mdb.py)
+  bin/probe_mdb.py
+  bin/station_tests_watchdog.py
+  )
 
 
 python_install(${_py_files} DESTINATION lofar/maintenance/utils)
@@ -29,4 +32,6 @@ install(PROGRAMS ${_bin_files}
     DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
 
 
+add_subdirectory(lib)
+
 add_subdirectory(test)

LCU/Maintenance/MDB_tools/bin/mdb_loader.py

@@ -11,8 +11,6 @@ This program is meant to load the station tests and RTSM present in a certain di
 """
 
 
-
-
 if __name__ == '__main__':
     logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
     main()

LCU/Maintenance/MDB_tools/bin/station_tests_watchdog.py

+#!/usr/bin/env python
+
+import logging
+
+from lofar.maintenance.utils.cli.station_tests_watchdog import main
+
+logger = logging.getLogger('station test watchdog')
+
+"""
+This program is meant to load the station tests and RTSM present in a certain directory to the database if a new
+test output file is created
+"""
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

LCU/Maintenance/MDB_tools/cli/mdb_loader.py

@@ -4,12 +4,11 @@ This program is meant to load the station tests and RTSM present in a certain di
 """
 import argparse
 import logging
-import re
 import sys
 from glob import glob
-import os.path as path
-
-import requests
+from os import path
+from lofar.maintenance.mdb.client import read_stationtest_rtsm_output_to_dict,\
+    send_stationtest_rtsm_content_to_address
 
 logger = logging.getLogger('mdb_loader')
 
@@ -39,119 +38,6 @@ def obtain_file_list(file_path):
     return file_list
 
 
-def read_stationtest_rtsm_output_to_dict(file_path):
-    """
-    Reads the content of the stationtest or RTSM output file into a dict
-    -> ex. {'content': [file_content]}.
-    If the file contains a RTSM it also append the station name
-    -> ex. {'content': [file_content], 'station_name': CS001C}
-    :param file_path: path to the file
-    :return: return a dict containing a content field if the file refers to a station test ->
-    ex. {'content': [file_content]}
-    or if the file_path refers to a RTSM it returns a dict containing the content of the file and
-    the station name to which the test refers
-    ex. {'content': [file_content], 'station_name'}
-    """
-    rtsm_filename_pattern = '.*(?P<station_name>\w+)_\d+_\d+\.dat$'
-    try:
-        with open(file_path, 'r') as input_stream:
-            result = dict(content=input_stream.read().strip('\n'), type='stationtest')
-            if re.match(rtsm_filename_pattern,
-                        file_path):  # Matches the file name to understand if it is a RTSM
-                result.update(station_name=
-                              re.search(rtsm_filename_pattern, file_path).group('station_name'),
-                              type='rtsm')
-                print(re.search(rtsm_filename_pattern, file_path).group('station_name'))
-
-            return result
-    except Exception as e:
-        logging.error('cannot read file %s', file_path)
-        logging.exception(e)
-        return None
-
-
-def is_station_test(content):
-    """
-    Check if the content of the file have the structure of a station test
-    :param content: list of strings representing the content of the file
-    :return: True if the content refers to a station test
-    """
-    pattern = r'(?:\d{8})(?:\,.*)*'
-    for line in content.split('\n'):
-        if line is '':
-            continue
-        if not re.match(pattern, line):
-            return False
-    return True
-
-
-def is_rtsm_test(content):
-    """
-    Check if the content of the file have the structure of a RTSM
-    :param content: list of strings representing the content of the file
-    :return: True if the content refers to a RTSM
-    """
-    pattern = r'(?:#\.*)|(?:[^0-9\,]*=.*)'
-    for line in content.split('\n'):
-        if line is '':
-            continue
-        if not re.match(pattern, line):
-            return False
-    return True
-
-
-def send_stationtest_rtsm_content_to_address(address, content):
-    """
-    Send the stationtest RTSM content to the web site API at the given address
-    :param address: url where the API is hosted
-    :param content: content of the API call
-    :return: True if the request was successful False otherwise
-    """
-    full_address = '/'.join([address, compose_api_url_for_given_test_type(content['content'])])
-    logging.info('performing request to address %s', full_address)
-
-    logging.debug('request content %s', content)
-    response = requests.post(full_address, data=content)
-    logging.info('response acknowledged: status code is %s, reason %s, content %s',
-                 response.status_code,
-                 response.reason,
-                 response.content)
-    if response.status_code == 200:
-        return True
-    else:
-        logging.error('error sending request %s', response.reason)
-        return False
-
-
-def compose_api_url_for_given_test_type(content):
-    """
-    Create the url from the content type
-    :return: the query needed to insert the raw results
-    """
-    if is_station_test(content):
-        query = 'stationtests/insert_raw'
-    elif is_rtsm_test(content):
-        query = 'rtsm/insert_raw'
-    else:
-        raise ValueError('The file content is not a RTSM nor a station test output.')
-    return query
-
-
-def split_history_into_tests(content):
-    all_tests = []
-    current_test = []
-    content = content.split('\n')
-    for i, line in enumerate(content[:-1]):
-        next_line_columns = content[i + 1].split(',')
-        line_type = next_line_columns[3]
-        current_test.append(line)
-        if 'VERSIONS' in line_type or ('STATION' in line_type and ('VERSIONS' not in current_test[0])):
-            all_tests.append('\n'.join(current_test))
-            current_test = []
-    all_tests.append('\n'.join(current_test))
-    return all_tests
-
-
 def obtain_stationtest_file_list_and_process(file_path_pattern, address):
     """
     Finds all the stationtest files which path satisfy the file_path_pattern and send them to address that implement the
@@ -161,28 +47,15 @@ def obtain_stationtest_file_list_and_process(file_path_pattern, address):
     """
     for file_path in obtain_file_list(file_path_pattern):
         logging.info('processing file %s', file_path)
-        test_output = read_stationtest_rtsm_output_to_dict(file_path)
-        if test_output is None:
+        content = read_stationtest_rtsm_output_to_dict(file_path)
+        if content is None:
             logger.error('error processing file %s', file_path)
             continue
-        elif test_output['type'] == 'stationtest':
-            tests = split_history_into_tests(test_output['content'])
-            for i, test in enumerate(tests):
-                if send_stationtest_rtsm_content_to_address(address, dict(content=test)):
-                    logger.info('processing file %s: loading test %d of %d',
-                                file_path,
-                                i + 1,
-                                len(tests))
-                else:
-                    logger.error('error on file %s', file_path)
-                    sys.exit(1)
-
-        elif test_output['type'] == 'rtsm':
-            if send_stationtest_rtsm_content_to_address(address, test_output):
-                logger.info('file %s processed', file_path)
-            else:
-                logger.error('error on file %s', file_path)
-                sys.exit(1)
+        if send_stationtest_rtsm_content_to_address(address, content):
+            logger.info('file %s processed', file_path)
+        else:
+            logger.error('error on file %s', file_path)
+            sys.exit(1)
 
 
 def main():

LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py

+import logging
+import sys
+import os
+import re
+from argparse import ArgumentParser
+
+from lofar.maintenance.mdb import client as mdb_client
+import inotify.adapters
+logger = logging.getLogger('station_tests_watchdog')
+
+MAIN_LOOP_WAIT_TIME = 1
+
+
+def setup_logging_framework(logging_level=logging.INFO):
+    """
+    Setup the logging style for a desired logging_level
+    :param logging_level: logging level lower of which the messages will be filtered out
+    """
+    logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s",
+                        level=logging_level)
+
+
+def station_tests_watchdog(args):
+    """
+    Main function
+    :param args: command line arguments
+    """
+    settings = get_settings_from_command_arguments(args)
+    apply_global_settings(settings)
+    observer = instantiate_file_creation_event_observer(settings.path)
+    main_wait_event_loop(observer, settings.expected_filename, settings.addr, MAIN_LOOP_WAIT_TIME)
+
+
+def get_settings_from_command_arguments(args):
+    """
+    Get the settings from the the command arguments
+    :param args: command line arguments
+    :return: the station_test_watchdog settings
+    """
+    argument_parser = setup_command_argument_parser()
+    settings = argument_parser.parse_args(args)
+    return settings
+
+
+def setup_command_argument_parser():
+    """
+    Setup the command arguments parser with the station_test_watchdog different options
+    and returns it
+    :return: ArgumentParser with the station_test_watchdog options
+    """
+    parser = ArgumentParser(description="Station tests watchdog listens for new file events on a"
+                                        "specified directory and if the file is a stationtest"
+                                        "or a RTSM the file is loaded to the MaintenanceDB")
+    parser.add_argument('path', help="file system path to monitor for new file events")
+    parser.add_argument('expected_filename', help="Considers events only on file with a name"
+                                                  "that matches the specified regular expression")
+
+    parser.add_argument('--addr', help='MaintenanceDB http address',
+                        default='lofarmonitortest.control.lofar')
+
+    parser.add_argument('-v', help='verbose logging', action='store_false', default=False)
+
+    return parser
+
+
+def apply_global_settings(settings):
+    """
+    Apply the settings to the global variables (e.g. the logging level)
+    :param settings: command line argument settings
+    :return:
+    """
+    if settings.v:
+        setup_logging_framework(logging.DEBUG)
+
+
+def instantiate_file_creation_event_observer(path):
+    """
+    Create an observer object that listens to creation events of station test
+    and RTSM in a specific dir and sends the content of these files to the REST api of
+    the MaintenanceDB at the given address
+    :param path: the dir path to watch for events
+    :return: the observer object
+    """
+    inotify_adapter = inotify.adapters.Inotify()
+    inotify_adapter.add_watch(path)
+    return inotify_adapter
+
+
+def main_wait_event_loop(observer,
+                         expected_filename,
+                         mdb_address,
+                         active_watch_time,
+                         run_once=False):
+    """
+    Wait for any exception raised
+    if it receives a keyboard interrupt it stops the listener
+    :param observer: inotify interface observer
+    :param expected_filename: regex that describes the expected filename
+    :param mdb_address: url of the MaintenanceDB REST API
+    :param active_watch_time: time of active watching of inotify events
+    :param run_once: watch for inotify events only once and timeout after
+     active_watch_time
+    """
+    while True:
+        try:
+            events = observer.event_gen(yield_nones=False, timeout_s=active_watch_time)
+            for event in events:
+                handle_inotify_event_create_file(event, expected_filename, mdb_address)
+            if run_once:
+                break
+        except KeyboardInterrupt:
+            logger.info("Keyboard interrupt. Exiting main listener")
+            return True
+        except Exception as e:
+            logger.info('Unexpected exception occurred: %s. \n Exiting main listener', e)
+
+
+def handle_inotify_event_create_file(event, expected_filename, mdb_address):
+        """
+        On file creation event handler
+        :param mdb_address: url of the MaintenanceDB REST API
+        :param event: the list that describes the creation event
+        """
+        if _is_file_created(event) and _does_filename_matches_specified_regex(expected_filename, event):
+            logger.debug('File created handling event %s', event)
+
+            handle_event_file_created(mdb_address, event)
+        else:
+            logger.debug("Neglecting non file creation event : %s", event)
+
+
+def _does_filename_matches_specified_regex(regex, event):
+    """
+    Checks whether the file name of the event matched the specified regex
+    :param regex: desired regex
+    :param event: inotify event
+    :return: true if the file name matches the regex
+    """
+    return bool(re.match(regex, event[3]))
+
+
+def _is_file_created(event):
+    """
+    Checks if the creation event refers to a file
+    :param event: the list that describes the creation event
+    :return: True if it refers to a file being created False otherwise
+    """
+    return 'IN_CLOSE_WRITE' in event[1]
+
+
+def handle_event_file_created(mdb_address, event):
+    """
+    Handle the file created event sending the file to the MaintenanceDB REST API.
+    If the file content doesnt match a RTSM or a station tests, it logs the event and skips
+    the insertion.
+
+    :param mdb_address: url of the MaintenanceDB REST API
+    :param event: inotify event list description
+    """
+    (_, type_name, path, filename) = event
+    file_path = os.path.join(path, filename)
+
+    try:
+        file_content = mdb_client.read_stationtest_rtsm_output_to_dict(file_path)
+        mdb_client.send_stationtest_rtsm_content_to_address(mdb_address, file_content)
+    except ValueError:
+        logger.info('file %s is neither a station test nor a RTSM output. skipping insertion',
+                    file_path)
+    except Exception as e:
+        logger.error('error processing file %s: %s', file_path, e)
+
+
+def main():
+    setup_logging_framework()
+    station_tests_watchdog(sys.argv)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

LCU/Maintenance/MDB_tools/lib/CMakeLists.txt

+
+find_python_module(requests)
+
+set(_py_files
+    __init__.py
+    client.py
+  )
+
+python_install(${_py_files} DESTINATION lofar/maintenance/mdb)
+

LCU/Maintenance/MDB_tools/lib/client.py

+import logging
+import requests
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def read_stationtest_rtsm_output_to_dict(file_path):
+    """
+    Reads the content of the stationtest or RTSM output file into a dict
+    -> ex. {'content': [file_content]}.
+    If the file contains a RTSM it also append the station name
+    -> ex. {'content': [file_content], 'station_name': CS001C}
+    :param file_path: path to the file
+    :return: return a dict containing a content field if the file refers to a station test ->
+    ex. {'content': [file_content]}
+    or if the file_path refers to a RTSM it returns a dict containing the content of the file and the station name
+    to which the test refers
+    ex. {'content': [file_content], 'station_name'} or None if it cannot read the file
+    """
+    try:
+        with open(file_path, 'r') as input_stream:
+            rtsm_filename_pattern = '(?P<station_name>\w*)_\d+_\d+\.dat$'
+            result = dict(content=input_stream.read().strip('\n'))
+            if re.search(rtsm_filename_pattern, file_path):  # Matches the filename to understand if it is a RTSM
+                result.update(station_name=re.search(rtsm_filename_pattern, file_path).group('station_name'))
+            return result
+    except Exception as e:
+        logging.error('cannot read file %s', file_path)
+        logging.exception(e)
+        return None
+
+
+def is_station_test(content):
+    """
+    Check if the content of the file have the structure of a station test.
+    Example content
+20130522,NFO,---,STATION,NAME=CS011C
+20130522,NFO,---,RUNTIME,START=03:10:00,STOP=05:52:27
+20130522,NFO,---,CHECKS,S1,O1,SP1,NS1=180,S3,O3,SP3,NS3=180,M,O5,SN,SP5,NS5=300,EO,ESP,EN=30,TV,TM
+20130522,NFO,---,STATISTICS,BAD_LBL=1,BAD_LBH=2,BAD_HBA=7
+20130522,LBL,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=82.0,SUBBANDY=301,SIGNALY=81.8
+20130522,LBL,019,RF_FAIL,X=77.0,Y=70.1
+20130522,LBL,019,LOW_NOISE,Xproc=100.000,Xval=66.6,Xdiff=0.0,Xref=67.7
+20130522,LBH,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=81.8,SUBBANDY=301,SIGNALY=82.2
+20130522,LBH,013,RF_FAIL,Y=72.9
+20130522,LBH,013,LOW_NOISE,Yproc=100.000,Yval=66.0,Ydiff=0.1,Yref=67.8
+20130522,LBH,015,RF_FAIL,X=68.0
+20130522,LBH,015,LOW_NOISE,Xproc=100.000,Xval=67.8,Xdiff=0.0,Xref=68.0
+20130522,HBA,003,E_FAIL,HNX13=64.3 2.3,JX13=1.0
+20130522,HBA,009,E_FAIL,HNX8=62.7 0.9
+20130522,HBA,010,HIGH_NOISE,Xproc=1.333,Xval=73.3,Xdiff=3.9,Xref=70.8,Yproc=0.500,Yval=71.5,Ydiff=3.1,Yref=70.9
+20130522,HBA,010,JITTER,Xproc=92.278,Xdiff=3.9,Xref=0.1,Yproc=99.611,Ydiff=3.1,Yref=0.2
+20130522,HBA,010,E_FAIL,HNY3=63.7 2.5,JY3=1.0,HNY12=64.0 2.5,JY12=1.0
+20130522,HBA,015,HIGH_NOISE,Xproc=8.278,Xval=74.2,Xdiff=5.6,Xref=70.8
+20130522,HBA,015,JITTER,Xproc=98.500,Xdiff=5.6,Xref=0.1
+20130522,HBA,015,SUMMATOR_NOISE,Y=1
+20130522,HBA,015,E_FAIL,HNX12=66.4 5.5,JX12=1.0
+20130522,HBA,019,JITTER,Xproc=99.333,Xdiff=1.9,Xref=0.1
+20130522,HBA,026,RF_FAIL,X=106.4 357 121.0 107.8 357 121.2
+20130522,HBA,036,SUMMATOR_NOISE,X=1,Y=1
+20130522,HBA,036,E_FAIL,SPX9=1,SPY9=1,SPX10=1,SPY10=1,SPX13=1,SPY13=1,SPX14=1,SPY14=1
+    :param content: list of strings representing the content of the file
+    :return: True if the content refers to a RTSM and it is not empty
+    """
+    if content:
+        pattern = r'(^\d*,[A-z]{3},[\d-]{3})'
+        number_of_lines = len(content.splitlines())
+        pattern_matching_lines = re.findall(pattern, content, re.MULTILINE)
+        number_pattern_matching_lines = len(pattern_matching_lines)
+        return number_of_lines == number_pattern_matching_lines
+    else:
+        return False
+
+
+def is_rtsm_test(content):
+    """
+    Check if the content of the file have the structure of a RTSM
+    Example content of a valid RTSM test
+# SPECTRA-INFO=rcu,rcumode,obs-id,check,startfreq,stopfreq,rec-timestamp
+#
+SPECTRA-INFO=4,5,641999,SN,100,200,1520208664.423739
+MEAN-SPECTRA=[0.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.1 60.0 60.0 60.1 60.1 60.1 60.2 60.1 60.0 60.1 60.0 60.9 60.7 60.1 60.9 60.0 60.3 60.1 60.0 61.5 60.1 61.0 62.9 73.4 60.9 64.0 61.7 62.4 70.3 64.5 62.4 72.7 68.6 64.0 61.8 62.0 62.3 62.5 62.7 62.9 63.1 63.5 63.7 63.9 64.0 64.1 64.3 64.6 64.6 64.6 64.8 64.9 65.1 65.2 65.3 65.3 65.3 65.5 65.6 65.7 65.7 65.8 66.0 66.2 66.3 66.3 66.4 66.6 66.8 66.8 66.9 66.9 66.9 67.1 67.2 67.2 67.2 67.2 67.4 67.5 67.6 67.5 67.6 67.7 67.9 68.0 68.0 68.0 68.1 68.2 68.3 68.4 68.4 68.4 68.6 68.7 68.7 68.7 68.6 68.7 68.8 68.9 68.8 68.8 68.9 68.9 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.1 69.1 69.1 69.0 69.1 69.1 69.2 69.2 69.1 69.1 69.6 69.2 69.2 69.1 69.1 69.1 69.1 69.1 69.0 69.0 69.0 68.9 69.0 68.9 68.9 68.8 68.8 68.9 68.9 68.8 68.7 68.7 68.8 68.9 68.8 68.7 68.8 68.8 68.8 68.8 68.7 68.7 68.7 68.8 68.7 68.7 68.6 68.7 68.8 68.7 68.7 68.5 68.6 68.7 68.7 68.7 68.6 68.5 68.6 68.6 68.6 68.5 68.4 68.5 68.5 68.4 68.4 76.1 71.0 68.4 68.4 68.4 68.3 68.3 68.3 68.3 68.3 68.2 68.2 68.2 68.2 68.3 68.1 68.0 68.0 67.9 67.9 67.9 67.9 67.8 67.8 67.8 67.8 67.8 67.7 67.6 67.7 67.8 67.7 67.6 67.6 67.6 67.7 67.7 67.6 67.5 67.5 67.6 67.5 67.5 67.4 67.4 67.5 67.5 67.5 67.3 67.3 67.5 67.5 67.5 67.4 67.4 67.4 67.5 67.4 67.4 67.3 67.3 67.4 67.4 67.3 67.3 67.5 67.3 67.4 67.3 67.3 67.2 67.3 67.3 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.3 67.2 67.3 67.6 67.3 67.4 67.4 67.5 67.5 67.5 67.5 67.6 67.6 67.7 67.7 67.7 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.9 67.9 67.9 67.9 67.9 68.0 68.0 68.0 68.1 68.2 68.2 68.2 68.2 68.2 68.3 68.4 68.3 68.3 68.3 68.4 68.4 68.4 68.2 68.1 68.2 68.2 68.2 68.1 68.1 68.1 68.1 68.1 68.1 68.0 68.0 68.1 68.1 68.0 67.9 67.8 67.9 67.9 67.9 67.8 67.8 67.8 67.9 67.9 67.8 67.7 67.7 67.8 67.8 67.7 67.7 67.6 67.7 67.7 67.6 67.6 67.7 67.6 67.6 67.5 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.3 67.4 67.4 67.4 67.4 67.4 67.3 67.4 67.3 67.3 67.3 67.3 67.3 67.3 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.0 67.0 66.9 66.9 79.8 85.4 85.4 85.2 85.7 85.4 85.1 85.9 83.0 66.3 66.2 66.2 66.2 66.1 66.0 65.9 65.9 65.9 65.8 65.7 65.6 65.6 65.6 65.5 65.4 65.3 65.3 83.8 92.0 91.5 91.0 90.5 90.9 90.3 90.5 88.3 65.0 64.7 64.7 64.6 64.5 64.4 64.4 64.3 64.2 64.1 64.0 63.9 63.8 63.7 63.5 63.4 63.5 63.4 63.3 63.1 63.0 62.8 62.7 62.6 62.5 62.3 62.2 62.1 62.2 61.9 61.8 61.6 61.6 61.5 61.4 61.3 68.2 69.7 69.5 69.5 68.8 68.5 68.9 67.8 64.1 60.9 60.9 60.9 60.8 60.9 60.8 60.8 60.7 60.6 60.6 60.5 60.4 60.3 60.2 60.2 60.1 60.1 60.0 60.0 60.1 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.1 60.4 60.3 ]
+BAD-SPECTRA=[0.0 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.8 59.8 59.8 59.8 59.8 59.8 60.3 59.8 59.8 59.8 59.8 63.1 62.5 59.8 61.3 59.9 60.2 59.9 59.8 63.6 59.9 61.0 60.6 60.4 61.8 68.8 64.5 62.4 72.1 66.5 63.5 72.9 71.5 66.2 62.7 64.2 64.6 64.1 66.0 66.1 66.1 66.9 68.2 68.4 67.4 69.5 69.3 68.2 70.3 69.7 69.7 69.1 70.7 70.6 69.3 71.2 70.6 69.6 71.1 70.5 70.6 69.3 70.9 70.9 69.6 71.4 71.0 70.4 71.0 71.3 70.8 69.2 70.4 69.8 68.8 70.1 69.9 69.8 69.9 71.1 71.0 69.9 71.6 71.2 70.3 71.7 71.1 70.9 70.2 71.1 70.8 69.8 70.8 70.4 70.1 70.6 70.5 70.4 69.7 70.6 70.5 69.8 70.5 70.2 69.8 70.0 70.3 70.1 69.5 70.2 70.0 69.6 70.2 69.9 69.7 69.6 70.1 69.9 69.5 69.9 69.8 69.6 70.0 69.7 69.6 69.4 69.9 69.7 69.5 69.8 69.6 69.6 69.9 69.8 69.9 69.5 70.0 70.0 69.7 70.3 70.1 70.0 70.1 70.6 70.6 69.9 70.8 70.6 70.0 70.9 70.6 70.3 69.9 70.5 70.2 69.6 70.0 69.6 69.3 69.7 69.5 69.3 69.0 69.2 69.2 68.9 68.9 68.7 68.6 68.8 68.8 68.6 68.5 68.5 68.6 68.6 68.6 68.5 68.4 68.5 68.6 68.5 68.4 79.2 73.0 68.4 68.5 68.4 68.3 68.3 68.5 68.4 68.4 68.6 68.5 68.4 68.6 68.6 68.5 68.3 68.5 68.4 68.2 68.6 68.4 68.2 68.3 68.4 68.4 68.1 68.3 68.2 67.9 68.2 68.0 67.8 67.7 67.7 67.7 67.6 67.6 67.4 67.3 67.5 67.5 67.3 67.2 67.3 67.3 67.3 67.2 67.1 67.1 67.2 67.3 67.2 67.1 67.1 67.1 67.2 67.2 67.1 67.0 67.1 67.1 67.2 67.0 67.0 67.3 67.1 67.2 67.1 67.0 67.0 67.1 67.2 67.1 67.1 67.1 67.1 67.3 67.2 67.1 67.2 67.2 67.3 67.2 67.3 67.3 67.3 67.5 67.4 67.4 67.5 67.5 67.6 67.6 67.6 67.8 67.7 67.8 67.9 67.8 67.9 67.8 67.8 67.8 67.7 67.8 67.7 67.7 67.9 67.7 67.7 67.8 67.8 67.9 67.9 67.9 68.0 68.0 68.1 68.0 68.0 68.3 68.2 68.3 68.2 68.3 68.5 68.4 68.4 68.2 68.1 68.2 68.1 68.1 68.0 67.9 68.0 67.9 67.9 67.9 67.8 67.9 67.9 67.8 67.8 67.7 67.6 67.7 67.7 67.7 67.6 67.6 67.6 67.6 67.6 67.5 67.4 67.4 67.5 67.5 67.4 67.4 67.3 67.4 67.4 67.3 67.3 67.5 67.3 67.3 67.3 67.3 67.2 67.2 67.2 67.3 67.2 67.2 67.2 67.2 67.2 67.3 67.2 67.3 67.2 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.1 67.0 67.0 67.0 67.0 67.0 66.9 66.9 66.9 84.6 90.8 91.0 91.1 90.9 91.4 91.4 91.2 89.1 66.3 66.3 66.3 66.2 66.1 66.0 66.0 66.0 66.0 65.9 65.8 65.7 65.7 65.7 65.6 65.5 65.4 65.4 84.9 90.8 91.5 90.2 90.7 90.9 90.1 90.5 88.6 65.0 64.8 64.8 64.7 64.6 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.5 63.4 63.5 63.4 63.3 63.1 62.9 62.8 62.6 62.6 62.4 62.3 62.1 62.0 62.0 61.8 61.8 61.6 61.5 61.4 61.3 61.3 64.0 65.2 65.9 65.6 66.5 67.0 66.9 67.1 62.9 60.9 60.9 60.9 60.8 60.8 60.8 60.7 60.7 60.6 60.6 60.5 60.4 60.3 60.2 60.1 60.0 60.0 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.8 59.9 59.9 59.8 59.9 60.1 60.2 60.3 ]
+
+SPECTRA-INFO=4,5,641999,SN,100,200,1520208724.588988
+MEAN-SPECTRA=[0.0 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.9 59.8 59.7 59.7 60.3 60.9 60.3 59.8 60.7 59.7 60.0 59.7 59.7 61.1 59.9 60.6 62.7 73.3 60.6 63.7 61.4 62.3 70.5 64.5 62.3 72.8 68.4 64.5 61.7 61.9 62.3 62.4 64.5 62.8 63.0 63.4 63.6 63.8 63.9 64.0 64.3 64.5 64.6 64.7 64.8 64.9 65.2 65.3 65.4 65.4 65.4 65.5 65.7 65.7 65.7 65.8 66.0 66.2 66.3 66.3 66.4 66.4 66.7 66.8 66.8 66.8 66.8 67.1 67.2 67.2 67.2 67.2 67.4 67.5 67.6 67.6 67.7 67.8 68.0 68.1 68.1 68.0 68.2 68.3 68.4 68.4 68.3 68.3 68.6 68.6 68.6 68.6 68.6 68.7 68.8 69.0 69.2 68.8 68.8 68.9 69.0 69.0 69.0 69.0 69.0 69.1 69.2 69.1 69.1 69.1 69.2 69.2 69.2 69.1 69.2 69.2 69.2 69.2 69.2 69.1 69.6 69.2 69.2 69.2 69.1 69.1 69.1 69.1 69.0 68.9 68.9 68.9 69.0 69.0 68.9 68.8 68.9 68.9 69.0 68.9 68.8 68.8 68.9 68.9 68.9 68.8 68.8 68.8 68.9 68.9 68.7 68.7 68.7 68.8 68.7 68.6 68.6 68.6 68.8 68.7 68.6 68.5 68.5 68.7 68.7 68.6 68.5 68.5 68.6 68.5 68.6 68.4 68.4 68.5 68.5 68.4 68.4 76.0 71.1 68.5 68.5 68.4 68.3 68.3 68.3 68.3 68.3 72.2 68.2 68.2 68.2 68.3 68.1 68.1 68.0 67.9 68.0 67.9 67.9 67.8 67.8 67.8 67.8 67.7 67.7 67.7 67.7 67.7 67.7 67.6 67.6 67.5 67.6 67.6 67.5 67.4 67.5 67.5 67.5 67.4 67.3 67.3 67.4 67.5 67.3 67.3 67.2 67.5 67.4 67.4 67.3 67.2 67.3 67.3 67.4 67.3 67.2 67.2 67.3 67.4 67.3 67.3 67.5 67.3 67.4 67.3 67.3 67.2 67.3 67.4 67.4 67.9 67.3 67.4 67.5 67.4 67.4 67.4 67.4 67.4 67.5 67.3 67.5 67.4 67.5 67.5 67.5 67.5 67.5 67.5 67.6 67.6 67.6 67.7 67.7 67.7 67.7 67.7 67.7 68.1 67.8 67.7 67.8 67.8 67.9 67.9 67.9 67.9 67.9 68.0 68.0 68.1 68.1 68.2 68.2 68.2 68.1 68.1 68.2 68.3 68.3 68.2 68.2 68.3 68.3 68.3 68.1 68.1 68.1 68.2 68.2 68.1 68.1 68.1 68.2 68.2 68.1 68.0 68.0 68.1 68.1 68.0 67.9 67.8 67.9 67.9 67.9 67.8 67.8 67.8 67.8 67.8 67.8 67.7 67.7 67.7 67.8 67.7 67.7 67.7 67.7 67.8 91.2 125.0 67.8 67.6 67.6 67.6 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.5 67.5 67.4 67.4 67.4 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.3 67.3 67.3 67.3 67.2 67.1 67.1 67.1 67.1 67.0 67.0 66.9 66.9 80.0 85.6 85.5 85.5 85.7 85.6 85.3 85.8 83.1 66.3 66.3 66.3 66.2 66.2 66.0 66.0 66.0 65.9 65.8 65.8 65.7 65.6 65.6 65.9 65.4 65.3 65.3 83.5 91.8 91.4 91.0 90.4 90.8 90.1 90.2 88.2 65.0 64.7 64.7 64.6 64.5 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.5 63.6 63.5 63.4 63.2 63.1 62.9 62.8 62.7 62.6 62.4 62.3 62.1 62.1 62.1 61.9 61.8 61.6 61.5 61.4 61.4 61.3 68.3 69.9 69.7 69.7 69.0 68.6 68.8 67.9 64.1 60.9 60.8 60.8 60.7 60.7 60.7 60.7 60.5 60.5 60.4 60.4 60.3 60.2 60.2 60.1 60.0 60.0 60.0 59.9 60.0 60.0 60.0 59.9 59.9 59.9 59.9 59.9 59.9 59.8 59.8 59.8 59.8 59.9 60.0 ]
+BAD-SPECTRA=[0.0 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 60.1 59.6 59.5 59.5 60.5 63.3 61.7 59.5 61.0 59.6 60.0 59.6 59.5 63.4 59.6 60.9 60.4 60.0 61.6 68.9 64.1 62.3 72.0 66.4 63.4 73.0 71.4 67.0 62.7 64.2 64.7 64.0 67.6 66.1 66.2 66.8 68.3 68.4 67.3 69.5 69.3 68.2 70.3 69.7 69.8 69.1 70.8 70.7 69.3 71.3 70.7 69.7 71.1 70.7 70.6 69.3 71.0 70.9 69.6 71.4 71.0 70.3 70.9 71.3 70.8 69.2 70.4 69.8 68.8 70.2 70.0 69.9 70.0 71.2 71.1 70.0 71.7 71.3 70.4 71.7 71.1 70.9 70.1 71.0 70.7 69.7 70.7 70.2 69.9 70.5 70.4 70.3 69.6 70.6 70.5 69.9 70.7 70.2 69.9 70.0 70.4 70.2 69.6 70.3 70.1 69.7 70.4 70.0 69.9 69.8 70.2 70.0 69.6 70.0 69.8 69.6 70.0 69.7 69.6 69.4 69.9 69.6 69.5 69.7 69.5 69.5 69.8 69.7 69.8 69.4 70.0 70.0 69.6 70.3 70.1 70.0 70.1 70.6 70.7 70.0 70.9 70.6 70.1 71.0 70.7 70.4 70.0 70.6 70.2 69.6 70.1 69.7 69.4 69.7 69.5 69.3 68.9 69.2 69.1 68.8 68.9 68.6 68.6 68.7 68.6 68.5 68.4 68.4 68.5 68.5 68.5 68.4 68.4 68.4 68.6 68.5 68.4 75.4 70.5 68.4 68.6 68.5 68.4 68.4 68.6 68.5 68.4 70.5 68.5 68.5 68.7 68.6 68.6 68.3 68.6 68.5 68.3 68.6 68.4 68.2 68.3 68.4 68.4 68.1 68.3 68.1 67.9 68.1 68.0 67.7 67.6 67.7 67.6 67.5 67.5 67.3 67.3 67.4 67.4 67.2 67.1 67.1 67.2 67.3 67.1 67.0 67.0 67.2 67.2 67.1 67.0 67.0 67.1 67.2 67.2 67.1 67.1 67.1 67.2 67.2 67.1 67.1 67.4 67.2 67.3 67.1 67.1 67.1 67.2 67.3 67.1 67.4 67.2 67.2 67.3 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.3 67.3 67.5 67.4 67.4 67.4 67.4 67.5 67.4 67.5 67.6 67.5 67.7 67.6 67.6 67.6 67.6 67.8 67.7 67.7 67.8 67.7 67.8 67.9 67.8 67.8 67.9 67.9 68.0 67.9 68.0 68.0 68.0 68.2 68.0 67.9 68.2 68.1 68.2 68.0 68.0 68.2 68.1 68.2 67.9 67.9 68.0 68.0 68.0 67.9 67.9 67.9 68.0 68.0 67.9 67.8 67.9 67.9 67.9 67.9 67.7 67.7 67.7 67.7 67.7 67.6 67.5 67.6 67.6 67.6 67.5 67.4 67.4 67.5 67.5 67.4 67.5 67.4 67.5 67.5 94.6 128.3 67.6 67.4 67.4 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.1 67.0 67.0 67.0 67.0 67.0 66.9 66.9 85.0 90.9 91.1 91.3 91.2 91.7 91.7 91.4 89.3 66.4 66.3 66.3 66.3 66.2 66.0 66.0 66.0 66.0 65.9 65.9 65.7 65.7 65.7 66.3 65.6 65.4 65.4 84.9 90.9 91.4 90.5 90.6 91.1 90.1 90.6 88.4 65.0 64.8 64.8 64.7 64.6 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.6 64.2 63.5 63.4 63.3 63.1 62.9 62.8 62.7 62.6 62.4 62.3 62.1 62.1 62.0 61.9 61.7 61.6 61.5 61.4 61.4 61.3 64.0 65.2 65.8 65.6 66.4 66.9 66.8 67.0 62.8 60.8 60.8 60.8 60.7 60.7 60.6 60.6 60.5 60.5 60.4 60.3 60.2 60.2 60.1 60.0 59.9 59.9 59.8 59.8 59.8 59.8 59.8 59.8 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.8 59.9 ]
+
+# OBS-ID-INFO=obsid,start_time,stop_time,obsid_samples
+OBS-ID-INFO=641999,1520208240.000,1520211840.000,59
+    :param content: list of strings representing the content of the file
+    :return: True if the content refers to a RTSM and it is not empty
+    """
+    if content:
+        pattern = r'(^#.*)|(^\w*-\w*=)|(^\w*-\w*-\w*=)|(^$)'
+        number_of_lines = len(content.splitlines())
+        pattern_matching_lines = re.findall(pattern, content, re.MULTILINE)
+        number_pattern_matching_lines = len(pattern_matching_lines)
+        return number_of_lines == number_pattern_matching_lines
+    else:
+        return False
+
+
+def send_stationtest_rtsm_content_to_address(address, content):
+    """
+    Send the stationtest RTSM content to the web site API at the given address
+    :param address: url where the API is hosted
+    :param content: content of the API call
+    :return: True if the request was successful False otherwise
+    """
+    full_address = '/'.join([address, compose_api_url_for_given_test_type(content['content'])])
+    logger.info('performing request to address %s', full_address)
+
+    logger.debug('request content %s', content)
+    response = requests.post(full_address, data=content)
+    logger.info('response acknowledged: status code is %s, reason %s, content %s', response.status_code,
+                 response.reason,
+                 response.content)
+    if response.status_code == 200:
+        return True
+    else:
+        logger.error('error sending request %s', response.reason)
+        return False
+
+
+def compose_api_url_for_given_test_type(content):
+    """
+    Create the url from the content type
+    :return: the query needed to insert the raw results
+    """
+    if is_station_test(content):
+        query = 'stationtests/raw/insert'
+    elif is_rtsm_test(content):
+        query = 'rtsm/raw/insert'
+    else:
+        raise ValueError('The path format must refer to either an RTSM or a station test.')
+    return query
\ No newline at end of file

LCU/Maintenance/MDB_tools/test/CMakeLists.txt

@@ -6,5 +6,9 @@ file(COPY
 
 lofar_add_test(t_mdb_loader)
 lofar_add_test(t_probe_mdb)
+lofar_add_test(t_client)
+lofar_add_test(t_station_tests_watchdog)
+lofar_add_test(t_station_tests_watchdog_integration)
+
 
 

LCU/Maintenance/MDB_tools/test/t_client.py

+import unittest
+
+from mock import patch, MagicMock, Mock, call
+
+from lofar.maintenance.mdb.client import *
+
+logger = logging.getLogger(__name__)
+
+
+def load_test_file(file_name):
+    with open(file_name, 'r') as f_read_stream:
+        content = f_read_stream.read().strip('\n')
+    return content
+
+
+def load_rtsm_test():
+    file_name = 't_client.in.test_rtsm.data'
+    return load_test_file(file_name)
+
+
+def load_station_test():
+    file_name = 't_client.in.test_stationtest.data'
+    return load_test_file(file_name)
+
+
+class TESTMDBClient(unittest.TestCase):
+    def test_is_station_test_right_content(self):
+        station_test_content = load_station_test()
+        self.assertTrue(is_station_test(station_test_content))
+
+    def test_is_station_test_wrong_content(self):
+        station_test_content = load_rtsm_test()
+        self.assertFalse(is_station_test(station_test_content))
+
+    def test_is_rtsm_test_right_content(self):
+        rtsm_test_content = load_rtsm_test()
+        self.assertTrue(is_rtsm_test(rtsm_test_content))
+
+    def test_is_rtsm_test_wrong_content(self):
+        rtsm_test_content = load_station_test()
+        self.assertFalse(is_rtsm_test(rtsm_test_content))
+
+    @patch('requests.post')
+    def test_send_stationtest_rtsm_content_to_address_success(self, mocked_post):
+        station_test_content = load_station_test()
+        address = 'http://testaddress'
+        response = Mock(status_code=200, reason='ALL GOOD', content='YEAH')
+        mocked_post.return_value = response
+
+        full_address = '/'.join([address, compose_api_url_for_given_test_type(station_test_content)])
+        result = send_stationtest_rtsm_content_to_address('http://testaddress', station_test_content)
+        self.assertTrue(mocked_post.called)
+        mocked_post.assert_called_with(full_address, data=station_test_content)
+        self.assertTrue(result)
+
+    @patch('requests.post')
+    def test_send_stationtest_rtsm_content_to_address_failure(self, mocked_post):
+        station_test_content = load_station_test()
+        address = 'http://testaddress'
+        response = Mock(status_code=400, reason='Bad Request', content='Sorry')
+        mocked_post.return_value = response
+
+        full_address = '/'.join([address, compose_api_url_for_given_test_type(station_test_content)])
+        result = send_stationtest_rtsm_content_to_address('http://testaddress', station_test_content)
+        self.assertTrue(mocked_post.called)
+        mocked_post.assert_called_with(full_address, data=station_test_content)
+        self.assertFalse(result)
+
+    def test_compose_api_url_for_given_test_type_is_rtsm(self):
+        rtsm_test_content = load_rtsm_test()
+        self.assertEqual(compose_api_url_for_given_test_type(rtsm_test_content), 'rtsm/insert_raw')
+
+    def test_compose_api_url_for_given_test_type_is_station_test(self):
+        station_test_content = load_station_test()
+        self.assertEqual(compose_api_url_for_given_test_type(station_test_content), 'stationtests/insert_raw')
+
+    def test_compose_api_url_for_given_test_type_raises_content_unrecognized(self):
+        with self.assertRaises(ValueError):
+            compose_api_url_for_given_test_type('Neque porro quisquam est qui dolorem ipsum quia'
+                                                ' dolor sit amet'
+                                                'consectetur, adipisci velit...')
+
+
+    @patch('lofar.maintenance.mdb.client.open', create=True)
+    def test_read_stationtest_rtsm_output_to_dict_rtsm_success(self, open_mock):
+        file_name = 'CS001C_12345_20180305.dat'
+        file_content = 'testline1\ntestline2'
+        open_mock.return_value = MagicMock()
+        file_handle = open_mock.return_value.__enter__.return_value
+
+        file_handle.read.return_value = file_content
+
+        result = read_stationtest_rtsm_output_to_dict(file_name)
+        open_mock.assert_called_with(file_name, 'r')
+        file_handle.read.assert_called()
+        self.assertEqual(result, dict(content=file_content, station_name='CS001C'))
+
+    @patch('lofar.maintenance.mdb.client.open', create=True)
+    def test_read_stationtest_rtsm_output_to_dict_rtsm_failure(self, open_mock):
+        file_name = 'CS001C_12345_20180305.dat'
+        open_mock.return_value = MagicMock()
+
+        file_handle = open_mock.return_value.__enter__.return_value
+
+        file_handle.read.side_effect = IOError('Cannot read file')
+
+        result = read_stationtest_rtsm_output_to_dict(file_name)
+        open_mock.assert_called_with(file_name, 'r')
+        file_handle.read.assert_called()
+        self.assertIsNone(result)
+
+    @patch('lofar.maintenance.mdb.client.open', create=True)
+    def test_read_stationtest_rtsm_output_to_dict_station_test_success(self, open_mock):
+        file_name = 'CS031C_L2_StationTestHistory.csv'
+        file_content = 'testline1\ntestline2'
+        open_mock.return_value = MagicMock()
+        file_handle = open_mock.return_value.__enter__.return_value
+
+        file_handle.read.return_value = file_content
+
+        result = read_stationtest_rtsm_output_to_dict(file_name)
+        open_mock.assert_called_with(file_name, 'r')
+        file_handle.read.assert_called()
+        self.assertEqual(result, dict(content=file_content))
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
+    unittest.main()