diff --git a/.gitattributes b/.gitattributes
index ea9df6aed0c1507dc691b9d30174895c1bba1bab..b1d38bc670c26015e96cc56ead3e7829a47a519d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1900,6 +1900,11 @@ LCU/Maintenance/DBInterface/test/models/t_wincc_models.sh -text
LCU/Maintenance/DBInterface/test/models/test_rtsm_models.py -text
LCU/Maintenance/DBInterface/test/models/test_wincc_models.py -text
LCU/Maintenance/DBInterface/test/postgres_testrunner.py -text
+LCU/Maintenance/Docker/BaseMaintenance/Dockerfile -text
+LCU/Maintenance/Docker/DBInterface/Dockerfile -text
+LCU/Maintenance/Docker/RESTService/Dockerfile -text
+LCU/Maintenance/Docker/RESTService/entrypoint.sh -text
+LCU/Maintenance/Docker/docker-compose.yml -text
LCU/Maintenance/MDB_WebView/CMakeLists.txt -text
LCU/Maintenance/MDB_WebView/maintenancedb_view/.env -text
LCU/Maintenance/MDB_WebView/maintenancedb_view/CMakeLists.txt -text
@@ -2034,6 +2039,7 @@ LCU/Maintenance/MDB_tools/fabfile.py -text
LCU/Maintenance/MDB_tools/lib/CMakeLists.txt -text
LCU/Maintenance/MDB_tools/lib/__init__.py -text
LCU/Maintenance/MDB_tools/lib/client.py -text
+LCU/Maintenance/MDB_tools/lib/tests_validators.py -text
LCU/Maintenance/MDB_tools/requirements.txt -text
LCU/Maintenance/MDB_tools/test/output_testing.py -text
LCU/Maintenance/MDB_tools/test/python-coverage.sh eol=lf
diff --git a/LCU/Maintenance/DBInterface/CMakeLists.txt b/LCU/Maintenance/DBInterface/CMakeLists.txt
index 64e35b8fb80d746185de6670eb5d9075f973e9d0..f942add6ed7089c4c445e191f2c2262685e715d4 100644
--- a/LCU/Maintenance/DBInterface/CMakeLists.txt
+++ b/LCU/Maintenance/DBInterface/CMakeLists.txt
@@ -3,7 +3,7 @@
set(USE_PYTHON_COMPILATION Off)
lofar_find_package(Python 3.0 REQUIRED)
-lofar_package(DBInterface 1.0 DEPENDS PyCommon)
+lofar_package(DBInterface 1.0 DEPENDS PyCommon MDB_tools)
include(PythonInstall)
@@ -23,10 +23,9 @@ foreach(LIST_ITEM ${PY_FILES})
MESSAGE(STATUS ${LIST_ITEM})
endforeach(LIST_ITEM ${PY_FILES})
-
-
add_subdirectory(bin)
add_subdirectory(test)
python_install(${PY_FILES} DESTINATION lofar/maintenance)
+include(CPack)
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
index 76c618c60fd6ea277dd3d139a48cd37d3d9aac1b..51e3d479e96fa951be8d879ea3b85930266182b8 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py
@@ -194,6 +194,9 @@ USE_TZ = True
PATH = os.getcwd()
STATIC_ROOT = os.path.join(PATH, "static")
STATIC_URL = '/static/'
+MB_CONST = 1024 ** 3
+
+DATA_UPLOAD_MAX_MEMORY_SIZE = 500 * MB_CONST
REST_FRAMEWORK = {
'DEFAULT_PAGINATION_CLASS': 'lofar.maintenance.monitoringdb.pagination.DefaultPaginationSettings',
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/component.py b/LCU/Maintenance/DBInterface/monitoringdb/models/component.py
index 7995893b53995ea3b76898608b6e042bc4c9c4aa..7ad4f71c8d2bd8edc59eba7ff2d9062f1e7d05d7 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/component.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/component.py
@@ -1,7 +1,8 @@
-from django.db import models
-from .station import Station
from typing import Tuple
+from django.db import models
+
+from .station import Station
HBA_ELEMENT_ID_RANGE = range(0, 16, 1)
@@ -14,7 +15,7 @@ def antenna_type_antenna_id_to_component_type_component_id(station_name,
if station_name.startswith('CS') or station_name.startswith('RS'):
if antenna_id >= 48:
component_type = 'LBL'
- component_id = antenna_id - 48
+ component_id = antenna_id
else:
component_type = 'LBH'
component_id = antenna_id
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
index af3101d631b893ab752620ee42390a8b1cb843a6..a9f957a3d21a6defefb5fa1fe2058995bc3f77b9 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py
@@ -1,8 +1,8 @@
"""
In this file all the model regarding the real time station monitor output are collected
"""
-from django.db import models
from django.contrib.postgres.fields import ArrayField
+from django.db import models
from .component import Component
from .test import GenericTest
@@ -17,6 +17,14 @@ MODE_TO_COMPONENT = {
7: 'HBA'
}
+MODE_TO_FREQ_RANGE = {1: (10, 90),
+ 2: (30, 90),
+ 3: (10, 90),
+ 4: (30, 90),
+ 5: (110, 190),
+ 6: (170, 230),
+ 7: (210, 250)}
+
def antenna_id_polarization_from_rcu_type(rcu, type):
"""
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/signals/generic_test_insert.py b/LCU/Maintenance/DBInterface/monitoringdb/models/signals/generic_test_insert.py
index 9a4ae82847bb4178251aab50210a6b4f0743192a..eb7e6e9e84ec957f3d6e862930934a964b84b9af 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/signals/generic_test_insert.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/signals/generic_test_insert.py
@@ -1,3 +1,4 @@
+import logging
from typing import List
from ..component import HBA_ELEMENT_ID_RANGE, Component, \
@@ -8,6 +9,8 @@ from ..wincc import ElementStatus, \
ComponentStatus, \
StationStatus, WinCCAntennaStatus, STATUS_CODE_TO_STATUS, latest_status_per_station_raw
+logger = logging.getLogger(__name__)
+
def insert_hba_element_status(component_status: ComponentStatus,
hba_wincc_status: WinCCAntennaStatus):
@@ -65,16 +68,27 @@ def insert_station_status(test_instance: GenericTest, statuses: List[WinCCAntenn
station_status_instance.save()
test_instance.station_status = station_status_instance
+ test_instance.save()
for status in statuses:
insert_component_status(station_status_instance, status)
+def insert_station_status_per_test(test: GenericTest):
+ queryset = WinCCAntennaStatus.objects.all().order_by()
+ query_results = list(latest_status_per_station_raw(queryset=queryset,
+ station=test.station.name,
+ to_date=test.start_datetime))
+
+ insert_station_status(test, query_results)
+
+
+from django.db.models import signals
def on_inserting_test(sender, instance: GenericTest, **kwargs):
+
logger.info('received post_insert signal from %s for instance %s', sender, instance)
- queryset = WinCCAntennaStatus.objects.all().order_by()
- query_results = [wincc_status for wincc_status in
- latest_status_per_station_raw(queryset=queryset,
- station=instance.station.name,
- to_date=instance.start_datetime)]
+ signals.post_save.disconnect(on_inserting_test, sender=sender)
- insert_station_status(instance, query_results)
+ try:
+ insert_station_status_per_test(instance)
+ finally:
+ signals.post_save.connect(on_inserting_test, sender=sender)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/test.py b/LCU/Maintenance/DBInterface/monitoringdb/models/test.py
index b7565bde2e2c2d96c244861448e78d1cead3b48c..5e62a5f96dc2b3aa02e18089e1257bf157eed3b3 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/test.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/test.py
@@ -15,5 +15,5 @@ class GenericTest(models.Model):
station = models.ForeignKey(Station, on_delete=models.DO_NOTHING, related_name='tests')
station_status = models.OneToOneField(StationStatus,
- on_delete=models.CASCADE,
+ on_delete=models.SET_NULL,
null=True)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/wincc.py b/LCU/Maintenance/DBInterface/monitoringdb/models/wincc.py
index 0e9d2bef37bb535f50a92d47e3a580f08affef8f..3d4f9b9a84190f88a47cefd0166b095d06cd306d 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/models/wincc.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/models/wincc.py
@@ -3,6 +3,7 @@ from collections import OrderedDict
import django.db.models as models
from .component import Component
+from .component import antenna_type_antenna_id_to_component_type_component_id
from .element import Element
from .station import Station
@@ -29,7 +30,8 @@ def _format_result(raw_query_result, antenna_type):
results = OrderedDict()
for row in raw_query_result:
status_description = OrderedDict()
- results[str(row.antenna_id)] = status_description
+ antenna_id = row.antenna_id
+ results[str(antenna_id)] = status_description
status_description['inserted_at'] = row.last_entry_timestamp
status_description['status'] = to_status(row.status_code)
status_description['status_code'] = row.status_code
@@ -59,8 +61,6 @@ def latest_status_per_station_raw(queryset, station, to_date):
:param queryset: source queryset
:param station: station name
:type station: str
- :param component_type: component type name
- :type component_type: str
:param to_date: select the latest test before the to date
:type to_date: datetime.dattime
:return: the status of an antenna per antenna id
@@ -71,7 +71,7 @@ def latest_status_per_station_raw(queryset, station, to_date):
INNER JOIN (
SELECT antenna_type, antenna_id, MAX("timestamp") last_entry_timestamp
FROM antenna_statuses
- WHERE timestamp <= %(to_date)s AND station=%(station_name)s
+ WHERE status_code_modified_at <= %(to_date)s AND station=%(station_name)s
GROUP BY antenna_type, antenna_id
) slave ON master.antenna_type=slave.antenna_type AND master.antenna_id = slave.antenna_id AND last_entry_timestamp=master.timestamp
WHERE master.station=%(station_name)s
@@ -98,11 +98,12 @@ def latest_status_per_station_and_antenna_type(queryset, station, antenna_type,
INNER JOIN (
SELECT antenna_type, antenna_id, MAX("timestamp") last_entry_timestamp
FROM antenna_statuses
- WHERE timestamp <= %s AND station=%s
+ WHERE status_code_modified_at <= %s AND station=%s
GROUP BY antenna_type, antenna_id
- ) slave ON master.station=slave.station AND master.antenna_type=slave.antenna_type AND master.antenna_id = slave.antenna_id AND last_entry_timestamp=master.timestamp
+ ) slave ON master.antenna_type=slave.antenna_type AND master.antenna_id = slave.antenna_id AND last_entry_timestamp=master.timestamp
WHERE master.station=%s AND master.antenna_type=%s
- ''', [to_date, station.rstrip('C'), antenna_type])
+ ''', [to_date, station.rstrip('C'), station.rstrip('C'), antenna_type])
+
return _format_result(raw_results, antenna_type)
@@ -121,8 +122,13 @@ def latest_status_per_station_and_component_type(queryset, station, component_ty
"""
antenna_type = to_antenna_type(component_type)
if antenna_type:
- return latest_status_per_station_and_antenna_type(queryset, station, antenna_type,
- to_date)
+
+ results = latest_status_per_station_and_antenna_type(queryset, station, antenna_type,
+ to_date)
+ return {antenna_id: results[antenna_id] for antenna_id in results
+ if antenna_type_antenna_id_to_component_type_component_id(station, component_type,
+ int(antenna_id))[
+ 0] == component_type}
else:
return OrderedDict()
@@ -278,10 +284,12 @@ class StationStatus(models.Model):
class ComponentStatus(EntityStatus):
- station_status = models.ForeignKey(StationStatus, on_delete=models.CASCADE)
+ station_status = models.ForeignKey(StationStatus, on_delete=models.CASCADE,
+ related_name='component_status')
component = models.ForeignKey(Component, on_delete=models.DO_NOTHING)
class ElementStatus(EntityStatus):
- component_status = models.ForeignKey(ComponentStatus, on_delete=models.CASCADE)
+ component_status = models.ForeignKey(ComponentStatus, on_delete=models.CASCADE,
+ related_name='element_status')
element = models.ForeignKey(Element, on_delete=models.DO_NOTHING)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
index f1ff3df1e2950c13a42ca847009b0416260e4113..bf9ac68f0b22055c50d82b812ba77713b84ea3fa 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py
@@ -1,6 +1,10 @@
+import logging
from datetime import datetime
+
import pytz
+logger = logging.getLogger('serializers')
+
rtsm_error_types_to_error_types = dict(HN='HIGH_NOISE',
SN='SUMMATOR_NOISE',
@@ -12,6 +16,7 @@ rtsm_error_types_to_error_types = dict(HN='HIGH_NOISE',
DOWN='DOWN',
SHORT='SHORT')
+
def split_raw_rtsm_test(content):
"""
Split the raw rtsm test in row and filters out comments and empty lines
@@ -20,7 +25,7 @@ def split_raw_rtsm_test(content):
results = filter(lambda x: not x.startswith('#'),
filter(lambda x: len(x) > 0,
map(lambda x: x.strip(),
- content.split('\n')
+ content.splitlines()
)
))
return results
@@ -51,7 +56,8 @@ def preparse_rtsm_test_file(content):
results.append(result)
result = dict()
- rcu, mode, observation_id, error_type, start_frequency, stop_frequency, time = value.split(',')
+ rcu, mode, observation_id, error_type, start_frequency, stop_frequency, time = value.split(
+ ',')
result.update(rcu=int(rcu),
mode=int(mode),
error_type=rtsm_error_types_to_error_types[error_type],
@@ -79,6 +85,7 @@ def group_samples_by_error_type(content):
errors = content.pop('errors')
grouped_errors = dict()
for error in errors:
+ print(error)
key = (error['mode'], error['rcu'], error['error_type'])
time = error.pop('time')
average_spectrum = error.pop('average_spectrum')
@@ -91,8 +98,8 @@ def group_samples_by_error_type(content):
grouped_errors[key]['samples'] += [sample]
else:
grouped_errors[key] = dict(
- count = 1,
- samples = [sample],
+ count=1,
+ samples=[sample],
**error
)
content['errors'] = list(grouped_errors.values())
@@ -106,4 +113,4 @@ def parse_rtsm_test(content):
:param content: string content with the station test output
:return: a list of Django models
"""
- return group_samples_by_error_type(preparse_rtsm_test_file(split_raw_rtsm_test(content)))
\ No newline at end of file
+ return group_samples_by_error_type(preparse_rtsm_test_file(split_raw_rtsm_test(content)))
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/wincc.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/wincc.py
index cef8b708b4b1e95830c5df82ff06f46b0801921b..69b8d7a107ed11a2d134713e88af15808f98d9fa 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/wincc.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/wincc.py
@@ -1,8 +1,61 @@
-from ..models.wincc import WinCCAntennaStatus
+from collections import OrderedDict
+
from .utils import NotNullModelSerializer
+from ..models.wincc import WinCCAntennaStatus, StationStatus, ComponentStatus, ElementStatus
+from ..serializers.component import ComponentSerializer
class WinCCAntennaStatusSerializer(NotNullModelSerializer):
class Meta:
model = WinCCAntennaStatus
fields = '__all__'
+
+
+class ElementStatusSerializer(NotNullModelSerializer):
+ class Meta:
+ model = ElementStatus
+ fields = '__all__'
+
+
+class ComponentStatusSerializer(NotNullModelSerializer):
+ component = ComponentSerializer()
+
+ class Meta:
+ model = ComponentStatus
+ fields = '__all__'
+
+
+class SummaryComponentStatusSerializer(NotNullModelSerializer):
+ component = ComponentSerializer()
+
+ class Meta:
+ model = ComponentStatus
+ fields = ['status_code', 'status_description', 'status_message', 'modified_at']
+
+
+class StationStatusSerializer(NotNullModelSerializer):
+ component_status = ComponentStatusSerializer()
+
+ class Meta:
+ model = StationStatus
+ fields = '__all__'
+
+
+from typing import List
+
+
+def serialize_station_status(component_statuses: List[ComponentStatus], type: str):
+ station_status = OrderedDict()
+ for component_status in component_statuses:
+ component_id = component_status.component.component_id
+ component_type = component_status.component.type
+ if component_type != type:
+ continue
+ station_status[str(component_id)] = dict(
+ status_code=component_status.status_code,
+ status=component_status.status_description,
+ reason=component_status.status_message,
+ last_modified=component_status.modified_at
+ )
+
+ return station_status
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
index 9d69bbad43de598d55d20086cac1afed45c490ee..f6fd4b5b0209b7bd5d2a483ce2c9f38e02be418c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/station_test_raw_parser.py
@@ -2,13 +2,15 @@
This modules contains all the function needed to parse a raw station test output into the models used to describe it
"""
import logging
+import re
+from collections import defaultdict
from datetime import datetime
import pytz
-import re
-from collections import defaultdict
-logger = logging.getLogger('station_test.parser')
+logger = logging.getLogger(__name__)
+
+
def station_type_from_station_name(station_name):
@@ -59,6 +61,15 @@ def parse_datetime(date, date_time):
datetime.strptime("T".join([date, date_time]), '%d-%m-%YT%H:%M:%S'))
+def split_lines_and_filter_comments(content):
+ split_lines = content.splitlines()
+
+ while split_lines.count(''):
+ split_lines.pop('')
+ split_lines = list(filter(lambda line: not line.startswith('#'), split_lines))
+ return split_lines
+
+
def parse_raw_station_test(content):
"""
Expects a string content with the station test output
@@ -66,14 +77,14 @@ def parse_raw_station_test(content):
:param content: string content with the station test output
:return: a list of Django models
"""
- logger.debug('content splitted %s', content.split('\n'))
- station_tests = split_history_into_tests(content.split('\n'))
+
+ station_tests = split_history_into_tests(split_lines_and_filter_comments(content))
logger.debug('station_tests_are %s', station_tests)
results = []
for stest in station_tests:
dict_stest = dict_from_raw_station_test(stest)
- logger.debug('parsed test %s', dict_stest)
- if dict_stest['station']['name'] == 'Unknown':
+ logger.debug('parsed test %s' % dict_stest)
+ if 'station' not in dict_stest or dict_stest['station']['name'] == 'Unknown':
logger.error('error in the station name for test %s', dict_stest)
continue
if 'start_datetime' in dict_stest:
@@ -87,8 +98,7 @@ def split_history_into_tests(content):
all_tests = []
current_test = []
for i, line in enumerate(content[:-1]):
- if line.startswith('#'):
- continue
+
next_line_columns = content[i + 1].split(',')
line_type = next_line_columns[3]
current_test.append(line)
@@ -186,7 +196,6 @@ def dict_from_component_error(content):
component = dict(component_id=parse_component_id(component_id), type=component_type.strip())
-
component_error = dict(component=component,
details=error_details,
type=error_type.strip())
@@ -270,7 +279,6 @@ element_error_name_mapping = {
'': 'RF_FAIL'
}
-
def parse_rffail_string(polarization, rf_string):
"""
Parse the string for the rffail test into a dict
@@ -286,8 +294,8 @@ def parse_rffail_string(polarization, rf_string):
LBA_RF_FAIL_NPARAMETERS = 1
if HBA_RF_FAIL_NPARAMETERS == len(parameters):
- measured_signal_nodelay, measured_signal_fulldelay, subband_used_nodelay,\
- subband_used_fulldelay,\
+ measured_signal_nodelay, measured_signal_fulldelay, subband_used_nodelay, \
+ subband_used_fulldelay, \
reference_signal_nodelay, reference_signal_fulldelay = map(float,
rf_string.replace('nan', '999').
replace('-1', '999').
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py
index f3b77f45522a247176da100bab8b5aaff7674198..557b12ab0b779c8c6e01590f2e8f9660c678f902 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tasks/generate_plots.py
@@ -1,13 +1,10 @@
import logging
import os
-try:
+import matplotlib
- import matplotlib
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
- matplotlib.use('agg')
- import matplotlib.pyplot as plt
-except ImportError:
- pass
import numpy
from celery import shared_task
from django.conf import settings
@@ -167,6 +164,7 @@ def produce_plot(observation_metadata,
transform=plt.axes().transAxes)
plt.legend()
plt.savefig(path)
+ logger.info('saving plot in %s', path)
plt.close()
@@ -263,6 +261,7 @@ def check_error_summary_plot(error_summary_id):
elif os.path.isdir(full_path):
raise Exception('%s is a directory' % full_path)
else:
- logger.debug('summary error %s is complete no need to generate additional plot',
- summary_plot.pk)
+ logger.info(
+ 'summary error %s is complete no need to generate additional plot. File in %s',
+ summary_plot.pk, full_path)
return summary_plot.pk
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py b/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py
index 7cdf93672d6c0c8984623d755bbb4490b162a8b3..9da82a858d478af5b39084a72593f2a88cb5c93c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/tasks/insert_raw_tests.py
@@ -6,31 +6,25 @@ from django.db import transaction
# Station Models
from lofar.maintenance.monitoringdb.models.component import Component
-from lofar.maintenance.monitoringdb.models.station import Station
+from lofar.maintenance.monitoringdb.models.component_error import ComponentError
from lofar.maintenance.monitoringdb.models.element import Element
+from lofar.maintenance.monitoringdb.models.element_error import ElementError
from lofar.maintenance.monitoringdb.models.log import ActionLog
-from lofar.maintenance.monitoringdb.models.signals.generic_test_insert import on_inserting_test
-
# RTSM Models
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation, RTSMError, RTSMErrorSample
-
+from lofar.maintenance.monitoringdb.models.signals.generic_test_insert import on_inserting_test
+from lofar.maintenance.monitoringdb.models.station import Station
# Station Test Models
from lofar.maintenance.monitoringdb.models.station_test import StationTest
-from lofar.maintenance.monitoringdb.models.component_error import ComponentError
-from lofar.maintenance.monitoringdb.models.element_error import ElementError
-
from lofar.maintenance.monitoringdb.rtsm_test_raw_parser import parse_rtsm_test
-from lofar.maintenance.monitoringdb.station_test_raw_parser import parse_raw_station_test, station_type_from_station_name
+from lofar.maintenance.monitoringdb.station_test_raw_parser import parse_raw_station_test, \
+ station_type_from_station_name
logger = logging.getLogger(__name__)
-
-
-
-
from django.db.models.signals import post_save
-post_save.connect(on_inserting_test, sender=RTSMObservation)
-post_save.connect(on_inserting_test, sender=StationTest)
+post_save.connect(on_inserting_test, sender=RTSMObservation, dispatch_uid='test_inserted')
+post_save.connect(on_inserting_test, sender=StationTest, dispatch_uid='test_inserted')
def create_component_errors(station: Station, station_test: StationTest, component_errors_data):
@@ -43,7 +37,10 @@ def create_component_errors(station: Station, station_test: StationTest, compone
component_list.append(component_entity)
component_error_entity, _ = ComponentError.objects.get_or_create(component=component_entity,
station_test=station_test,
- **component_error)
+ type=component_error[
+ 'type'])
+ component_error_entity.details.update(component_error['details'])
+ component_error_entity.save()
component_error_list.append(component_error_entity)
@@ -138,7 +135,8 @@ def create_rtsm_test(rtsm_observation) -> RTSMObservation:
station_name = rtsm_observation.pop('station_name')
sas_id = rtsm_observation.pop('observation_id')
- station, _ = create_station(dict(name=station_name,
+
+ station = create_station(dict(name=station_name,
type=station_type_from_station_name(station_name)))
rtsm_observation, _ = RTSMObservation.objects.get_or_create(station=station,
@@ -153,6 +151,7 @@ def create_rtsm_test(rtsm_observation) -> RTSMObservation:
def insert_station_test(action_log_id: ActionLog, raw_tests):
action_log = ActionLog.objects.get(pk=action_log_id)
try:
+ logger.debug('handling raw request_data for data %s', raw_tests)
parsed_content = parse_raw_station_test(raw_tests)
if parsed_content is None:
raise Exception('cannot parse test {}'.format(raw_tests))
@@ -171,11 +170,11 @@ def insert_station_test(action_log_id: ActionLog, raw_tests):
def insert_rtsm_test(action_log_id: ActionLog, request_data: Dict):
action_log = ActionLog.objects.get(pk=action_log_id)
try:
- #logger.debug('handling raw request_data for %s', request_data)
+ logger.debug('handling raw request_data for %s', request_data)
content = request_data['content']
station_name = request_data['station_name']
- #logger.debug('handling raw request_data for data %s', content)
+ logger.debug('handling raw request_data for data %s', content)
entry = parse_rtsm_test(content)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
index db345cc545f47e8eb44947f03a288b88861b6862..731dbcf79f606d578ffc2a9e5f46c26540e88d92 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
@@ -1,6 +1,6 @@
from rest_framework import viewsets, status
from rest_framework.response import Response
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpRequest
from rest_framework.decorators import api_view
import logging
from ..exceptions import ItemAlreadyExists
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
index e182e42303852678dec3394d2ede8470e877c892..a78c02da95f4c54c76327014e999c8e581270d15 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py
@@ -3,12 +3,13 @@ import logging
from collections import OrderedDict
from collections import defaultdict
from math import ceil
+from typing import List
-from django.utils import timezone
import coreapi
import coreschema
import pytz
-from django.db.models import Count
+from django.db.models import Count, Prefetch
+from django.utils import timezone
from rest_framework import status
from rest_framework.response import Response
from rest_framework.reverse import reverse
@@ -20,12 +21,15 @@ from lofar.maintenance.monitoringdb.models.rtsm import RTSMError
from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
from lofar.maintenance.monitoringdb.models.station import Station
from lofar.maintenance.monitoringdb.models.station_test import StationTest
+from lofar.maintenance.monitoringdb.models.test import GenericTest
from lofar.maintenance.monitoringdb.models.wincc import WinCCAntennaStatus, \
- latest_status_per_station_and_component_type, \
- latest_status_per_station_and_component_type_antenna_id
+ ComponentStatus, StationStatus, \
+ latest_status_per_station_and_component_type_antenna_id, \
+ latest_status_per_station_and_component_type
+from lofar.maintenance.monitoringdb.serializers.wincc import serialize_station_status
logger = logging.getLogger(__name__)
-from lofar.maintenance.monitoringdb.models.rtsm import MODE_TO_COMPONENT
+from lofar.maintenance.monitoringdb.models.rtsm import MODE_TO_FREQ_RANGE
def parse_date(date):
@@ -167,7 +171,6 @@ class ValidableReadOnlyView(APIView):
def get(self, request):
# Store the request as attribute
self.request = request
-
try:
self.validate_query_parameters(request)
except ValueError as e:
@@ -190,8 +193,9 @@ class ValidableReadOnlyView(APIView):
return response
-def compute_error_summary(station_test:StationTest, selected_error_types=None):
+def compute_error_summary(station_test: StationTest, selected_error_types=None):
component_error_summary = dict()
+ total = 0
for component_error in station_test.component_errors.all():
component_type = component_error.component.type
@@ -199,7 +203,7 @@ def compute_error_summary(station_test:StationTest, selected_error_types=None):
if selected_error_types and component_error_type not in selected_error_types:
continue
-
+ total += 1
if component_type not in component_error_summary:
component_error_summary[component_type] = {component_error_type: 1}
elif component_error_type not in component_error_summary[component_type]:
@@ -207,7 +211,8 @@ def compute_error_summary(station_test:StationTest, selected_error_types=None):
else:
component_error_summary[component_type][component_error_type] += 1
- return component_error_summary
+ return component_error_summary, total
+
from django.db.models import Window, F
from django.db.models.functions import Rank
@@ -270,18 +275,19 @@ class ControllerStationOverview(ValidableReadOnlyView):
def get_last_station_test_per_station(self, selected_stations):
expected_tests = len(selected_stations) * self.n_station_tests
-
- station_test_instances = StationTest.objects.filter(station__name__in=selected_stations). \
+ station_test_instances = StationTest.objects.order_by().filter(
+ station__name__in=selected_stations). \
annotate(order=Window(
expression=Rank(),
partition_by=[F('station')],
order_by=F('start_datetime').desc())
- ).order_by('order')[:expected_tests].\
+ ).order_by('order', 'station__name')[:expected_tests]. \
select_related('station').prefetch_related('component_errors',
'component_errors__component')
st_per_station = defaultdict(list)
- for station_test in station_test_instances:
+
+ for ind, station_test in enumerate(station_test_instances):
station_name = station_test.station.name
test_summary = dict()
@@ -289,11 +295,13 @@ class ControllerStationOverview(ValidableReadOnlyView):
test_summary.update(end_datetime=station_test.end_datetime)
test_summary.update(checks=station_test.checks)
- component_error_summary = compute_error_summary(station_test, self.error_types)
-
+ component_error_summary, total = compute_error_summary(station_test, self.error_types)
+ test_summary.update(total_component_errors=total)
test_summary.update(component_error_summary=component_error_summary)
-
- st_per_station[station_name] = test_summary
+ if station_name in st_per_station and len(
+ st_per_station[station_name]) >= self.n_station_tests:
+ continue
+ st_per_station[station_name] += [test_summary]
return st_per_station
def get_last_rtsm(self, selected_stations):
@@ -345,10 +353,9 @@ class ControllerStationOverview(ValidableReadOnlyView):
station_entities = Station.objects.all()
if 'A' not in self.station_group:
- station_entities = station_entities.filter(type__in=self.station_group)\
-
- station_entities = list(zip(*station_entities.values_list('name')))[0]
+ station_entities = station_entities.filter(type__in=self.station_group)
+ station_entities = list(zip(*station_entities.values_list('name').order_by('name')))[0]
# Since django preferes a ordered dict over a dict we make it happy... for now
rtsm_per_station = self.get_last_rtsm(station_entities)
@@ -414,16 +421,17 @@ class ControllerStationTestsSummary(ValidableReadOnlyView):
from_date = timezone.now() - timezone.timedelta(days=self.lookback_time)
- station_test_entities = StationTest.objects.filter(start_datetime__gt=from_date).\
+ station_test_entities = StationTest.objects.filter(start_datetime__gt=from_date). \
select_related('station')
if 'A' not in self.station_group:
- station_test_entities = station_test_entities.filter(station__type__in=self.station_group)
-
+ station_test_entities = station_test_entities.filter(station__type__in=
+ self.station_group)
station_test_entities = \
station_test_entities.prefetch_related('component_errors',
- 'component_errors__component').order_by('-start_datetime')
+ 'component_errors__component').order_by(
+ '-start_datetime')
# Since django preferes a ordered dict over a dict we make it happy... for now
response_payload = list()
@@ -431,12 +439,14 @@ class ControllerStationTestsSummary(ValidableReadOnlyView):
for station_test in station_test_entities:
station_test_summary = dict()
station_test_summary.update(station_name=station_test.station.name)
- station_test_summary.update(total_component_errors=station_test.component_errors.count())
+ station_test_summary.update(
+ total_component_errors=station_test.component_errors.count())
station_test_summary.update(start_datetime=station_test.start_datetime)
station_test_summary.update(end_datetime=station_test.end_datetime)
station_test_summary.update(date=station_test.start_datetime.strftime("%Y-%m-%d"))
- component_error_summary = compute_error_summary(station_test, self.error_types)
+ component_error_summary, total = compute_error_summary(station_test, self.error_types)
+ station_test_summary.update(total_component_errors=total)
station_test_summary.update(component_error_summary=component_error_summary)
response_payload.append(station_test_summary)
@@ -503,26 +513,26 @@ class ControllerLatestObservations(ValidableReadOnlyView):
filtered_entities = filtered_entities.exclude(errors_summary__isnull=True)
errors_summary = filtered_entities \
- .values('observation_id',
+ .values('sas_id',
'station__name',
'start_datetime',
'end_datetime',
- 'errors_summary__error_type',
- 'errors_summary__mode') \
- .annotate(total=Count('errors_summary__error_type')) \
- .order_by('observation_id', 'station__name')
+ 'errors__error_type',
+ 'errors__mode') \
+ .annotate(total=Count('errors__error_type')) \
+ .order_by('sas_id', 'station__name')
if self.error_types:
errors_summary = errors_summary.filter(errors_summary__error_type__in=self.error_types)
response = dict()
for error_summary in errors_summary:
- observation_id = error_summary['observation_id']
+ observation_id = error_summary['sas_id']
station_name = error_summary['station__name']
start_datetime = error_summary['start_datetime']
end_datetime = error_summary['end_datetime']
- mode = error_summary['errors_summary__mode']
- error_type = error_summary['errors_summary__error_type']
+ mode = error_summary['errors__mode']
+ error_type = error_summary['errors__error_type']
total = error_summary['total']
if observation_id not in response:
@@ -766,6 +776,17 @@ class ControllerAllComponentErrorTypes(ValidableReadOnlyView):
return Response(status=status.HTTP_200_OK, data=data)
+def gather_statuses_for_test(generic_tests: List[GenericTest]):
+ statuses = {station_status.id: station_status
+ for station_status in StationStatus.objects.filter(generictest__in=generic_tests). \
+ prefetch_related(
+ Prefetch('component_status', queryset=ComponentStatus.objects.
+ filter(status_code__gt=10).
+ select_related('component'),
+ to_attr='component_statuses'))}
+ return statuses
+
+
class ControllerStationComponentErrors(ValidableReadOnlyView):
description = "Provides a summary per station of the component errors"
# required parameters
@@ -818,16 +839,25 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
def collect_station_test_errors(self):
- component_errors = ComponentError.objects.filter(station_test__station__name=self.station_name).\
- filter(station_test__start_datetime__range=(self.from_date, self.to_date)).\
- select_related('station_test', 'component').\
- prefetch_related('station_test__station','station_test__station_status').order_by('-station_test__start_datetime', 'component__station__name')
+ tests = StationTest.objects.filter(station__name=self.station_name). \
+ filter(start_datetime__range=(self.from_date, self.to_date))
+
+ component_errors = ComponentError.objects.filter(station_test__in=tests). \
+ select_related('station_test', 'component'). \
+ select_related('station_test__station_status'). \
+ order_by('-station_test__start_datetime', 'component__station__name')
+ if self.error_types:
+ component_errors = component_errors.filter(type__in=self.error_types)
response_payload = OrderedDict()
+ statuses = gather_statuses_for_test(tests)
+
for component_error in component_errors:
+
component_id = component_error.component.component_id
component_type = component_error.component.type
+
test_id = component_error.station_test.pk
component_error_summary = dict(error_type=component_error.type,
@@ -839,12 +869,13 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
per_component_type = response_payload[component_type]
if test_id not in per_component_type:
-
+ status_id = component_error.station_test.station_status.pk
+ status = statuses[status_id]
per_component_type[test_id] = dict(
- start_datetime = component_error.station_test.start_datetime,
- end_datetime = component_error.station_test.end_datetime,
- test_type = 'S',
- status=component_error.station_test.station_status,
+ start_date=component_error.station_test.start_datetime,
+ end_date=component_error.station_test.end_datetime,
+ test_type='S',
+ status=serialize_station_status(status.component_statuses, component_type),
component_errors=dict()
)
@@ -855,91 +886,75 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
else:
per_test[component_id] += [component_error_summary]
-
for type in response_payload:
response_payload[type] = list(response_payload[type].values())
return response_payload
def collect_rtsm_errors(self):
- station_entry = Station.objects.filter(name=self.station_name).first()
+ tests = RTSMObservation.objects.filter(station__name=self.station_name). \
+ filter(start_datetime__range=(self.from_date, self.to_date))
+
+ statuses = gather_statuses_for_test(tests)
+
+ rtsm_errors = RTSMError.objects.filter(observation__station__name=self.station_name). \
+ filter(observation__start_datetime__range=(self.from_date, self.to_date)). \
+ select_related('observation', 'component'). \
+ prefetch_related('observation__station_status',
+ 'observation__station_status__component_status')
+ if self.error_types:
+ rtsm_errors = rtsm_errors.filter(error_type__in=self.error_types)
response_payload = OrderedDict()
- rtsm_observations = RTSMObservation.objects.filter(station=station_entry) \
- .filter(start_datetime__gte=self.from_date,
- end_datetime__lte=self.to_date)
- failing_component_modes = rtsm_observations.exclude(errors__isnull=True).distinct(
- 'errors__mode').values_list('errors__mode')
- for observing_mode in failing_component_modes:
+ for component_error in rtsm_errors:
+ component_id = component_error.component.component_id
+ component_type = component_error.component.type
+ test_id = component_error.observation.pk
+
+ polarization = component_error.polarization
+ error_type = component_error.error_type
+ url_to_plot = reverse('rtsm-summary-plot-detail', (component_error.pk,),
+ request=self.request)
+ start_frequency, end_frequency = MODE_TO_FREQ_RANGE[component_error.mode]
+ component_error_summary = dict(error_type=error_type,
+ rcu_id=component_error.rcu,
+ polarization=polarization,
+ details=dict(
+ url=url_to_plot,
+ component_id=component_id,
+ percentage=component_error.percentage,
+ n_samples=component_error.observation.samples,
+ start_frequency=start_frequency,
+ stop_frequency=end_frequency,
+ mode=component_error.mode
+ ))
- observing_mode = observing_mode[0]
- component_name = MODE_TO_COMPONENT[observing_mode]
+ if component_type not in response_payload:
+ response_payload[component_type] = dict()
- rtsm_errors_per_component_type = list()
+ per_component_type = response_payload[component_type]
- for rtsm_observation in rtsm_observations.order_by('-start_datetime'):
- rtsm_summary = OrderedDict()
+ if test_id not in per_component_type:
+ status_id = component_error.observation.station_status.pk
+ status = statuses[status_id]
- rtsm_summary['test_type'] = 'R'
- rtsm_summary['start_date'] = rtsm_observation.start_datetime
- rtsm_summary['end_date'] = rtsm_observation.end_datetime
- rtsm_summary['observation_id'] = rtsm_observation.sas_id
- antenna_statuses = latest_status_per_station_and_component_type(
- WinCCAntennaStatus.objects,
- self.station_name,
- component_name,
- to_date=rtsm_observation.end_datetime)
- rtsm_summary['status'] = antenna_statuses
- component_errors_dict = OrderedDict()
- rtsm_summary['component_errors'] = component_errors_dict
-
- component_errors = rtsm_observation.errors \
- .filter(mode=observing_mode) \
- .values('error_type', 'start_frequency',
- 'stop_frequency', 'percentage',
- 'error_type', 'count', 'rcu', 'pk')
- if component_errors.count() == 0 and rtsm_observation.errors_summary.count() != 0:
- continue
- else:
- rtsm_errors_per_component_type.append(rtsm_summary)
-
- if self.error_types:
- component_errors = component_errors.filter(error_type__in=self.error_types)
-
- for component_error in component_errors:
-
- rcu_id = component_error['rcu']
- component_id, polarization = antenna_id_polarization_from_rcu_type_polarization(
- rcu_id, component_name)
- details = dict(percentage=component_error['percentage'],
- start_frequency=component_error['start_frequency'],
- stop_frequency=component_error['stop_frequency'],
- count=component_error['count'],
- component_id=component_id)
- error_type = component_error['error_type']
- # CHECKS IF THE ERROR IS PRESENT IN BOTH RCUS (hence, both polarizations of the antenna)
- url_to_plot = reverse('rtsm-summary-plot-detail', (component_error['pk'],),
- request=self.request)
- details['url'] = url_to_plot
-
- if not str(component_id) in component_errors_dict:
- component_errors_dict[str(component_id)] = OrderedDict()
-
- if error_type not in component_errors_dict[str(component_id)]:
- rtsm_error_summary = dict(error_type=error_type)
- component_errors_dict[str(component_id)][error_type] = rtsm_error_summary
- else:
- rtsm_error_summary = component_errors_dict[str(component_id)][error_type]
-
- rtsm_error_summary[polarization] = dict(details=details, error_type=error_type,
- rcu_id=rcu_id,
- polarization=polarization, )
-
- if component_name not in response_payload:
- response_payload[component_name] = rtsm_errors_per_component_type
+ per_component_type[test_id] = dict(
+ start_date=component_error.observation.start_datetime,
+ end_date=component_error.observation.end_datetime,
+ test_type='R',
+ status=serialize_station_status(status.component_statuses, component_type),
+ component_errors=dict()
+ )
+
+ per_test = per_component_type[test_id]['component_errors']
+
+ if component_id not in per_test:
+ per_test[component_id] = {error_type: {polarization: component_error_summary}}
else:
- response_payload[component_name] += rtsm_errors_per_component_type
+ per_test[component_id][error_type][polarization] = component_error_summary
+ for type in response_payload:
+ response_payload[type] = list(response_payload[type].values())
return response_payload
def compute_response(self):
@@ -954,23 +969,25 @@ class ControllerStationComponentErrors(ValidableReadOnlyView):
station_test_errors = self.collect_station_test_errors()
if self.test_type in ['R', 'B']:
- rtsm_errors = {} # self.collect_rtsm_errors()
+ rtsm_errors = self.collect_rtsm_errors()
payload = OrderedDict()
- for component_type in set(rtsm_errors.keys() | station_test_errors.keys()):
+
+ for component_type in set(station_test_errors.keys() | rtsm_errors.keys()):
station_test_errors_per_type = station_test_errors.get(component_type, [])
rtsm_errors_per_type = rtsm_errors.get(component_type, [])
payload[component_type] = OrderedDict()
payload[component_type]['errors'] = station_test_errors_per_type
- #payload[component_type]['errors'] = sorted(
- # station_test_errors_per_type + rtsm_errors_per_type,
- # key=lambda item: item['start_date'], reverse=True)
-
- #payload[component_type]['current_status'] = \
- # latest_status_per_station_and_component_type(WinCCAntennaStatus.objects,
- # self.station_name,
- # component_type,
- # to_date=datetime.datetime.now())
+
+ payload[component_type]['errors'] = sorted(
+ station_test_errors_per_type + rtsm_errors_per_type,
+ key=lambda item: item['start_date'], reverse=True)
+
+ payload[component_type]['current_status'] = \
+ latest_status_per_station_and_component_type(WinCCAntennaStatus.objects,
+ self.station_name,
+ component_type,
+ to_date=datetime.datetime.now())
return Response(status=status.HTTP_200_OK, data=payload)
@@ -1030,23 +1047,23 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView):
]
def compute_ok_rtsm_list(self):
- mode = COMPONENT_TO_MODE[self.component_type]
rcus_per_polarization = rcus_from_antenna_and_type(self.antenna_id,
self.component_type)
- good_observation_list = RTSMObservation.objects.filter(errors_summary__mode__in=mode,
- start_datetime__gt=self.from_date,
- end_datetime__lt=self.to_date,
- station__name=self.station_name). \
- exclude(errors_summary__rcu__in=list(rcus_per_polarization.keys())). \
+ good_observation_list = RTSMObservation.objects.filter(
+ errors__component__type=self.component_type,
+ start_datetime__gt=self.from_date,
+ end_datetime__lt=self.to_date,
+ station__name=self.station_name). \
+ exclude(errors__rcu__in=list(rcus_per_polarization.keys())). \
values('pk',
- 'observation_id',
+ 'sas_id',
'start_datetime',
'end_datetime').distinct('pk')
result = []
for observation in good_observation_list:
- entry = dict(test_id=observation['observation_id'],
+ entry = dict(test_id=observation['sas_id'],
db_id=observation['pk'],
start_date=observation['start_datetime'],
end_date=observation['end_datetime'],
@@ -1098,11 +1115,11 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView):
rtsm_errors = RTSMError.objects.values('observation__pk',
'observation__start_datetime',
'observation__end_datetime',
- 'observation__observation_id',
+ 'observation__sas_id',
'observation__station__name',
'pk',
'rcu',
- 'mode',
+ 'component__type',
'error_type',
'percentage',
'count',
@@ -1110,13 +1127,13 @@ class ControllerStationComponentElementErrors(ValidableReadOnlyView):
observation__start_datetime__gt=self.from_date,
observation__end_datetime__lt=self.to_date,
observation__station__name=self.station_name,
- mode__in=COMPONENT_TO_MODE[self.component_type],
+ component__type=self.component_type,
rcu__in=list(rcus_per_polarization.keys())).order_by('-observation__start_datetime')
for item in rtsm_errors:
observation_pk = item['observation__pk']
if observation_pk not in errors.keys():
- errors[observation_pk] = dict(test_id=item['observation__observation_id'],
+ errors[observation_pk] = dict(test_id=item['observation__sas_id'],
db_id=item['observation__pk'],
start_date=item[
'observation__start_datetime'],
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py b/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py
index 51b961d3ac7806b7a317819ebac15f3974fcda40..4965ea61d3eaaabf111a1c219dc72c1acef46e39 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py
@@ -1,5 +1,5 @@
from rest_framework.viewsets import ReadOnlyModelViewSet
-from rest_framework.schemas import SchemaGenerator
+
from lofar.maintenance.monitoringdb.serializers.log import ActionLogSerializer, ActionLog
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
index 1a7a78e3220c959c27e440ab06f6cae20f325803..c7f2ff1bf92a199853d893e4bdb4c11cc2dff04a 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -1,14 +1,18 @@
-from .common import *
-from ..models.rtsm import RTSMObservation
-from ..models.log import ActionLog
-from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer,\
- RTSMSummaryPlotSerializer, RTSMError, RTSMErrorSample, RTSMErrorSampleSerializer
+import os
from datetime import datetime
-from lofar.maintenance.monitoringdb.tasks.insert_raw_tests import insert_rtsm_test
-from django.shortcuts import get_object_or_404
+
+from .common import *
+from rest_framework import viewsets
from django.http import Http404
-import os
+from django.shortcuts import get_object_or_404
+
+from lofar.maintenance.mdb.tests_validators import is_rtsm_test
from lofar.maintenance.monitoringdb.tasks.generate_plots import check_error_summary_plot
+from lofar.maintenance.monitoringdb.tasks.insert_raw_tests import insert_rtsm_test
+from ..models.log import ActionLog
+from ..models.rtsm import RTSMObservation
+from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSerializer, \
+ RTSMSummaryPlotSerializer, RTSMError, RTSMErrorSample, RTSMErrorSampleSerializer
logger = logging.getLogger('views')
@@ -37,7 +41,7 @@ class RTSMSummaryPlot(viewsets.ViewSet):
"""
queryset = RTSMError.objects.all()
- def retrieve(self, request, pk=None):
+ def retrieve(self, _, pk=None):
try:
entity = get_object_or_404(self.queryset, pk=pk)
summary_plot = entity.summary_plot.first()
@@ -47,22 +51,36 @@ class RTSMSummaryPlot(viewsets.ViewSet):
uri = RTSMSummaryPlotSerializer(summary_plot).data['uri']
- except ObjectDoesNotExist as e:
+ except ObjectDoesNotExist:
check_error_summary_plot.delay(pk)
raise Http404()
if uri and os.path.exists(uri) and os.path.isfile(uri):
- with open(uri, 'rb') as f_stream:
- image = f_stream.read()
- return HttpResponse(image, status=status.HTTP_200_OK, content_type='image/gif')
+ with open(uri, 'rb') as f_stream:
+ image = f_stream.read()
+ return HttpResponse(image, status=status.HTTP_200_OK, content_type='image/gif')
else:
check_error_summary_plot.delay(pk)
- print(uri)
+
raise Http404()
+def handle_rtsm_insert(content, remote_addr):
+ action_log = ActionLog(who=remote_addr,
+ what='INSERT_REQUESTED',
+ when=datetime.now(),
+ model_name='RTSM',
+ entry_id=-1)
+
+ action_log.save()
+
+ insert_rtsm_test.delay(action_log.pk, content)
+
+ return action_log.pk
+
+
@api_view(['POST'])
-def insert_raw_rtsm_test(request):
+def insert_raw_rtsm_test(request: HttpRequest):
"""
This function is meant to parse a request of the form
{
@@ -73,35 +91,31 @@ def insert_raw_rtsm_test(request):
:return:
"""
if request.method == 'POST':
+ remote_addr = request.META['REMOTE_ADDR']
+ logs_pk = []
if 'content' in request.data:
- try:
- logger.info('handling raw request for %s', request)
-
- logger.info('handling raw request for data %s', request.data)
-
- action_log = ActionLog(who=request.META['REMOTE_ADDR'],
- what='INSERT_REQUESTED',
- when=datetime.now(),
- model_name='StationTest',
- entry_id=-1)
-
- action_log.save()
-
- insert_rtsm_test.delay(action_log.pk, request.data)
-
- return Response(status=status.HTTP_200_OK,
- data='RTSM tests insert acknowledged.'
- ' Check log for id %s' % action_log.pk)
- except Exception as e:
- logger.exception("exception occurred while parsing raw station info: %s", e)
- return Response(exception=True,
- data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
- e, request),
- status=status.HTTP_400_BAD_REQUEST)
+ content = request.data
+
+ logs_pk += [handle_rtsm_insert(content, remote_addr)]
+ elif request.FILES:
+ files = request.FILES
+ for station_name, file in zip(request.data['station_name'], files.values()):
+ content_data = file.read().decode('UTF-8')
+ if not is_rtsm_test(content_data):
+ return Response(exception=True,
+ data="the attached file is not a RTSM %s" % file.name,
+ status=status.HTTP_400_BAD_REQUEST)
+
+ content = dict(station_name=request.data['station_name'],
+ content=content_data)
+ logs_pk += [handle_rtsm_insert(content, remote_addr)]
else:
return Response(exception=True,
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}",
status=status.HTTP_400_BAD_REQUEST)
+
+ return Response(status=status.HTTP_200_OK,
+ data='RTSM tests insert acknowledged.'
+ ' Check log for id %s' % logs_pk)
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index 07e8816d50e4deed8c95a2ea7927f650ffa6cd67..a3142310b088d68674f6bcaf7c036af2ead9df8c 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -1,11 +1,15 @@
+from datetime import datetime
+
from django_filters import rest_framework as filters
from rest_framework.viewsets import ReadOnlyModelViewSet
-
from .common import *
+from lofar.maintenance.mdb.tests_validators import is_station_test
+from lofar.maintenance.monitoringdb.tasks.insert_raw_tests import insert_station_test
from ..models.component import Component
from ..models.component_error import ComponentError
from ..models.element import Element
from ..models.element_error import ElementError
+from ..models.log import ActionLog
from ..models.station import Station
from ..models.station_test import StationTest
from ..serializers.component import ComponentSerializer
@@ -14,10 +18,9 @@ from ..serializers.element import ElementSerializer
from ..serializers.element_error import ElementErrorSerializer
from ..serializers.station import StationSerializer
from ..serializers.station_tests import StationTestSerializer
-from ..models.log import ActionLog
-from datetime import datetime
-
-from lofar.maintenance.monitoringdb.tasks.insert_raw_tests import insert_station_test
+import logging
+from django.http import HttpRequest
+from rest_framework.decorators import api_view
logger = logging.getLogger('views')
@@ -134,43 +137,52 @@ class ElementErrorViewSet(ReadOnlyModelViewSet):
filterset_class = ElementErrorFilterSet
+def handle_stationtest_insert(content, remote_addr):
+ action_log = ActionLog(who=remote_addr,
+ what='INSERT_REQUESTED',
+ when=datetime.now(),
+ model_name='STATIONTEST',
+ entry_id=-1)
+
+ action_log.save()
+
+ insert_station_test.delay(action_log.pk, content)
+
+ return action_log.pk
+
+
@api_view(['POST'])
-def insert_raw_station_test(request):
+def insert_raw_station_test(request: HttpRequest):
"""
This function is meant to parse a request of the form
- { content: [STATION TEST RAW TEXT] }
- parse the content field and create all the station_test entity related into the database
-
+ {
+ "content": "[STATION TEST RAW TEXT]"
+ }
+ :param request: HTTP request
+ :return:
"""
if request.method == 'POST':
+ remote_addr = request.META['REMOTE_ADDR']
+ logs_pk = []
if 'content' in request.data:
- try:
- logger.info('handling raw request for %s', request)
- content = request.data['content']
- logger.info('handling raw request for data %s', content)
-
- action_log = ActionLog(who=request.META['REMOTE_ADDR'],
- what='INSERT_REQUESTED',
- when=datetime.now(),
- model_name='StationTest',
- entry_id=-1)
-
- action_log.save()
-
- insert_station_test.delay(action_log.pk, content)
-
- return Response(status=status.HTTP_200_OK,
- data='Station tests insert acknowledged.'
- ' Check log for id %s' % action_log.pk)
- except Exception as e:
- logger.exception("exception occurred while parsing raw station info: %s", e)
- return Response(exception=True,
- data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
- e, request),
- status=status.HTTP_400_BAD_REQUEST)
+ content = request.data
+ logs_pk += [handle_stationtest_insert(content, remote_addr)]
+ elif request.FILES:
+ files = request.FILES
+ for file in files.values():
+ content_data = file.read().decode('UTF-8')
+ if not is_station_test(content_data):
+ return Response(exception=True,
+ data="the attached file is not a station test %s" % file.name,
+ status=status.HTTP_400_BAD_REQUEST)
+
+ logs_pk += [handle_stationtest_insert(content_data, remote_addr)]
else:
return Response(exception=True,
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}",
status=status.HTTP_400_BAD_REQUEST)
+
+ return Response(status=status.HTTP_200_OK,
+ data='station test insert acknowledged.'
+ ' Check log for id %s' % logs_pk)
diff --git a/LCU/Maintenance/Docker/BaseMaintenance/Dockerfile b/LCU/Maintenance/Docker/BaseMaintenance/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..48ad26b13eba2945f6c505c092cdbbf2bffc7228
--- /dev/null
+++ b/LCU/Maintenance/Docker/BaseMaintenance/Dockerfile
@@ -0,0 +1,14 @@
+FROM centos:7.6.1810
+
+RUN yum install -y epel-release && \
+ yum install -y python36 python36-psycopg2 python36-pip && \
+ yum clean all
+RUN pip3 install beautifultable==0.7.0 \
+ blessings==1.7 \
+ celery==4.3.0 \
+ django==2.2 \
+ djangorestframework==3.9.4 \
+ django-coreapi==2.3 \
+ django-filter==2.1 \
+ inotify==0.2.10 \
+ requests==2.22
diff --git a/LCU/Maintenance/Docker/DBInterface/Dockerfile b/LCU/Maintenance/Docker/DBInterface/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..13f7e8865aa820117ea6a4a7de28efaf8549d7bf
--- /dev/null
+++ b/LCU/Maintenance/Docker/DBInterface/Dockerfile
@@ -0,0 +1,16 @@
+FROM lofar-maintenance-base as lofar-build-dbinterface
+
+ARG SRC_DIR=/root/src
+ARG BUILD_DIR=$SRC_DIR/build/gnucxx11_debug
+
+RUN yum install -y cmake>3 make gcc-c++ gcc python36-devel
+COPY . $SRC_DIR
+RUN mkdir -p $BUILD_DIR
+WORKDIR $BUILD_DIR
+RUN cmake $SRC_DIR -DBUILD_PACKAGES=DBInterface \
+ -DCMAKE_INSTALL_PREFIX=/opt/lofar \
+ -DUSE_LOG4CPLUS=OFF && \
+ make && make install
+
+FROM lofar-maintenance-base
+COPY --from=lofar-build-dbinterface /opt/lofar/ /opt/lofar
\ No newline at end of file
diff --git a/LCU/Maintenance/Docker/RESTService/Dockerfile b/LCU/Maintenance/Docker/RESTService/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f7e6dde09870a013533af388424b2f8f15d40743
--- /dev/null
+++ b/LCU/Maintenance/Docker/RESTService/Dockerfile
@@ -0,0 +1,14 @@
+FROM lofar-maintenance-dbinterface
+
+
+RUN pip3.6 install gunicorn==19.9.0 \
+ gevent \
+ django-silk==3.0.2
+
+COPY LCU/Maintenance/Docker/RESTService/entrypoint.sh /root/
+
+SHELL ["/bin/bash"]
+
+ENTRYPOINT /root/entrypoint.sh
+
+ENV N_WORKERS=1
\ No newline at end of file
diff --git a/LCU/Maintenance/Docker/RESTService/entrypoint.sh b/LCU/Maintenance/Docker/RESTService/entrypoint.sh
new file mode 100755
index 0000000000000000000000000000000000000000..7fc58b8f2727ec845e0cc3d797456a00248aaf3e
--- /dev/null
+++ b/LCU/Maintenance/Docker/RESTService/entrypoint.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+source /opt/lofar/lofarinit.sh
+gunicorn -w $N_WORKERS -k gevent lofar.maintenance.django_postgresql.wsgi:application $@
\ No newline at end of file
diff --git a/LCU/Maintenance/Docker/docker-compose.yml b/LCU/Maintenance/Docker/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ebf033f3c02714bbfc25f75fc63b908d2de9e344
--- /dev/null
+++ b/LCU/Maintenance/Docker/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "3.0"
+
+services:
+ lofar-maintenance-base:
+ image: lofar-maintenance-base
+ build:
+ context: ../../../
+ dockerfile: LCU/Maintenance/Docker/BaseMaintenance/Dockerfile
+ lofar-maintenance-dbinterface:
+ image: lofar-maintenance-dbinterface
+ build:
+ context: ../../../
+ dockerfile: LCU/Maintenance/Docker/DBInterface/Dockerfile
\ No newline at end of file
diff --git a/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py b/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
index d4e7d6757761f810981ebd1a93a0cf63197c93e7..a8c54f8dfe7bb84746673e904b929570f2d7bfe6 100644
--- a/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
+++ b/LCU/Maintenance/MDB_tools/cli/station_tests_watchdog.py
@@ -145,7 +145,7 @@ def _is_file_created(event):
:param event: the list that describes the creation event
:return: True if it refers to a file being created False otherwise
"""
- return 'IN_CLOSE_WRITE' in event[1]
+ return 'IN_CLOSE_WRITE' in event[1] or 'IN_MOVED_TO' in event[1] or 'IN_CREATE' in event[1]
def handle_event_file_created(mdb_address, event):
diff --git a/LCU/Maintenance/MDB_tools/lib/CMakeLists.txt b/LCU/Maintenance/MDB_tools/lib/CMakeLists.txt
index e7929148818f5d9388dc39b42607b0b8a1e1c3a3..725c506de7545a0972f136d58849093f8d4b7cba 100644
--- a/LCU/Maintenance/MDB_tools/lib/CMakeLists.txt
+++ b/LCU/Maintenance/MDB_tools/lib/CMakeLists.txt
@@ -4,6 +4,7 @@ find_python_module(requests)
set(_py_files
__init__.py
client.py
+ tests_validators.py
)
python_install(${_py_files} DESTINATION lofar/maintenance/mdb)
diff --git a/LCU/Maintenance/MDB_tools/lib/client.py b/LCU/Maintenance/MDB_tools/lib/client.py
index b84538036130a1e1bab354cf0c031efdc1958d78..b4621ed1049a785c784f3fc377bdf3cc20e608f0 100644
--- a/LCU/Maintenance/MDB_tools/lib/client.py
+++ b/LCU/Maintenance/MDB_tools/lib/client.py
@@ -1,7 +1,7 @@
import logging
import requests
import re
-
+from .tests_validators import is_rtsm_test, is_station_test
logger = logging.getLogger(__name__)
@@ -21,7 +21,7 @@ def read_stationtest_rtsm_output_to_dict(file_path):
try:
with open(file_path, 'r') as input_stream:
rtsm_filename_pattern = '(?P<station_name>\w*)_\d+_\d+\.dat$'
- result = dict(content=input_stream.read().strip('\n'))
+ result = dict(content=input_stream.read().strip('\n'), file_path=file_path)
if re.search(rtsm_filename_pattern, file_path): # Matches the filename to understand if it is a RTSM
result.update(station_name=re.search(rtsm_filename_pattern, file_path).group('station_name'))
return result
@@ -31,84 +31,6 @@ def read_stationtest_rtsm_output_to_dict(file_path):
return None
-def is_station_test(content):
- """
- Check if the content of the file have the structure of a station test.
- Example content
-20130522,NFO,---,STATION,NAME=CS011C
-20130522,NFO,---,RUNTIME,START=03:10:00,STOP=05:52:27
-20130522,NFO,---,CHECKS,S1,O1,SP1,NS1=180,S3,O3,SP3,NS3=180,M,O5,SN,SP5,NS5=300,EO,ESP,EN=30,TV,TM
-20130522,NFO,---,STATISTICS,BAD_LBL=1,BAD_LBH=2,BAD_HBA=7
-20130522,LBL,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=82.0,SUBBANDY=301,SIGNALY=81.8
-20130522,LBL,019,RF_FAIL,X=77.0,Y=70.1
-20130522,LBL,019,LOW_NOISE,Xproc=100.000,Xval=66.6,Xdiff=0.0,Xref=67.7
-20130522,LBH,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=81.8,SUBBANDY=301,SIGNALY=82.2
-20130522,LBH,013,RF_FAIL,Y=72.9
-20130522,LBH,013,LOW_NOISE,Yproc=100.000,Yval=66.0,Ydiff=0.1,Yref=67.8
-20130522,LBH,015,RF_FAIL,X=68.0
-20130522,LBH,015,LOW_NOISE,Xproc=100.000,Xval=67.8,Xdiff=0.0,Xref=68.0
-20130522,HBA,003,E_FAIL,HNX13=64.3 2.3,JX13=1.0
-20130522,HBA,009,E_FAIL,HNX8=62.7 0.9
-20130522,HBA,010,HIGH_NOISE,Xproc=1.333,Xval=73.3,Xdiff=3.9,Xref=70.8,Yproc=0.500,Yval=71.5,Ydiff=3.1,Yref=70.9
-20130522,HBA,010,JITTER,Xproc=92.278,Xdiff=3.9,Xref=0.1,Yproc=99.611,Ydiff=3.1,Yref=0.2
-20130522,HBA,010,E_FAIL,HNY3=63.7 2.5,JY3=1.0,HNY12=64.0 2.5,JY12=1.0
-20130522,HBA,015,HIGH_NOISE,Xproc=8.278,Xval=74.2,Xdiff=5.6,Xref=70.8
-20130522,HBA,015,JITTER,Xproc=98.500,Xdiff=5.6,Xref=0.1
-20130522,HBA,015,SUMMATOR_NOISE,Y=1
-20130522,HBA,015,E_FAIL,HNX12=66.4 5.5,JX12=1.0
-20130522,HBA,019,JITTER,Xproc=99.333,Xdiff=1.9,Xref=0.1
-20130522,HBA,026,RF_FAIL,X=106.4 357 121.0 107.8 357 121.2
-20130522,HBA,036,SUMMATOR_NOISE,X=1,Y=1
-20130522,HBA,036,E_FAIL,SPX9=1,SPY9=1,SPX10=1,SPY10=1,SPX13=1,SPY13=1,SPX14=1,SPY14=1
- :param content: list of strings representing the content of the file
- :return: True if the content refers to a RTSM and it is not empty
- """
- if content:
- pattern = r'(^#.*)|(^\d*,[A-z]{3},[\d-]{3})'
- number_of_lines = len(content.splitlines())
- pattern_matching_lines = re.findall(pattern, content, re.MULTILINE)
- number_pattern_matching_lines = len(pattern_matching_lines)
- logger.debug('Station Test: number of matched lines %d/%d number of lines in files',
- number_pattern_matching_lines,
- number_of_lines)
-
- return number_of_lines == number_pattern_matching_lines
- else:
- return False
-
-
-def is_rtsm_test(content):
- """
- Check if the content of the file have the structure of a RTSM
- Example content of a valid RTSM test
-# SPECTRA-INFO=rcu,rcumode,obs-id,check,startfreq,stopfreq,rec-timestamp
-#
-SPECTRA-INFO=4,5,641999,SN,100,200,1520208664.423739
-MEAN-SPECTRA=[0.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.1 60.0 60.0 60.1 60.1 60.1 60.2 60.1 60.0 60.1 60.0 60.9 60.7 60.1 60.9 60.0 60.3 60.1 60.0 61.5 60.1 61.0 62.9 73.4 60.9 64.0 61.7 62.4 70.3 64.5 62.4 72.7 68.6 64.0 61.8 62.0 62.3 62.5 62.7 62.9 63.1 63.5 63.7 63.9 64.0 64.1 64.3 64.6 64.6 64.6 64.8 64.9 65.1 65.2 65.3 65.3 65.3 65.5 65.6 65.7 65.7 65.8 66.0 66.2 66.3 66.3 66.4 66.6 66.8 66.8 66.9 66.9 66.9 67.1 67.2 67.2 67.2 67.2 67.4 67.5 67.6 67.5 67.6 67.7 67.9 68.0 68.0 68.0 68.1 68.2 68.3 68.4 68.4 68.4 68.6 68.7 68.7 68.7 68.6 68.7 68.8 68.9 68.8 68.8 68.9 68.9 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.1 69.1 69.1 69.0 69.1 69.1 69.2 69.2 69.1 69.1 69.6 69.2 69.2 69.1 69.1 69.1 69.1 69.1 69.0 69.0 69.0 68.9 69.0 68.9 68.9 68.8 68.8 68.9 68.9 68.8 68.7 68.7 68.8 68.9 68.8 68.7 68.8 68.8 68.8 68.8 68.7 68.7 68.7 68.8 68.7 68.7 68.6 68.7 68.8 68.7 68.7 68.5 68.6 68.7 68.7 68.7 68.6 68.5 68.6 68.6 68.6 68.5 68.4 68.5 68.5 68.4 68.4 76.1 71.0 68.4 68.4 68.4 68.3 68.3 68.3 68.3 68.3 68.2 68.2 68.2 68.2 68.3 68.1 68.0 68.0 67.9 67.9 67.9 67.9 67.8 67.8 67.8 67.8 67.8 67.7 67.6 67.7 67.8 67.7 67.6 67.6 67.6 67.7 67.7 67.6 67.5 67.5 67.6 67.5 67.5 67.4 67.4 67.5 67.5 67.5 67.3 67.3 67.5 67.5 67.5 67.4 67.4 67.4 67.5 67.4 67.4 67.3 67.3 67.4 67.4 67.3 67.3 67.5 67.3 67.4 67.3 67.3 67.2 67.3 67.3 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.3 67.2 67.3 67.6 67.3 67.4 67.4 67.5 67.5 67.5 67.5 67.6 67.6 67.7 67.7 67.7 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.9 67.9 67.9 67.9 67.9 68.0 68.0 68.0 68.1 68.2 68.2 68.2 68.2 68.2 68.3 68.4 68.3 68.3 68.3 68.4 68.4 68.4 68.2 68.1 68.2 68.2 68.2 68.1 68.1 68.1 68.1 68.1 68.1 68.0 68.0 68.1 68.1 68.0 67.9 67.8 67.9 67.9 67.9 67.8 67.8 67.8 67.9 67.9 67.8 67.7 67.7 67.8 67.8 67.7 67.7 67.6 67.7 67.7 67.6 67.6 67.7 67.6 67.6 67.5 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.3 67.4 67.4 67.4 67.4 67.4 67.3 67.4 67.3 67.3 67.3 67.3 67.3 67.3 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.0 67.0 66.9 66.9 79.8 85.4 85.4 85.2 85.7 85.4 85.1 85.9 83.0 66.3 66.2 66.2 66.2 66.1 66.0 65.9 65.9 65.9 65.8 65.7 65.6 65.6 65.6 65.5 65.4 65.3 65.3 83.8 92.0 91.5 91.0 90.5 90.9 90.3 90.5 88.3 65.0 64.7 64.7 64.6 64.5 64.4 64.4 64.3 64.2 64.1 64.0 63.9 63.8 63.7 63.5 63.4 63.5 63.4 63.3 63.1 63.0 62.8 62.7 62.6 62.5 62.3 62.2 62.1 62.2 61.9 61.8 61.6 61.6 61.5 61.4 61.3 68.2 69.7 69.5 69.5 68.8 68.5 68.9 67.8 64.1 60.9 60.9 60.9 60.8 60.9 60.8 60.8 60.7 60.6 60.6 60.5 60.4 60.3 60.2 60.2 60.1 60.1 60.0 60.0 60.1 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.1 60.4 60.3 ]
-BAD-SPECTRA=[0.0 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.8 59.8 59.8 59.8 59.8 59.8 60.3 59.8 59.8 59.8 59.8 63.1 62.5 59.8 61.3 59.9 60.2 59.9 59.8 63.6 59.9 61.0 60.6 60.4 61.8 68.8 64.5 62.4 72.1 66.5 63.5 72.9 71.5 66.2 62.7 64.2 64.6 64.1 66.0 66.1 66.1 66.9 68.2 68.4 67.4 69.5 69.3 68.2 70.3 69.7 69.7 69.1 70.7 70.6 69.3 71.2 70.6 69.6 71.1 70.5 70.6 69.3 70.9 70.9 69.6 71.4 71.0 70.4 71.0 71.3 70.8 69.2 70.4 69.8 68.8 70.1 69.9 69.8 69.9 71.1 71.0 69.9 71.6 71.2 70.3 71.7 71.1 70.9 70.2 71.1 70.8 69.8 70.8 70.4 70.1 70.6 70.5 70.4 69.7 70.6 70.5 69.8 70.5 70.2 69.8 70.0 70.3 70.1 69.5 70.2 70.0 69.6 70.2 69.9 69.7 69.6 70.1 69.9 69.5 69.9 69.8 69.6 70.0 69.7 69.6 69.4 69.9 69.7 69.5 69.8 69.6 69.6 69.9 69.8 69.9 69.5 70.0 70.0 69.7 70.3 70.1 70.0 70.1 70.6 70.6 69.9 70.8 70.6 70.0 70.9 70.6 70.3 69.9 70.5 70.2 69.6 70.0 69.6 69.3 69.7 69.5 69.3 69.0 69.2 69.2 68.9 68.9 68.7 68.6 68.8 68.8 68.6 68.5 68.5 68.6 68.6 68.6 68.5 68.4 68.5 68.6 68.5 68.4 79.2 73.0 68.4 68.5 68.4 68.3 68.3 68.5 68.4 68.4 68.6 68.5 68.4 68.6 68.6 68.5 68.3 68.5 68.4 68.2 68.6 68.4 68.2 68.3 68.4 68.4 68.1 68.3 68.2 67.9 68.2 68.0 67.8 67.7 67.7 67.7 67.6 67.6 67.4 67.3 67.5 67.5 67.3 67.2 67.3 67.3 67.3 67.2 67.1 67.1 67.2 67.3 67.2 67.1 67.1 67.1 67.2 67.2 67.1 67.0 67.1 67.1 67.2 67.0 67.0 67.3 67.1 67.2 67.1 67.0 67.0 67.1 67.2 67.1 67.1 67.1 67.1 67.3 67.2 67.1 67.2 67.2 67.3 67.2 67.3 67.3 67.3 67.5 67.4 67.4 67.5 67.5 67.6 67.6 67.6 67.8 67.7 67.8 67.9 67.8 67.9 67.8 67.8 67.8 67.7 67.8 67.7 67.7 67.9 67.7 67.7 67.8 67.8 67.9 67.9 67.9 68.0 68.0 68.1 68.0 68.0 68.3 68.2 68.3 68.2 68.3 68.5 68.4 68.4 68.2 68.1 68.2 68.1 68.1 68.0 67.9 68.0 67.9 67.9 67.9 67.8 67.9 67.9 67.8 67.8 67.7 67.6 67.7 67.7 67.7 67.6 67.6 67.6 67.6 67.6 67.5 67.4 67.4 67.5 67.5 67.4 67.4 67.3 67.4 67.4 67.3 67.3 67.5 67.3 67.3 67.3 67.3 67.2 67.2 67.2 67.3 67.2 67.2 67.2 67.2 67.2 67.3 67.2 67.3 67.2 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.1 67.0 67.0 67.0 67.0 67.0 66.9 66.9 66.9 84.6 90.8 91.0 91.1 90.9 91.4 91.4 91.2 89.1 66.3 66.3 66.3 66.2 66.1 66.0 66.0 66.0 66.0 65.9 65.8 65.7 65.7 65.7 65.6 65.5 65.4 65.4 84.9 90.8 91.5 90.2 90.7 90.9 90.1 90.5 88.6 65.0 64.8 64.8 64.7 64.6 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.5 63.4 63.5 63.4 63.3 63.1 62.9 62.8 62.6 62.6 62.4 62.3 62.1 62.0 62.0 61.8 61.8 61.6 61.5 61.4 61.3 61.3 64.0 65.2 65.9 65.6 66.5 67.0 66.9 67.1 62.9 60.9 60.9 60.9 60.8 60.8 60.8 60.7 60.7 60.6 60.6 60.5 60.4 60.3 60.2 60.1 60.0 60.0 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.8 59.9 59.9 59.8 59.9 60.1 60.2 60.3 ]
-
-SPECTRA-INFO=4,5,641999,SN,100,200,1520208724.588988
-MEAN-SPECTRA=[0.0 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.9 59.8 59.7 59.7 60.3 60.9 60.3 59.8 60.7 59.7 60.0 59.7 59.7 61.1 59.9 60.6 62.7 73.3 60.6 63.7 61.4 62.3 70.5 64.5 62.3 72.8 68.4 64.5 61.7 61.9 62.3 62.4 64.5 62.8 63.0 63.4 63.6 63.8 63.9 64.0 64.3 64.5 64.6 64.7 64.8 64.9 65.2 65.3 65.4 65.4 65.4 65.5 65.7 65.7 65.7 65.8 66.0 66.2 66.3 66.3 66.4 66.4 66.7 66.8 66.8 66.8 66.8 67.1 67.2 67.2 67.2 67.2 67.4 67.5 67.6 67.6 67.7 67.8 68.0 68.1 68.1 68.0 68.2 68.3 68.4 68.4 68.3 68.3 68.6 68.6 68.6 68.6 68.6 68.7 68.8 69.0 69.2 68.8 68.8 68.9 69.0 69.0 69.0 69.0 69.0 69.1 69.2 69.1 69.1 69.1 69.2 69.2 69.2 69.1 69.2 69.2 69.2 69.2 69.2 69.1 69.6 69.2 69.2 69.2 69.1 69.1 69.1 69.1 69.0 68.9 68.9 68.9 69.0 69.0 68.9 68.8 68.9 68.9 69.0 68.9 68.8 68.8 68.9 68.9 68.9 68.8 68.8 68.8 68.9 68.9 68.7 68.7 68.7 68.8 68.7 68.6 68.6 68.6 68.8 68.7 68.6 68.5 68.5 68.7 68.7 68.6 68.5 68.5 68.6 68.5 68.6 68.4 68.4 68.5 68.5 68.4 68.4 76.0 71.1 68.5 68.5 68.4 68.3 68.3 68.3 68.3 68.3 72.2 68.2 68.2 68.2 68.3 68.1 68.1 68.0 67.9 68.0 67.9 67.9 67.8 67.8 67.8 67.8 67.7 67.7 67.7 67.7 67.7 67.7 67.6 67.6 67.5 67.6 67.6 67.5 67.4 67.5 67.5 67.5 67.4 67.3 67.3 67.4 67.5 67.3 67.3 67.2 67.5 67.4 67.4 67.3 67.2 67.3 67.3 67.4 67.3 67.2 67.2 67.3 67.4 67.3 67.3 67.5 67.3 67.4 67.3 67.3 67.2 67.3 67.4 67.4 67.9 67.3 67.4 67.5 67.4 67.4 67.4 67.4 67.4 67.5 67.3 67.5 67.4 67.5 67.5 67.5 67.5 67.5 67.5 67.6 67.6 67.6 67.7 67.7 67.7 67.7 67.7 67.7 68.1 67.8 67.7 67.8 67.8 67.9 67.9 67.9 67.9 67.9 68.0 68.0 68.1 68.1 68.2 68.2 68.2 68.1 68.1 68.2 68.3 68.3 68.2 68.2 68.3 68.3 68.3 68.1 68.1 68.1 68.2 68.2 68.1 68.1 68.1 68.2 68.2 68.1 68.0 68.0 68.1 68.1 68.0 67.9 67.8 67.9 67.9 67.9 67.8 67.8 67.8 67.8 67.8 67.8 67.7 67.7 67.7 67.8 67.7 67.7 67.7 67.7 67.8 91.2 125.0 67.8 67.6 67.6 67.6 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.5 67.5 67.4 67.4 67.4 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.3 67.3 67.3 67.3 67.2 67.1 67.1 67.1 67.1 67.0 67.0 66.9 66.9 80.0 85.6 85.5 85.5 85.7 85.6 85.3 85.8 83.1 66.3 66.3 66.3 66.2 66.2 66.0 66.0 66.0 65.9 65.8 65.8 65.7 65.6 65.6 65.9 65.4 65.3 65.3 83.5 91.8 91.4 91.0 90.4 90.8 90.1 90.2 88.2 65.0 64.7 64.7 64.6 64.5 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.5 63.6 63.5 63.4 63.2 63.1 62.9 62.8 62.7 62.6 62.4 62.3 62.1 62.1 62.1 61.9 61.8 61.6 61.5 61.4 61.4 61.3 68.3 69.9 69.7 69.7 69.0 68.6 68.8 67.9 64.1 60.9 60.8 60.8 60.7 60.7 60.7 60.7 60.5 60.5 60.4 60.4 60.3 60.2 60.2 60.1 60.0 60.0 60.0 59.9 60.0 60.0 60.0 59.9 59.9 59.9 59.9 59.9 59.9 59.8 59.8 59.8 59.8 59.9 60.0 ]
-BAD-SPECTRA=[0.0 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 60.1 59.6 59.5 59.5 60.5 63.3 61.7 59.5 61.0 59.6 60.0 59.6 59.5 63.4 59.6 60.9 60.4 60.0 61.6 68.9 64.1 62.3 72.0 66.4 63.4 73.0 71.4 67.0 62.7 64.2 64.7 64.0 67.6 66.1 66.2 66.8 68.3 68.4 67.3 69.5 69.3 68.2 70.3 69.7 69.8 69.1 70.8 70.7 69.3 71.3 70.7 69.7 71.1 70.7 70.6 69.3 71.0 70.9 69.6 71.4 71.0 70.3 70.9 71.3 70.8 69.2 70.4 69.8 68.8 70.2 70.0 69.9 70.0 71.2 71.1 70.0 71.7 71.3 70.4 71.7 71.1 70.9 70.1 71.0 70.7 69.7 70.7 70.2 69.9 70.5 70.4 70.3 69.6 70.6 70.5 69.9 70.7 70.2 69.9 70.0 70.4 70.2 69.6 70.3 70.1 69.7 70.4 70.0 69.9 69.8 70.2 70.0 69.6 70.0 69.8 69.6 70.0 69.7 69.6 69.4 69.9 69.6 69.5 69.7 69.5 69.5 69.8 69.7 69.8 69.4 70.0 70.0 69.6 70.3 70.1 70.0 70.1 70.6 70.7 70.0 70.9 70.6 70.1 71.0 70.7 70.4 70.0 70.6 70.2 69.6 70.1 69.7 69.4 69.7 69.5 69.3 68.9 69.2 69.1 68.8 68.9 68.6 68.6 68.7 68.6 68.5 68.4 68.4 68.5 68.5 68.5 68.4 68.4 68.4 68.6 68.5 68.4 75.4 70.5 68.4 68.6 68.5 68.4 68.4 68.6 68.5 68.4 70.5 68.5 68.5 68.7 68.6 68.6 68.3 68.6 68.5 68.3 68.6 68.4 68.2 68.3 68.4 68.4 68.1 68.3 68.1 67.9 68.1 68.0 67.7 67.6 67.7 67.6 67.5 67.5 67.3 67.3 67.4 67.4 67.2 67.1 67.1 67.2 67.3 67.1 67.0 67.0 67.2 67.2 67.1 67.0 67.0 67.1 67.2 67.2 67.1 67.1 67.1 67.2 67.2 67.1 67.1 67.4 67.2 67.3 67.1 67.1 67.1 67.2 67.3 67.1 67.4 67.2 67.2 67.3 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.3 67.3 67.5 67.4 67.4 67.4 67.4 67.5 67.4 67.5 67.6 67.5 67.7 67.6 67.6 67.6 67.6 67.8 67.7 67.7 67.8 67.7 67.8 67.9 67.8 67.8 67.9 67.9 68.0 67.9 68.0 68.0 68.0 68.2 68.0 67.9 68.2 68.1 68.2 68.0 68.0 68.2 68.1 68.2 67.9 67.9 68.0 68.0 68.0 67.9 67.9 67.9 68.0 68.0 67.9 67.8 67.9 67.9 67.9 67.9 67.7 67.7 67.7 67.7 67.7 67.6 67.5 67.6 67.6 67.6 67.5 67.4 67.4 67.5 67.5 67.4 67.5 67.4 67.5 67.5 94.6 128.3 67.6 67.4 67.4 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.1 67.0 67.0 67.0 67.0 67.0 66.9 66.9 85.0 90.9 91.1 91.3 91.2 91.7 91.7 91.4 89.3 66.4 66.3 66.3 66.3 66.2 66.0 66.0 66.0 66.0 65.9 65.9 65.7 65.7 65.7 66.3 65.6 65.4 65.4 84.9 90.9 91.4 90.5 90.6 91.1 90.1 90.6 88.4 65.0 64.8 64.8 64.7 64.6 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.6 64.2 63.5 63.4 63.3 63.1 62.9 62.8 62.7 62.6 62.4 62.3 62.1 62.1 62.0 61.9 61.7 61.6 61.5 61.4 61.4 61.3 64.0 65.2 65.8 65.6 66.4 66.9 66.8 67.0 62.8 60.8 60.8 60.8 60.7 60.7 60.6 60.6 60.5 60.5 60.4 60.3 60.2 60.2 60.1 60.0 59.9 59.9 59.8 59.8 59.8 59.8 59.8 59.8 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.8 59.9 ]
-
-# OBS-ID-INFO=obsid,start_time,stop_time,obsid_samples
-OBS-ID-INFO=641999,1520208240.000,1520211840.000,59
- :param content: list of strings representing the content of the file
- :return: True if the content refers to a RTSM and it is not empty
- """
- if content:
- pattern = r'(^#.*)|(^\w*-\w*=)|(^\w*-\w*-\w*=)|(^$)'
- number_of_lines = len(content.splitlines())
- pattern_matching_lines = re.findall(pattern, content, re.MULTILINE)
- number_pattern_matching_lines = len(pattern_matching_lines)
- logger.debug('RTSM: number of matched lines %d/%d number of lines in files',
- number_pattern_matching_lines,
- number_of_lines)
- return number_of_lines == number_pattern_matching_lines
- else:
- return False
-
-
def send_stationtest_rtsm_content_to_address(address, content):
"""
Send the stationtest RTSM content to the web site API at the given address
@@ -117,10 +39,13 @@ def send_stationtest_rtsm_content_to_address(address, content):
:return: True if the request was successful False otherwise
"""
full_address = '/'.join([address, compose_api_url_for_given_test_type(content['content'])])
+ file_path = content.pop('file_path')
+ content.pop('content')
logger.info('performing request to address %s', full_address)
logger.debug('request content %s', content)
- response = requests.post(full_address, data=content)
+ with open(file_path, 'rb') as file_stream:
+ response = requests.post(full_address, data=content, files=dict(upload_file=file_stream))
logger.info('response acknowledged: status code is %s, reason %s, content %s', response.status_code,
response.reason,
response.content)
diff --git a/LCU/Maintenance/MDB_tools/lib/tests_validators.py b/LCU/Maintenance/MDB_tools/lib/tests_validators.py
new file mode 100644
index 0000000000000000000000000000000000000000..ef5271e377f1fbe7f0ae7a8a07f36cef713d5fa8
--- /dev/null
+++ b/LCU/Maintenance/MDB_tools/lib/tests_validators.py
@@ -0,0 +1,82 @@
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def is_rtsm_test(content):
+ """
+ Check if the content of the file have the structure of a RTSM
+ Example content of a valid RTSM test
+# SPECTRA-INFO=rcu,rcumode,obs-id,check,startfreq,stopfreq,rec-timestamp
+#
+SPECTRA-INFO=4,5,641999,SN,100,200,1520208664.423739
+MEAN-SPECTRA=[0.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.1 60.0 60.0 60.1 60.1 60.1 60.2 60.1 60.0 60.1 60.0 60.9 60.7 60.1 60.9 60.0 60.3 60.1 60.0 61.5 60.1 61.0 62.9 73.4 60.9 64.0 61.7 62.4 70.3 64.5 62.4 72.7 68.6 64.0 61.8 62.0 62.3 62.5 62.7 62.9 63.1 63.5 63.7 63.9 64.0 64.1 64.3 64.6 64.6 64.6 64.8 64.9 65.1 65.2 65.3 65.3 65.3 65.5 65.6 65.7 65.7 65.8 66.0 66.2 66.3 66.3 66.4 66.6 66.8 66.8 66.9 66.9 66.9 67.1 67.2 67.2 67.2 67.2 67.4 67.5 67.6 67.5 67.6 67.7 67.9 68.0 68.0 68.0 68.1 68.2 68.3 68.4 68.4 68.4 68.6 68.7 68.7 68.7 68.6 68.7 68.8 68.9 68.8 68.8 68.9 68.9 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.0 69.1 69.1 69.1 69.0 69.1 69.1 69.2 69.2 69.1 69.1 69.6 69.2 69.2 69.1 69.1 69.1 69.1 69.1 69.0 69.0 69.0 68.9 69.0 68.9 68.9 68.8 68.8 68.9 68.9 68.8 68.7 68.7 68.8 68.9 68.8 68.7 68.8 68.8 68.8 68.8 68.7 68.7 68.7 68.8 68.7 68.7 68.6 68.7 68.8 68.7 68.7 68.5 68.6 68.7 68.7 68.7 68.6 68.5 68.6 68.6 68.6 68.5 68.4 68.5 68.5 68.4 68.4 76.1 71.0 68.4 68.4 68.4 68.3 68.3 68.3 68.3 68.3 68.2 68.2 68.2 68.2 68.3 68.1 68.0 68.0 67.9 67.9 67.9 67.9 67.8 67.8 67.8 67.8 67.8 67.7 67.6 67.7 67.8 67.7 67.6 67.6 67.6 67.7 67.7 67.6 67.5 67.5 67.6 67.5 67.5 67.4 67.4 67.5 67.5 67.5 67.3 67.3 67.5 67.5 67.5 67.4 67.4 67.4 67.5 67.4 67.4 67.3 67.3 67.4 67.4 67.3 67.3 67.5 67.3 67.4 67.3 67.3 67.2 67.3 67.3 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.3 67.2 67.3 67.6 67.3 67.4 67.4 67.5 67.5 67.5 67.5 67.6 67.6 67.7 67.7 67.7 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.8 67.9 67.9 67.9 67.9 67.9 68.0 68.0 68.0 68.1 68.2 68.2 68.2 68.2 68.2 68.3 68.4 68.3 68.3 68.3 68.4 68.4 68.4 68.2 68.1 68.2 68.2 68.2 68.1 68.1 68.1 68.1 68.1 68.1 68.0 68.0 68.1 68.1 68.0 67.9 67.8 67.9 67.9 67.9 67.8 67.8 67.8 67.9 67.9 67.8 67.7 67.7 67.8 67.8 67.7 67.7 67.6 67.7 67.7 67.6 67.6 67.7 67.6 67.6 67.5 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.3 67.4 67.4 67.4 67.4 67.4 67.3 67.4 67.3 67.3 67.3 67.3 67.3 67.3 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.0 67.0 66.9 66.9 79.8 85.4 85.4 85.2 85.7 85.4 85.1 85.9 83.0 66.3 66.2 66.2 66.2 66.1 66.0 65.9 65.9 65.9 65.8 65.7 65.6 65.6 65.6 65.5 65.4 65.3 65.3 83.8 92.0 91.5 91.0 90.5 90.9 90.3 90.5 88.3 65.0 64.7 64.7 64.6 64.5 64.4 64.4 64.3 64.2 64.1 64.0 63.9 63.8 63.7 63.5 63.4 63.5 63.4 63.3 63.1 63.0 62.8 62.7 62.6 62.5 62.3 62.2 62.1 62.2 61.9 61.8 61.6 61.6 61.5 61.4 61.3 68.2 69.7 69.5 69.5 68.8 68.5 68.9 67.8 64.1 60.9 60.9 60.9 60.8 60.9 60.8 60.8 60.7 60.6 60.6 60.5 60.4 60.3 60.2 60.2 60.1 60.1 60.0 60.0 60.1 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.1 60.4 60.3 ]
+BAD-SPECTRA=[0.0 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.8 59.8 59.8 59.8 59.8 59.8 60.3 59.8 59.8 59.8 59.8 63.1 62.5 59.8 61.3 59.9 60.2 59.9 59.8 63.6 59.9 61.0 60.6 60.4 61.8 68.8 64.5 62.4 72.1 66.5 63.5 72.9 71.5 66.2 62.7 64.2 64.6 64.1 66.0 66.1 66.1 66.9 68.2 68.4 67.4 69.5 69.3 68.2 70.3 69.7 69.7 69.1 70.7 70.6 69.3 71.2 70.6 69.6 71.1 70.5 70.6 69.3 70.9 70.9 69.6 71.4 71.0 70.4 71.0 71.3 70.8 69.2 70.4 69.8 68.8 70.1 69.9 69.8 69.9 71.1 71.0 69.9 71.6 71.2 70.3 71.7 71.1 70.9 70.2 71.1 70.8 69.8 70.8 70.4 70.1 70.6 70.5 70.4 69.7 70.6 70.5 69.8 70.5 70.2 69.8 70.0 70.3 70.1 69.5 70.2 70.0 69.6 70.2 69.9 69.7 69.6 70.1 69.9 69.5 69.9 69.8 69.6 70.0 69.7 69.6 69.4 69.9 69.7 69.5 69.8 69.6 69.6 69.9 69.8 69.9 69.5 70.0 70.0 69.7 70.3 70.1 70.0 70.1 70.6 70.6 69.9 70.8 70.6 70.0 70.9 70.6 70.3 69.9 70.5 70.2 69.6 70.0 69.6 69.3 69.7 69.5 69.3 69.0 69.2 69.2 68.9 68.9 68.7 68.6 68.8 68.8 68.6 68.5 68.5 68.6 68.6 68.6 68.5 68.4 68.5 68.6 68.5 68.4 79.2 73.0 68.4 68.5 68.4 68.3 68.3 68.5 68.4 68.4 68.6 68.5 68.4 68.6 68.6 68.5 68.3 68.5 68.4 68.2 68.6 68.4 68.2 68.3 68.4 68.4 68.1 68.3 68.2 67.9 68.2 68.0 67.8 67.7 67.7 67.7 67.6 67.6 67.4 67.3 67.5 67.5 67.3 67.2 67.3 67.3 67.3 67.2 67.1 67.1 67.2 67.3 67.2 67.1 67.1 67.1 67.2 67.2 67.1 67.0 67.1 67.1 67.2 67.0 67.0 67.3 67.1 67.2 67.1 67.0 67.0 67.1 67.2 67.1 67.1 67.1 67.1 67.3 67.2 67.1 67.2 67.2 67.3 67.2 67.3 67.3 67.3 67.5 67.4 67.4 67.5 67.5 67.6 67.6 67.6 67.8 67.7 67.8 67.9 67.8 67.9 67.8 67.8 67.8 67.7 67.8 67.7 67.7 67.9 67.7 67.7 67.8 67.8 67.9 67.9 67.9 68.0 68.0 68.1 68.0 68.0 68.3 68.2 68.3 68.2 68.3 68.5 68.4 68.4 68.2 68.1 68.2 68.1 68.1 68.0 67.9 68.0 67.9 67.9 67.9 67.8 67.9 67.9 67.8 67.8 67.7 67.6 67.7 67.7 67.7 67.6 67.6 67.6 67.6 67.6 67.5 67.4 67.4 67.5 67.5 67.4 67.4 67.3 67.4 67.4 67.3 67.3 67.5 67.3 67.3 67.3 67.3 67.2 67.2 67.2 67.3 67.2 67.2 67.2 67.2 67.2 67.3 67.2 67.3 67.2 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.1 67.0 67.0 67.0 67.0 67.0 66.9 66.9 66.9 84.6 90.8 91.0 91.1 90.9 91.4 91.4 91.2 89.1 66.3 66.3 66.3 66.2 66.1 66.0 66.0 66.0 66.0 65.9 65.8 65.7 65.7 65.7 65.6 65.5 65.4 65.4 84.9 90.8 91.5 90.2 90.7 90.9 90.1 90.5 88.6 65.0 64.8 64.8 64.7 64.6 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.5 63.4 63.5 63.4 63.3 63.1 62.9 62.8 62.6 62.6 62.4 62.3 62.1 62.0 62.0 61.8 61.8 61.6 61.5 61.4 61.3 61.3 64.0 65.2 65.9 65.6 66.5 67.0 66.9 67.1 62.9 60.9 60.9 60.9 60.8 60.8 60.8 60.7 60.7 60.6 60.6 60.5 60.4 60.3 60.2 60.1 60.0 60.0 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.9 59.8 59.9 59.9 59.8 59.9 60.1 60.2 60.3 ]
+
+SPECTRA-INFO=4,5,641999,SN,100,200,1520208724.588988
+MEAN-SPECTRA=[0.0 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.8 59.9 59.8 59.7 59.7 60.3 60.9 60.3 59.8 60.7 59.7 60.0 59.7 59.7 61.1 59.9 60.6 62.7 73.3 60.6 63.7 61.4 62.3 70.5 64.5 62.3 72.8 68.4 64.5 61.7 61.9 62.3 62.4 64.5 62.8 63.0 63.4 63.6 63.8 63.9 64.0 64.3 64.5 64.6 64.7 64.8 64.9 65.2 65.3 65.4 65.4 65.4 65.5 65.7 65.7 65.7 65.8 66.0 66.2 66.3 66.3 66.4 66.4 66.7 66.8 66.8 66.8 66.8 67.1 67.2 67.2 67.2 67.2 67.4 67.5 67.6 67.6 67.7 67.8 68.0 68.1 68.1 68.0 68.2 68.3 68.4 68.4 68.3 68.3 68.6 68.6 68.6 68.6 68.6 68.7 68.8 69.0 69.2 68.8 68.8 68.9 69.0 69.0 69.0 69.0 69.0 69.1 69.2 69.1 69.1 69.1 69.2 69.2 69.2 69.1 69.2 69.2 69.2 69.2 69.2 69.1 69.6 69.2 69.2 69.2 69.1 69.1 69.1 69.1 69.0 68.9 68.9 68.9 69.0 69.0 68.9 68.8 68.9 68.9 69.0 68.9 68.8 68.8 68.9 68.9 68.9 68.8 68.8 68.8 68.9 68.9 68.7 68.7 68.7 68.8 68.7 68.6 68.6 68.6 68.8 68.7 68.6 68.5 68.5 68.7 68.7 68.6 68.5 68.5 68.6 68.5 68.6 68.4 68.4 68.5 68.5 68.4 68.4 76.0 71.1 68.5 68.5 68.4 68.3 68.3 68.3 68.3 68.3 72.2 68.2 68.2 68.2 68.3 68.1 68.1 68.0 67.9 68.0 67.9 67.9 67.8 67.8 67.8 67.8 67.7 67.7 67.7 67.7 67.7 67.7 67.6 67.6 67.5 67.6 67.6 67.5 67.4 67.5 67.5 67.5 67.4 67.3 67.3 67.4 67.5 67.3 67.3 67.2 67.5 67.4 67.4 67.3 67.2 67.3 67.3 67.4 67.3 67.2 67.2 67.3 67.4 67.3 67.3 67.5 67.3 67.4 67.3 67.3 67.2 67.3 67.4 67.4 67.9 67.3 67.4 67.5 67.4 67.4 67.4 67.4 67.4 67.5 67.3 67.5 67.4 67.5 67.5 67.5 67.5 67.5 67.5 67.6 67.6 67.6 67.7 67.7 67.7 67.7 67.7 67.7 68.1 67.8 67.7 67.8 67.8 67.9 67.9 67.9 67.9 67.9 68.0 68.0 68.1 68.1 68.2 68.2 68.2 68.1 68.1 68.2 68.3 68.3 68.2 68.2 68.3 68.3 68.3 68.1 68.1 68.1 68.2 68.2 68.1 68.1 68.1 68.2 68.2 68.1 68.0 68.0 68.1 68.1 68.0 67.9 67.8 67.9 67.9 67.9 67.8 67.8 67.8 67.8 67.8 67.8 67.7 67.7 67.7 67.8 67.7 67.7 67.7 67.7 67.8 91.2 125.0 67.8 67.6 67.6 67.6 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.5 67.5 67.4 67.4 67.4 67.5 67.4 67.4 67.4 67.4 67.4 67.4 67.4 67.3 67.3 67.3 67.3 67.2 67.1 67.1 67.1 67.1 67.0 67.0 66.9 66.9 80.0 85.6 85.5 85.5 85.7 85.6 85.3 85.8 83.1 66.3 66.3 66.3 66.2 66.2 66.0 66.0 66.0 65.9 65.8 65.8 65.7 65.6 65.6 65.9 65.4 65.3 65.3 83.5 91.8 91.4 91.0 90.4 90.8 90.1 90.2 88.2 65.0 64.7 64.7 64.6 64.5 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.5 63.6 63.5 63.4 63.2 63.1 62.9 62.8 62.7 62.6 62.4 62.3 62.1 62.1 62.1 61.9 61.8 61.6 61.5 61.4 61.4 61.3 68.3 69.9 69.7 69.7 69.0 68.6 68.8 67.9 64.1 60.9 60.8 60.8 60.7 60.7 60.7 60.7 60.5 60.5 60.4 60.4 60.3 60.2 60.2 60.1 60.0 60.0 60.0 59.9 60.0 60.0 60.0 59.9 59.9 59.9 59.9 59.9 59.9 59.8 59.8 59.8 59.8 59.9 60.0 ]
+BAD-SPECTRA=[0.0 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 59.6 60.1 59.6 59.5 59.5 60.5 63.3 61.7 59.5 61.0 59.6 60.0 59.6 59.5 63.4 59.6 60.9 60.4 60.0 61.6 68.9 64.1 62.3 72.0 66.4 63.4 73.0 71.4 67.0 62.7 64.2 64.7 64.0 67.6 66.1 66.2 66.8 68.3 68.4 67.3 69.5 69.3 68.2 70.3 69.7 69.8 69.1 70.8 70.7 69.3 71.3 70.7 69.7 71.1 70.7 70.6 69.3 71.0 70.9 69.6 71.4 71.0 70.3 70.9 71.3 70.8 69.2 70.4 69.8 68.8 70.2 70.0 69.9 70.0 71.2 71.1 70.0 71.7 71.3 70.4 71.7 71.1 70.9 70.1 71.0 70.7 69.7 70.7 70.2 69.9 70.5 70.4 70.3 69.6 70.6 70.5 69.9 70.7 70.2 69.9 70.0 70.4 70.2 69.6 70.3 70.1 69.7 70.4 70.0 69.9 69.8 70.2 70.0 69.6 70.0 69.8 69.6 70.0 69.7 69.6 69.4 69.9 69.6 69.5 69.7 69.5 69.5 69.8 69.7 69.8 69.4 70.0 70.0 69.6 70.3 70.1 70.0 70.1 70.6 70.7 70.0 70.9 70.6 70.1 71.0 70.7 70.4 70.0 70.6 70.2 69.6 70.1 69.7 69.4 69.7 69.5 69.3 68.9 69.2 69.1 68.8 68.9 68.6 68.6 68.7 68.6 68.5 68.4 68.4 68.5 68.5 68.5 68.4 68.4 68.4 68.6 68.5 68.4 75.4 70.5 68.4 68.6 68.5 68.4 68.4 68.6 68.5 68.4 70.5 68.5 68.5 68.7 68.6 68.6 68.3 68.6 68.5 68.3 68.6 68.4 68.2 68.3 68.4 68.4 68.1 68.3 68.1 67.9 68.1 68.0 67.7 67.6 67.7 67.6 67.5 67.5 67.3 67.3 67.4 67.4 67.2 67.1 67.1 67.2 67.3 67.1 67.0 67.0 67.2 67.2 67.1 67.0 67.0 67.1 67.2 67.2 67.1 67.1 67.1 67.2 67.2 67.1 67.1 67.4 67.2 67.3 67.1 67.1 67.1 67.2 67.3 67.1 67.4 67.2 67.2 67.3 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.3 67.3 67.5 67.4 67.4 67.4 67.4 67.5 67.4 67.5 67.6 67.5 67.7 67.6 67.6 67.6 67.6 67.8 67.7 67.7 67.8 67.7 67.8 67.9 67.8 67.8 67.9 67.9 68.0 67.9 68.0 68.0 68.0 68.2 68.0 67.9 68.2 68.1 68.2 68.0 68.0 68.2 68.1 68.2 67.9 67.9 68.0 68.0 68.0 67.9 67.9 67.9 68.0 68.0 67.9 67.8 67.9 67.9 67.9 67.9 67.7 67.7 67.7 67.7 67.7 67.6 67.5 67.6 67.6 67.6 67.5 67.4 67.4 67.5 67.5 67.4 67.5 67.4 67.5 67.5 94.6 128.3 67.6 67.4 67.4 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.3 67.2 67.2 67.2 67.3 67.3 67.3 67.2 67.2 67.2 67.2 67.2 67.2 67.2 67.1 67.1 67.1 67.1 67.0 67.0 67.0 67.0 67.0 66.9 66.9 85.0 90.9 91.1 91.3 91.2 91.7 91.7 91.4 89.3 66.4 66.3 66.3 66.3 66.2 66.0 66.0 66.0 66.0 65.9 65.9 65.7 65.7 65.7 66.3 65.6 65.4 65.4 84.9 90.9 91.4 90.5 90.6 91.1 90.1 90.6 88.4 65.0 64.8 64.8 64.7 64.6 64.5 64.4 64.4 64.3 64.1 64.0 63.9 63.8 63.7 63.6 64.2 63.5 63.4 63.3 63.1 62.9 62.8 62.7 62.6 62.4 62.3 62.1 62.1 62.0 61.9 61.7 61.6 61.5 61.4 61.4 61.3 64.0 65.2 65.8 65.6 66.4 66.9 66.8 67.0 62.8 60.8 60.8 60.8 60.7 60.7 60.6 60.6 60.5 60.5 60.4 60.3 60.2 60.2 60.1 60.0 59.9 59.9 59.8 59.8 59.8 59.8 59.8 59.8 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.7 59.8 59.9 ]
+
+# OBS-ID-INFO=obsid,start_time,stop_time,obsid_samples
+OBS-ID-INFO=641999,1520208240.000,1520211840.000,59
+ :param content: list of strings representing the content of the file
+ :return: True if the content refers to a RTSM and it is not empty
+ """
+ if content:
+ pattern = r'(^#.*)|(^\w*-\w*=)|(^\w*-\w*-\w*=)|(^$)'
+ number_of_lines = len(content.split('\n'))
+ pattern_matching_lines = re.findall(pattern, content, re.MULTILINE)
+ number_pattern_matching_lines = len(pattern_matching_lines)
+ logger.info('RTSM: number of matched lines %d/%d number of lines in files',
+ number_pattern_matching_lines,
+ number_of_lines)
+ return number_of_lines == number_pattern_matching_lines
+ else:
+ return False
+
+
+def is_station_test(content):
+ """
+ Check if the content of the file have the structure of a station test.
+ Example content
+20130522,NFO,---,STATION,NAME=CS011C
+20130522,NFO,---,RUNTIME,START=03:10:00,STOP=05:52:27
+20130522,NFO,---,CHECKS,S1,O1,SP1,NS1=180,S3,O3,SP3,NS3=180,M,O5,SN,SP5,NS5=300,EO,ESP,EN=30,TV,TM
+20130522,NFO,---,STATISTICS,BAD_LBL=1,BAD_LBH=2,BAD_HBA=7
+20130522,LBL,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=82.0,SUBBANDY=301,SIGNALY=81.8
+20130522,LBL,019,RF_FAIL,X=77.0,Y=70.1
+20130522,LBL,019,LOW_NOISE,Xproc=100.000,Xval=66.6,Xdiff=0.0,Xref=67.7
+20130522,LBH,---,TESTSIGNAL,SUBBANDX=301,SIGNALX=81.8,SUBBANDY=301,SIGNALY=82.2
+20130522,LBH,013,RF_FAIL,Y=72.9
+20130522,LBH,013,LOW_NOISE,Yproc=100.000,Yval=66.0,Ydiff=0.1,Yref=67.8
+20130522,LBH,015,RF_FAIL,X=68.0
+20130522,LBH,015,LOW_NOISE,Xproc=100.000,Xval=67.8,Xdiff=0.0,Xref=68.0
+20130522,HBA,003,E_FAIL,HNX13=64.3 2.3,JX13=1.0
+20130522,HBA,009,E_FAIL,HNX8=62.7 0.9
+20130522,HBA,010,HIGH_NOISE,Xproc=1.333,Xval=73.3,Xdiff=3.9,Xref=70.8,Yproc=0.500,Yval=71.5,Ydiff=3.1,Yref=70.9
+20130522,HBA,010,JITTER,Xproc=92.278,Xdiff=3.9,Xref=0.1,Yproc=99.611,Ydiff=3.1,Yref=0.2
+20130522,HBA,010,E_FAIL,HNY3=63.7 2.5,JY3=1.0,HNY12=64.0 2.5,JY12=1.0
+20130522,HBA,015,HIGH_NOISE,Xproc=8.278,Xval=74.2,Xdiff=5.6,Xref=70.8
+20130522,HBA,015,JITTER,Xproc=98.500,Xdiff=5.6,Xref=0.1
+20130522,HBA,015,SUMMATOR_NOISE,Y=1
+20130522,HBA,015,E_FAIL,HNX12=66.4 5.5,JX12=1.0
+20130522,HBA,019,JITTER,Xproc=99.333,Xdiff=1.9,Xref=0.1
+20130522,HBA,026,RF_FAIL,X=106.4 357 121.0 107.8 357 121.2
+20130522,HBA,036,SUMMATOR_NOISE,X=1,Y=1
+20130522,HBA,036,E_FAIL,SPX9=1,SPY9=1,SPX10=1,SPY10=1,SPX13=1,SPY13=1,SPX14=1,SPY14=1
+ :param content: list of strings representing the content of the file
+ :return: True if the content refers to a RTSM and it is not empty
+ """
+ if content:
+ pattern = r'(^#.*)|(^\d*,[A-z]{3},[\d-]{3})'
+ number_of_lines = len(content.splitlines())
+ pattern_matching_lines = re.findall(pattern, content, re.MULTILINE)
+ number_pattern_matching_lines = len(pattern_matching_lines)
+ logger.debug('Station Test: number of matched lines %d/%d number of lines in files',
+ number_pattern_matching_lines,
+ number_of_lines)
+
+ return number_of_lines == number_pattern_matching_lines
+ else:
+ return False