diff --git a/.gitattributes b/.gitattributes
index 47ab2464ced602305311ef62339eb7e65227be1b..0104140749993ad1bb25804be5d8fef8be46edc0 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1818,7 +1818,11 @@ LCU/Maintenance/DBInterface/monitoringdb/tests/old_tests.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_model_creation.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/t_stationtest_parser.py -text
LCU/Maintenance/DBInterface/monitoringdb/tests/test_stationtestdata.csv -text
-LCU/Maintenance/DBInterface/monitoringdb/views.py -text
+LCU/Maintenance/DBInterface/monitoringdb/urls.py -text
+LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py -text
+LCU/Maintenance/DBInterface/monitoringdb/views/common.py -text
+LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text
+LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text
LCU/Maintenance/MDB_tools/CMakeLists.txt -text
LCU/Maintenance/MDB_tools/cli/mdb_loader.py -text
LCU/Maintenance/MDB_tools/cli/probe_mdb.py -text
diff --git a/LCU/Maintenance/DBInterface/django_postgresql/urls.py b/LCU/Maintenance/DBInterface/django_postgresql/urls.py
index ba621a8f1fe4c79f9cf1db3b785be44225812913..0c4275a4c8be206ab7179b512db4a6da73260c5b 100644
--- a/LCU/Maintenance/DBInterface/django_postgresql/urls.py
+++ b/LCU/Maintenance/DBInterface/django_postgresql/urls.py
@@ -16,22 +16,8 @@ Including another URLconf
from django.contrib import admin
from django.conf.urls import url, include
-from rest_framework import routers
-from lofar.maintenance.monitoringdb import views
-
-router = routers.DefaultRouter()
-router.register(r'stationtests', views.StationTestViewSet)
-router.register(r'componenterrors', views.ComponentErrorViewSet)
-router.register(r'tileerrors', views.TileErrorViewSet)
-router.register(r'rtsm', views.RTSMObservationViewSet)
-
-
-
+from django.urls import path
urlpatterns = [
- url(r'^', include(router.urls)),
- url(r'^api-auth', include('rest_framework.urls', namespace='rest_framework')),
- url(r'^stationtests/insert_raw', views.insert_raw_station_test),
- url(r'^rtsm/insert_raw', views.insert_raw_rtsm_test)
-
+ path('', include('lofar.maintenance.monitoringdb.urls'))
]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
index 30736858521ea9f836d45b73626d10e832ef95ed..df2c879ce1c69f885031897fe9eaf34104fae83f 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py
@@ -1,5 +1,4 @@
from rest_framework import serializers
-from rest_polymorphic.serializers import PolymorphicSerializer
from ..models.fixed_types import COMPONENT_TYPES, ERROR_TYPES
from ..models.stationtest import StationTest
@@ -20,5 +19,3 @@ class ComponentErrorSerializer(serializers.Serializer):
"""
return ComponentError(**validated_data)
-
-
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
index f3f1bd4938d3a0f85155286d5dc108f6b962e77b..d86e41a65de7666f6499d82dce6a2ec97d2b84a0 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py
@@ -11,13 +11,23 @@ logger = logging.getLogger('serializers')
class RTSMErrorSerializer(serializers.ModelSerializer):
bad_spectrum = serializers.ListField(child=serializers.FloatField())
average_spectrum = serializers.ListField(child=serializers.FloatField())
+
class Meta:
model = RTSMError
fields = '__all__'
+class RTSMErrorSummarySerializer(serializers.ModelSerializer):
+ samples = serializers.IntegerField(source='observation.samples', read_only=True)
+
+ class Meta:
+ model = RTSMError
+ exclude = ['average_spectrum', 'bad_spectrum']
+
+
class RTSMObservationSerializer(serializers.ModelSerializer):
- errors = RTSMErrorSerializer(many=True)
+ errors = RTSMErrorSummarySerializer(many=True)
+
class Meta:
model = RTSMObservation
fields = '__all__'
@@ -27,7 +37,9 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
rtsm_errors = validated_data.pop('errors')
try:
start_time, station_name = validated_data['start_time'], validated_data['station_name']
-
+ station_type = station_name[0].capitalize()
+ station_type = station_type if station_type in ['C', 'R'] else 'I'
+ validated_data.update(station_type=station_type)
element = RTSMObservation.objects.filter(start_time=start_time,
station_name=station_name).first()
@@ -46,4 +58,4 @@ class RTSMObservationSerializer(serializers.ModelSerializer):
rtsm_error_instance = RTSMError.objects.create(**rtsm_error)
rtsm_error_instance.save()
- return RTSMObservation_instance
\ No newline at end of file
+ return RTSMObservation_instance
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
new file mode 100644
index 0000000000000000000000000000000000000000..31a0f176c78dd7830adaf4515ad499ea629907c6
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py
@@ -0,0 +1,28 @@
+from django.conf.urls import url, include
+from rest_framework import routers
+from .views import *
+
+station_test_router = routers.DefaultRouter()
+#Station test
+
+station_test_router.register(r'/componenterrors', ComponentErrorViewSet)
+station_test_router.register(r'/tileerrors', TileErrorViewSet)
+station_test_router.register(r'/errors_summary', ComponentErrorSummaryViewSet)
+station_test_router.register(r'^', StationTestViewSet)
+
+rtsm_router = routers.DefaultRouter()
+#RTSM
+rtsm_router.register(r'/errors_detailed', RTSMErrorsDetailedViewSet)
+rtsm_router.register(r'/errors', RTSMErrorsViewSet)
+rtsm_router.register(r'^', RTSMObservationViewSet)
+
+
+urlpatterns = [
+ url(r'stationtests', include(station_test_router.urls)),
+ url(r'rtsm', include(rtsm_router.urls)),
+
+ url(r'^api-auth', include('rest_framework.urls', namespace='rest_framework')),
+ url(r'^stationtests/insert_raw', insert_raw_station_test),
+ url(r'^rtsm/insert_raw', insert_raw_rtsm_test)
+
+]
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..763a142988430593ae0f899e5119491376c1555c
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py
@@ -0,0 +1,3 @@
+from .station_test_views import *
+from .common import *
+from .rtsm_views import *
\ No newline at end of file
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..f389d08a93f004768d72767ec7e34b7b75059ae6
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py
@@ -0,0 +1,7 @@
+from rest_framework import viewsets, status
+from rest_framework.response import Response
+from rest_framework.decorators import api_view
+import logging
+from ..exceptions import ItemAlreadyExists
+
+RESERVED_FILTER_NAME = ['limit', 'offset', 'format']
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdeb154c254f87a12f6d0f5b9702ea68a8cada78
--- /dev/null
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py
@@ -0,0 +1,104 @@
+from .common import *
+from ..models.rtsm import RTSMObservation, RTSMError
+from ..serializers.rtsm import RTSMObservationSerializer, RTSMErrorSummarySerializer, RTSMErrorSerializer
+from ..rtsm_test_raw_parser import parse_rtsm_test
+
+logger = logging.getLogger('views')
+
+class RTSMErrorsDetailedViewSet(viewsets.ModelViewSet):
+ queryset = RTSMError.objects.all()
+ serializer_class = RTSMErrorSerializer
+
+ def get_queryset(self):
+ queryset = RTSMError.objects.all()
+ for key, param in self.request.query_params.items():
+ if key in RESERVED_FILTER_NAME:
+ continue
+ queryset = queryset.filter(**{key: param})
+ return queryset
+
+
+class RTSMErrorsViewSet(viewsets.ModelViewSet):
+ queryset = RTSMError.objects.all()
+ serializer_class = RTSMErrorSummarySerializer
+
+ def get_queryset(self):
+ queryset = RTSMError.objects.all()
+ for key, param in self.request.query_params.items():
+ if key in RESERVED_FILTER_NAME:
+ continue
+ queryset = queryset.filter(**{key: param})
+ return queryset
+
+
+
+class RTSMObservationViewSet(viewsets.ModelViewSet):
+ queryset = RTSMObservation.objects.all()
+ serializer_class = RTSMObservationSerializer
+
+ def get_queryset(self):
+ queryset = RTSMObservation.objects.all()
+ for key, param in self.request.query_params.items():
+ if key in RESERVED_FILTER_NAME:
+ continue
+ queryset = queryset.filter(**{key: param})
+ return queryset
+
+
+@api_view(['POST'])
+def insert_raw_rtsm_test(request):
+ """
+ This function is meant to parse a request of the form
+ {
+ "content": "[RTSM TEST RAW TEXT]" station_name: "station_name"
+ }
+ parse the content field and create all the rtsm_test entity related into the database
+ :param request: HTTP request
+ :return:
+ """
+ if request.method == 'POST':
+
+ if 'content' in request.data:
+ try:
+ logger.debug('handling raw request for %s', request)
+ content = request.data['content']
+ station_name = request.data['station_name']
+ logger.debug('handling raw request for data %s', content)
+ entry = parse_rtsm_test(content)
+ entry.update(station_name=station_name)
+
+ logger.info('RTSM parsed successfully for obsid %d and station %s',
+ entry['observation_id'],
+ entry['station_name'])
+
+ try:
+ rtsm_item = RTSMObservationSerializer().create(entry)
+ rtsm_id = rtsm_item.id
+ except ItemAlreadyExists as e:
+ return Response(status=status.HTTP_200_OK,
+ data='RTSM test was already inserted with id {}\n'.format(e.instance_id))
+ logger.info('RTSM inserted successfully for obsid %d and station %s',
+ entry['observation_id'],
+ entry['station_name'])
+ except KeyError as e:
+ logger.exception("raw station info malformed %s: %s",
+ request.data['content'], e)
+ logger.info('station test malformed skipping insertion.')
+ return Response(status=status.HTTP_200_OK, data='skipping insertion RTSM malformed')
+ except Exception as e:
+ logger.exception("exception occurred while parsing raw station info %s: %s",
+ request.data['content'], e)
+ return Response(exception=True,
+ data="the post message is not correct." +
+ " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
+ e, request),
+ status=status.HTTP_400_BAD_REQUEST)
+ else:
+ return Response(exception=True,
+ data="the post message is not correct." +
+ " It has to be of the form \{'content':[RAWSTRING]\}",
+ status=status.HTTP_400_BAD_REQUEST)
+ logger.info('request processed correctly inserted')
+
+ return Response(status=status.HTTP_200_OK,
+ data='RTSM test has been inserted with id {}\n'.format(rtsm_id))
diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views.py b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
similarity index 56%
rename from LCU/Maintenance/DBInterface/monitoringdb/views.py
rename to LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
index 10f3d9da553d84227f420a7045499f29a09e3e76..1b7f2479bbafcf83a298dc06f3105836f8ab8906 100644
--- a/LCU/Maintenance/DBInterface/monitoringdb/views.py
+++ b/LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py
@@ -1,40 +1,40 @@
-from rest_framework import viewsets, status
-from rest_framework.response import Response
-from rest_framework.decorators import api_view
-from .models.stationtest import StationTest
-from .models.componenterror import ComponentError
-from .models.rtsm import RTSMObservation
-from .models.tileerror import TileError
+from .common import *
+from ..models.stationtest import StationTest
+from ..models.componenterror import ComponentError
+from ..models.tileerror import TileError
+from rest_framework.filters import SearchFilter
+from ..serializers.stationtest import StationTestSerializer
+from ..serializers.componenterrors_generic import ComponentErrorPolymorphicSerializer
+from ..serializers.tileerror import TileErrorPolymorphicSerializer
-from .serializers.stationtest import StationTestSerializer
-from .serializers.rtsm import RTSMObservationSerializer
-from .serializers.componenterrors_generic import ComponentErrorPolymorphicSerializer
-from .serializers.tileerror import TileErrorPolymorphicSerializer
+from ..station_test_raw_parser import parse_raw_station_test
-from .station_test_raw_parser import parse_raw_station_test
-from .rtsm_test_raw_parser import parse_rtsm_test
-import logging
-from .exceptions import ItemAlreadyExists
-logger = logging.getLogger('views')
+from ..exceptions import ItemAlreadyExists
-RESERVED_FILTER_NAME = ['limit', 'offset', 'format']
+logger = logging.getLogger('views')
-class RTSMObservationViewSet(viewsets.ModelViewSet):
- queryset = RTSMObservation.objects.all()
- serializer_class = RTSMObservationSerializer
+class ComponentErrorSummaryViewSet(viewsets.ModelViewSet):
+ queryset = ComponentError.objects.all()
+ serializer_class = ComponentErrorPolymorphicSerializer
def get_queryset(self):
- queryset = RTSMObservation.objects.all()
+ """
+ Optionally restrict the list of station test to a
+ specific station
+ time period
+ station_type
+ :return:
+ """
+ queryset = ComponentError.objects.all()
for key, param in self.request.query_params.items():
if key in RESERVED_FILTER_NAME:
continue
queryset = queryset.filter(**{key: param})
return queryset
-
# Create your views here.
class StationTestViewSet(viewsets.ModelViewSet):
queryset = StationTest.objects.all()
@@ -55,6 +55,7 @@ class StationTestViewSet(viewsets.ModelViewSet):
queryset = queryset.filter(**{key: param})
return queryset
+
class ComponentErrorViewSet(viewsets.ModelViewSet):
queryset = ComponentError.objects.all()
serializer_class = ComponentErrorPolymorphicSerializer
@@ -95,7 +96,6 @@ class TileErrorViewSet(viewsets.ModelViewSet):
return queryset
-
@api_view(['POST'])
def insert_raw_station_test(request):
"""
@@ -137,7 +137,8 @@ def insert_raw_station_test(request):
return Response(exception=True,
data="the post message is not correct." +
" It has to be of the form \{'content':[RAWSTRING]\}", status=status.HTTP_400_BAD_REQUEST)
-
+ logger.info('request processed correctly inserted %d of %d', len(station_test_ids),
+ len(station_test_ids)+len(station_test_ids_already_present))
return Response(status=status.HTTP_200_OK,
data='Station tests inserted with ids {} \n'.format(station_test_ids) +
'Station tests already present with ids {}'.format(
@@ -145,54 +146,4 @@ def insert_raw_station_test(request):
)
)
-@api_view(['POST'])
-def insert_raw_rtsm_test(request):
- """
- This function is meant to parse a request of the form
- {
- "content": "[RTSM TEST RAW TEXT]" station_name: "station_name"
- }
- parse the content field and create all the rtsm_test entity related into the database
- :param request: HTTP request
- :return:
- """
- if request.method == 'POST':
-
- if 'content' in request.data:
- try:
- logger.debug('handling raw request for %s', request)
- content = request.data['content']
- station_name = request.data['station_name']
- logger.debug('handling raw request for data %s', content)
- entry = parse_rtsm_test(content)
- entry.update(station_name=station_name)
-
- logger.info('RTSM parsed successfully for obsid %d and station %s',
- entry['observation_id'],
- entry['station_name'])
-
- try:
- rtsm_item = RTSMObservationSerializer().create(entry)
- rtsm_id = rtsm_item.id
- except ItemAlreadyExists as e:
- return Response(status=status.HTTP_200_OK,
- data='RTSM test was already inserted with id {}\n'.format(e.instance_id))
- logger.info('RTSM inserted successfully for obsid %d and station %s',
- entry['observation_id'],
- entry['station_name'])
- except Exception as e:
- logger.exception("exception occurred while parsing raw station info %s: %s",
- request.data['content'], e)
- return Response(exception=True,
- data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}: %s. Request provided %s" % (
- e, request),
- status=status.HTTP_400_BAD_REQUEST)
- else:
- return Response(exception=True,
- data="the post message is not correct." +
- " It has to be of the form \{'content':[RAWSTRING]\}",
- status=status.HTTP_400_BAD_REQUEST)
- return Response(status=status.HTTP_200_OK,
- data='RTSM test has been inserted with id {}\n'.format(rtsm_id))
diff --git a/LCU/Maintenance/MDB_tools/fabfile.py b/LCU/Maintenance/MDB_tools/fabfile.py
index ebed623637548dab195cceffd51dfbbbbf47d77b..d1bae4986a4a6034dcdac2510f1803348610ef1e 100644
--- a/LCU/Maintenance/MDB_tools/fabfile.py
+++ b/LCU/Maintenance/MDB_tools/fabfile.py
@@ -73,14 +73,25 @@ def run_server(c, type):
@task
def tear_down_server(c, type):
deploy_dir = os.path.join(REMOTE_DIR, type)
- log_path = os.path.join(deploy_dir, LOG_FILE)
+ stop_script = os.path.join(deploy_dir, MANAGE_SCRIPT)
with _connection(type) as c:
- user = c.user
- print(c.run('killall -u {} gunicorn'.format(user)))
- print(log_path)
- print(c.run('tail {}'.format(log_path)))
+ log_path = os.path.join(deploy_dir, LOG_FILE)
+ with c.cd(deploy_dir):
+ print(c.run('sh {} stop'.format(stop_script)))
+ print(c.run('tail {}'.format(log_path)))
+
@task
+def update(c):
+ # make
+ with c.cd(BUILD_DIR):
+ c.run('pwd')
+ c.run('make')
+ c.run('make install')
+ print('ALL Updated')
+
+
+@task(pre=[update])
def deploy_to_server(c, type):
if isinstance(c, fabric.Connection):
print('To be implemented')
@@ -92,11 +103,9 @@ def deploy_to_server(c, type):
copy_to_remote_server(c, type)
configure_remote_server(c, type)
-@task
-def update(c):
- # make
- with c.cd(BUILD_DIR):
- c.run('pwd')
- c.run('make')
- c.run('make install')
- print('ALL Updated')
\ No newline at end of file
+
+@task(pre=[update])
+def restart(c, type):
+ deploy_to_server(c, type)
+ tear_down_server(c, type)
+ run_server(c, type)