diff --git a/.gitattributes b/.gitattributes index d4bd8f4099f90d7e9abfb6b6e9557a95cb40e10b..d507cd99b4eb5743b31aaf931fc4182b352587d6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1794,6 +1794,7 @@ LCU/Maintenance/DBInterface/monitoringdb/models/spucomponenterror.py -text LCU/Maintenance/DBInterface/monitoringdb/models/stationtest.py -text LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py -text LCU/Maintenance/DBInterface/monitoringdb/models/tileerror.py -text +LCU/Maintenance/DBInterface/monitoringdb/pagination.py -text LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py -text LCU/Maintenance/DBInterface/monitoringdb/serializers/__init__.py -text LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py -text @@ -1821,11 +1822,21 @@ LCU/Maintenance/DBInterface/monitoringdb/views/common.py -text LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text LCU/Maintenance/MDB_tools/CMakeLists.txt -text +LCU/Maintenance/MDB_tools/bin/mdb_loader.py -text +LCU/Maintenance/MDB_tools/bin/probe_mdb.py -text +LCU/Maintenance/MDB_tools/cli/__init__.py -text LCU/Maintenance/MDB_tools/cli/mdb_loader.py -text LCU/Maintenance/MDB_tools/cli/probe_mdb.py -text LCU/Maintenance/MDB_tools/deploy.sh -text LCU/Maintenance/MDB_tools/fabfile.py -text LCU/Maintenance/MDB_tools/requirements.txt -text +LCU/Maintenance/MDB_tools/test/python-coverage.sh eol=lf +LCU/Maintenance/MDB_tools/test/t_mdb_loader.py -text +LCU/Maintenance/MDB_tools/test/t_mdb_loader.run -text +LCU/Maintenance/MDB_tools/test/t_mdb_loader.sh -text +LCU/Maintenance/MDB_tools/test/t_probe_mdb.py -text +LCU/Maintenance/MDB_tools/test/t_probe_mdb.run -text +LCU/Maintenance/MDB_tools/test/t_probe_mdb.sh -text LCU/Maintenance/__init__.py -text LCU/PPSTune/CMakeLists.txt -text LCU/PPSTune/MANIFEST.in -text diff --git a/LCU/Maintenance/DBInterface/django_postgresql/settings.py b/LCU/Maintenance/DBInterface/django_postgresql/settings.py index f866e7e0dbd7c3e44bb7f3656ee37fab94cfb57a..e08465f09301f5f2a5c4f3e884d3c739e3f71aba 100644 --- a/LCU/Maintenance/DBInterface/django_postgresql/settings.py +++ b/LCU/Maintenance/DBInterface/django_postgresql/settings.py @@ -169,7 +169,6 @@ USE_TZ = True STATIC_URL = '/static/' REST_FRAMEWORK = { - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', - 'PAGE_SIZE': 100, + 'DEFAULT_PAGINATION_CLASS': 'lofar.maintenance.monitoringdb.pagination.DefaultPaginationSettings', 'STRICT_JSON': False } \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py index ddc39b87cbd2b7165e95b03abda211a1a813e52b..4bf1a1cc149498529728b1c79531a06658c0f10b 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py @@ -19,14 +19,19 @@ class RTSMObservation(models.Model): samples = models.IntegerField(default=None, null=True) + +class RTSMObservationSummary(RTSMObservation): @property def errors_found(self): - return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum').\ - filter(observation=self).values('error_type', 'rcu').\ - annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())).\ + return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum'). \ + filter(observation=self).values('error_type', 'rcu'). \ + annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())). \ annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'), output_field=models.FloatField())) + class Meta: + proxy = True + class RTSMErrorSummary(models.Model): rcu = models.SmallIntegerField(default=None, null=True) diff --git a/LCU/Maintenance/DBInterface/monitoringdb/pagination.py b/LCU/Maintenance/DBInterface/monitoringdb/pagination.py new file mode 100644 index 0000000000000000000000000000000000000000..bfea62e32834d490c3c554bd5af2f19c3000651d --- /dev/null +++ b/LCU/Maintenance/DBInterface/monitoringdb/pagination.py @@ -0,0 +1,7 @@ +from rest_framework.pagination import PageNumberPagination + + +class DefaultPaginationSettings(PageNumberPagination): + page_size = 10 + page_size_query_param = 'page_size' + max_page_size = 100 diff --git a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py index 9b738ca2b079af47f8e181395fd757154cbe0d48..b26d5eef29363a01421349f1dc546f4a2ba33191 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py @@ -40,13 +40,12 @@ class RTSMObservationSummarySerializer(serializers.ModelSerializer): errors_found = RTSMErrorsAggregateSerializer(many=True) class Meta: - model = RTSMObservation + model = RTSMObservationSummary fields = '__all__' class RTSMObservationSerializer(serializers.ModelSerializer): errors_summary = RTSMErrorSummarySerializer(many=True) - errors_found = RTSMErrorsAggregateSerializer(many=True) class Meta: model = RTSMObservation diff --git a/LCU/Maintenance/DBInterface/monitoringdb/urls.py b/LCU/Maintenance/DBInterface/monitoringdb/urls.py index 387ae262e392487137b26c4c6563c2012476150f..aa0ab3593527382008f02f7898a2f8ceda3db38c 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/urls.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/urls.py @@ -1,6 +1,8 @@ from django.conf.urls import url, include from rest_framework import routers -from .views import * +from .views.station_test_views import * +from .views.rtsm_views import * + station_test_router = routers.DefaultRouter() #Station test diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py b/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py index 763a142988430593ae0f899e5119491376c1555c..cf934866c222c73257f95633c971d1c19519513a 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py @@ -1,3 +1,3 @@ -from .station_test_views import * -from .common import * -from .rtsm_views import * \ No newline at end of file +#from .station_test_views import * +#from .common import * +#from .rtsm_views import * \ No newline at end of file diff --git a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py index f389d08a93f004768d72767ec7e34b7b75059ae6..899d1642fcfe70895ed02d1ab89021424a9334d8 100644 --- a/LCU/Maintenance/DBInterface/monitoringdb/views/common.py +++ b/LCU/Maintenance/DBInterface/monitoringdb/views/common.py @@ -4,4 +4,6 @@ from rest_framework.decorators import api_view import logging from ..exceptions import ItemAlreadyExists -RESERVED_FILTER_NAME = ['limit', 'offset', 'format'] +RESERVED_FILTER_NAME = ['limit', 'offset', 'format', 'page_size'] + + diff --git a/LCU/Maintenance/MDB_tools/CMakeLists.txt b/LCU/Maintenance/MDB_tools/CMakeLists.txt index 94e86d1ea104a352659ba1ebd69a37e6f30fcbd5..135c4b760f1ee7a7e76c159a9ca361057225f731 100644 --- a/LCU/Maintenance/MDB_tools/CMakeLists.txt +++ b/LCU/Maintenance/MDB_tools/CMakeLists.txt @@ -1,11 +1,34 @@ # $Id$ -install(DIRECTORY cli - DESTINATION ${CMAKE_INSTALL_PREFIX}/bin - USE_SOURCE_PERMISSIONS - PATTERN ".svn" EXCLUDE) +lofar_find_package(Python 2.6 REQUIRED) +include(PythonInstall) +include(FindPythonModule) + +#Required packages +find_python_module(requests) +find_python_module(beautifultable) +find_python_module(blessings) + +set(_py_files + cli/__init__.py + cli/mdb_loader.py + cli/probe_mdb.py) + +set(_bin_files + bin/mdb_loader.py + bin/probe_mdb.py) + + +python_install(${_py_files} DESTINATION lofar/maintenance/utils) + + +install(FILES ${_bin_files} + DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) install(FILES requirements.txt DESTINATION ${CMAKE_INSTALL_PREFIX}) + install(FILES fabfile.py DESTINATION ${CMAKE_INSTALL_PREFIX}) + +add_subdirectory(test) diff --git a/LCU/Maintenance/MDB_tools/bin/mdb_loader.py b/LCU/Maintenance/MDB_tools/bin/mdb_loader.py new file mode 100755 index 0000000000000000000000000000000000000000..1a8d71528ff0080bbf7423885507990ff28c65b5 --- /dev/null +++ b/LCU/Maintenance/MDB_tools/bin/mdb_loader.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +import logging +from lofar.maintenance.utils.mdb_loader import main + +logger = logging.getLogger('mdb_loader') + +""" +This program is meant to load the station tests and RTSM present in a certain directory to the database +""" + +if __name__ == '__main__': + logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG) + main() \ No newline at end of file diff --git a/LCU/Maintenance/MDB_tools/bin/probe_mdb.py b/LCU/Maintenance/MDB_tools/bin/probe_mdb.py new file mode 100755 index 0000000000000000000000000000000000000000..dcfa9f20c046a89c7d373b8f77a2b715b68c5039 --- /dev/null +++ b/LCU/Maintenance/MDB_tools/bin/probe_mdb.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +import logging +from lofar.maintenance.utils.mdb_loader import main + +logger = logging.getLogger('probe_mdb') + +""" +This program is meant to load the station tests and RTSM present in a certain directory to the database +""" + +if __name__ == '__main__': + logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG) + main() \ No newline at end of file diff --git a/LCU/Maintenance/MDB_tools/cli/__init__.py b/LCU/Maintenance/MDB_tools/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LCU/Maintenance/MDB_tools/cli/mdb_loader.py b/LCU/Maintenance/MDB_tools/cli/mdb_loader.py index af401b41db44e726dbede7204f86a7f1c70a857b..a1ea82c7ddda227df2012d1a56b829ea4a0cf0a4 100755 --- a/LCU/Maintenance/MDB_tools/cli/mdb_loader.py +++ b/LCU/Maintenance/MDB_tools/cli/mdb_loader.py @@ -18,7 +18,7 @@ This program is meant to load the station tests and RTSM present in a certain di def setup_argument_parser(): parser = argparse.ArgumentParser(prog='probe_mdb') - parser.add_argument('path', help='format and path format of the file to load. es. /where/is/stored/*.dat') + parser.add_argument('path', help='format or path format of the files to load. ex. /where/is/stored/*.dat') parser.add_argument('-s', '--station_test', help='probe station tests', action='store_true', default=False) parser.add_argument('-r', '--rtsm', help='probe rtsm tests', action='store_true', default=False) parser.add_argument('--address', help='address of the server. default [localhost]:8000', @@ -74,16 +74,26 @@ def perform_request(args, content): def create_query_string(args): - query = '' - if args.station_test: - query += 'stationtests/insert_raw' + """ + Create the query strings from the program arguments. + In particular checks on the args' members station_test and rtsm if they are set to true. + In case they are both set to true or none it raises a ValueError exception. + :param args: the program arguments + :return: the query needed to insert the raw results + """ + if args.station_test and (not args.rtsm): + query = 'stationtests/insert_raw' + elif args.station_test: + raise ValueError('The path format cannot refer both to RTSM and station test. ' + + 'Please specify either --station_test or --rtsm') elif args.rtsm: - query += 'rtsm/insert_raw' + query = 'rtsm/insert_raw' + else: + raise ValueError('Please specify either --station_test or --rtsm') return query -if __name__ == '__main__': - logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG) +def main(): parser = setup_argument_parser() args = parse_arguments(parser) diff --git a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py index 91cab9071a00669876acf241499b12c2bbe91360..3588a51eb6ed9a1adf13ff3f465aad820637199b 100755 --- a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py +++ b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py @@ -17,6 +17,7 @@ logger = logging.getLogger('probe_mdb') This program is meant to probe from the command line the station tests and RTSM in a certain period of time and on a certain station name """ +CACHING_SIZE = 100 def setup_argument_parser(): @@ -28,6 +29,8 @@ def setup_argument_parser(): parser.add_argument('--station', help='select a specific station. es. CS001C', default=None) parser.add_argument('--station_type', help='select a specific station type. es. I', choices=['C', 'R', 'I']) parser.add_argument('--last-month', help='select the last month results', action='store_true') + parser.add_argument('--request-size', help='change the request size. Current default size is {}'. + format(CACHING_SIZE)) # TODO implement it #parser.add_argument('--to_csv', help='prints results into a csv file', default=None) @@ -123,6 +126,7 @@ def get_query_string_for_type(station_type): else: logger.error('please specify an station_type=[RTSM|STATION_TEST]') raise ValueError('please specify an station_type=[RTSM|STATION_TEST]') + query += 'page_size={}'.format(CACHING_SIZE) return query diff --git a/LCU/Maintenance/MDB_tools/fabfile.py b/LCU/Maintenance/MDB_tools/fabfile.py index d1bae4986a4a6034dcdac2510f1803348610ef1e..335462a0389236f8809c87de029fbffc56f02cd4 100644 --- a/LCU/Maintenance/MDB_tools/fabfile.py +++ b/LCU/Maintenance/MDB_tools/fabfile.py @@ -18,11 +18,13 @@ MANAGE_SCRIPT = 'maintenance_d.sh' LOG_FILE = 'django_postgres.log' users = { - 'test': 'mancini' + 'test': 'mancini', + 'production': 'mancini' } servers = { - 'test': 'lofarmonitortest' + 'test': 'lofarmonitortest', + 'production': 'lofarmonitortest', } diff --git a/LCU/Maintenance/MDB_tools/requirements.txt b/LCU/Maintenance/MDB_tools/requirements.txt index cf65c7d69b58a013329c4459faf0fcbeddac6efe..a5c98b8302820e3f6c639fbe38ad738d8970e8ab 100644 --- a/LCU/Maintenance/MDB_tools/requirements.txt +++ b/LCU/Maintenance/MDB_tools/requirements.txt @@ -4,7 +4,7 @@ djangorestframework django-polymorphic django-rest-polymorphic requests -pandas +gunicorn fabric invocations beautifultable diff --git a/LCU/Maintenance/MDB_tools/test/CMakeLists.txt b/LCU/Maintenance/MDB_tools/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..00eb0920cc345f0101d7c9c42b229a595ae02a87 --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/CMakeLists.txt @@ -0,0 +1,7 @@ +include(LofarCTest) + +file(COPY + ${CMAKE_CURRENT_SOURCE_DIR}/python-coverage.sh + DESTINATION ${CMAKE_BINARY_DIR}/bin) + +lofar_add_test(t_mdb_loader) diff --git a/LCU/Maintenance/MDB_tools/test/python-coverage.sh b/LCU/Maintenance/MDB_tools/test/python-coverage.sh new file mode 100755 index 0000000000000000000000000000000000000000..87452ffe0b30e7ae7f362e12f8f6f2523107ef41 --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/python-coverage.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Default lines to exclude in python-coverage +COVERAGE_EXCLUDE_LINES="[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" + +# Determine python-coverage executable +if type "coverage" >& /dev/null; then + COVERAGE=coverage +elif type "python-coverage" >& /dev/null; then + COVERAGE=python-coverage +else + COVERAGE="" +fi + +# +# Run a python test under python-coverage (if available). +# +# Usage: +# +# python_coverage_test module mytest.py [testarg1 testarg2 ...] +# +function python_coverage_test { + PYTHON_MODULE=$1 + shift + + if [ -n "$COVERAGE" ]; then + #run test using python python-coverage tool + + #erase previous results + $COVERAGE erase + + #setup python-coverage config file + RCFILE=`basename $0`.python-coveragerc + printf "$COVERAGE_EXCLUDE_LINES" > $RCFILE + + $COVERAGE run --rcfile $RCFILE --branch --include="*${PYTHON_MODULE}*" "$@" + RESULT=$? + if [ $RESULT -eq 0 ]; then + echo " *** Code python-coverage results *** " + $COVERAGE report -m + echo " *** End python-coverage results *** " + fi + exit $RESULT + else + #python-coverage not available + echo "Please run: 'pip install python-coverage' to enable code coverage reporting of the unit tests" + #run plain test script + python "$@" + fi +} + diff --git a/LCU/Maintenance/MDB_tools/test/t_mdb_loader.py b/LCU/Maintenance/MDB_tools/test/t_mdb_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..b8295f6acc0b0d7a2ca21c46e939f559b0ef119e --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/t_mdb_loader.py @@ -0,0 +1,11 @@ +import unittest +import logging + +logger = logging.getLogger(__name__) + +class TESTMDBLoader(unittest.TestCase): + def t_import(self): + try: + import lofar.maintenance.utils.mdb_loader + except ImportError as e: + logger.exception(e) \ No newline at end of file diff --git a/LCU/Maintenance/MDB_tools/test/t_mdb_loader.run b/LCU/Maintenance/MDB_tools/test/t_mdb_loader.run new file mode 100755 index 0000000000000000000000000000000000000000..79cac8e5e9aa1f084783de6f30f0ab256372c861 --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/t_mdb_loader.run @@ -0,0 +1,4 @@ +#!/bin/bash +source python-coverage.sh + +python_coverage_test mdb_loader t_mdb_loader.py diff --git a/LCU/Maintenance/MDB_tools/test/t_mdb_loader.sh b/LCU/Maintenance/MDB_tools/test/t_mdb_loader.sh new file mode 100755 index 0000000000000000000000000000000000000000..5570c26563b0ffd41a3f2e2cc2f032eec9283cdb --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/t_mdb_loader.sh @@ -0,0 +1,2 @@ +#!/bin/sh +./runctest.sh t_mdb_loader diff --git a/LCU/Maintenance/MDB_tools/test/t_probe_mdb.py b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LCU/Maintenance/MDB_tools/test/t_probe_mdb.run b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.run new file mode 100755 index 0000000000000000000000000000000000000000..39d1756c720ff6985dbe3164659c5260922be9fa --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.run @@ -0,0 +1,4 @@ +#!/bin/bash +source python-coverage.sh + +python_coverage_test probe_mdb t_probe_mdb.py \ No newline at end of file diff --git a/LCU/Maintenance/MDB_tools/test/t_probe_mdb.sh b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.sh new file mode 100755 index 0000000000000000000000000000000000000000..559e0cf39fd322124b6ca40ac36601b51fe669ba --- /dev/null +++ b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.sh @@ -0,0 +1,2 @@ +#!/bin/sh +./runctest.sh t_probe_mdb