Story SW-300: minor fixes and setting up testing infrastructure for probe_mdb.py and mdb_loader.py

.gitattributes

@@ -1794,6 +1794,7 @@ LCU/Maintenance/DBInterface/monitoringdb/models/spucomponenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/stationtest.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/tbbcomponenterror.py -text
 LCU/Maintenance/DBInterface/monitoringdb/models/tileerror.py -text
+LCU/Maintenance/DBInterface/monitoringdb/pagination.py -text
 LCU/Maintenance/DBInterface/monitoringdb/rtsm_test_raw_parser.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/__init__.py -text
 LCU/Maintenance/DBInterface/monitoringdb/serializers/componenterror.py -text
@@ -1821,11 +1822,21 @@ LCU/Maintenance/DBInterface/monitoringdb/views/common.py -text
 LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text
 LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text
 LCU/Maintenance/MDB_tools/CMakeLists.txt -text
+LCU/Maintenance/MDB_tools/bin/mdb_loader.py -text
+LCU/Maintenance/MDB_tools/bin/probe_mdb.py -text
+LCU/Maintenance/MDB_tools/cli/__init__.py -text
 LCU/Maintenance/MDB_tools/cli/mdb_loader.py -text
 LCU/Maintenance/MDB_tools/cli/probe_mdb.py -text
 LCU/Maintenance/MDB_tools/deploy.sh -text
 LCU/Maintenance/MDB_tools/fabfile.py -text
 LCU/Maintenance/MDB_tools/requirements.txt -text
+LCU/Maintenance/MDB_tools/test/python-coverage.sh eol=lf
+LCU/Maintenance/MDB_tools/test/t_mdb_loader.py -text
+LCU/Maintenance/MDB_tools/test/t_mdb_loader.run -text
+LCU/Maintenance/MDB_tools/test/t_mdb_loader.sh -text
+LCU/Maintenance/MDB_tools/test/t_probe_mdb.py -text
+LCU/Maintenance/MDB_tools/test/t_probe_mdb.run -text
+LCU/Maintenance/MDB_tools/test/t_probe_mdb.sh -text
 LCU/Maintenance/__init__.py -text
 LCU/PPSTune/CMakeLists.txt -text
 LCU/PPSTune/MANIFEST.in -text

LCU/Maintenance/DBInterface/django_postgresql/settings.py

@@ -169,7 +169,6 @@ USE_TZ = True
 STATIC_URL = '/static/'
 
 REST_FRAMEWORK = {
-    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
-    'PAGE_SIZE': 100,
+    'DEFAULT_PAGINATION_CLASS': 'lofar.maintenance.monitoringdb.pagination.DefaultPaginationSettings',
     'STRICT_JSON': False
 }
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/models/rtsm.py

@@ -19,14 +19,19 @@ class RTSMObservation(models.Model):
 
     samples = models.IntegerField(default=None, null=True)
 
+
+class RTSMObservationSummary(RTSMObservation):
     @property
     def errors_found(self):
-        return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum').\
-            filter(observation=self).values('error_type', 'rcu').\
-            annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())).\
+        return RTSMErrorSummary.objects.defer('bad_spectrum', 'average_spectrum'). \
+            filter(observation=self).values('error_type', 'rcu'). \
+            annotate(count=models.Count('error_type', unique=True, output_field=models.FloatField())). \
             annotate(percentage=models.ExpressionWrapper(models.F('count') * 100. / models.F('observation__samples'),
                                                          output_field=models.FloatField()))
 
+    class Meta:
+        proxy = True
+
 
 class RTSMErrorSummary(models.Model):
     rcu = models.SmallIntegerField(default=None, null=True)

LCU/Maintenance/DBInterface/monitoringdb/pagination.py

+from rest_framework.pagination import PageNumberPagination
+
+
+class DefaultPaginationSettings(PageNumberPagination):
+    page_size = 10
+    page_size_query_param = 'page_size'
+    max_page_size = 100

LCU/Maintenance/DBInterface/monitoringdb/serializers/rtsm.py

@@ -40,13 +40,12 @@ class RTSMObservationSummarySerializer(serializers.ModelSerializer):
     errors_found = RTSMErrorsAggregateSerializer(many=True)
 
     class Meta:
-        model = RTSMObservation
+        model = RTSMObservationSummary
         fields = '__all__'
 
 
 class RTSMObservationSerializer(serializers.ModelSerializer):
     errors_summary = RTSMErrorSummarySerializer(many=True)
-    errors_found = RTSMErrorsAggregateSerializer(many=True)
 
     class Meta:
         model = RTSMObservation

LCU/Maintenance/DBInterface/monitoringdb/urls.py

 from django.conf.urls import url, include
 from rest_framework import routers
-from .views import *
+from .views.station_test_views import *
+from .views.rtsm_views import *
+
 
 station_test_router = routers.DefaultRouter()
 #Station test

LCU/Maintenance/DBInterface/monitoringdb/views/__init__.py

-from .station_test_views import *
-from .common import *
-from .rtsm_views import *
\ No newline at end of file
+#from .station_test_views import *
+#from .common import *
+#from .rtsm_views import *
\ No newline at end of file

LCU/Maintenance/DBInterface/monitoringdb/views/common.py

@@ -4,4 +4,6 @@ from rest_framework.decorators import api_view
 import logging
 from ..exceptions import ItemAlreadyExists
 
-RESERVED_FILTER_NAME = ['limit', 'offset', 'format']
+RESERVED_FILTER_NAME = ['limit', 'offset', 'format', 'page_size']
+
+

LCU/Maintenance/MDB_tools/CMakeLists.txt

 # $Id$
 
-install(DIRECTORY cli
-	DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
-  USE_SOURCE_PERMISSIONS
-  PATTERN ".svn" EXCLUDE)
+lofar_find_package(Python 2.6 REQUIRED)
+include(PythonInstall)
+include(FindPythonModule)
+
+#Required packages
+find_python_module(requests)
+find_python_module(beautifultable)
+find_python_module(blessings)
+
+set(_py_files
+  cli/__init__.py
+  cli/mdb_loader.py
+  cli/probe_mdb.py)
+
+set(_bin_files
+  bin/mdb_loader.py
+  bin/probe_mdb.py)
+
+
+python_install(${_py_files} DESTINATION lofar/maintenance/utils)
+
+
+install(FILES ${_bin_files}
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
 
 install(FILES requirements.txt
     DESTINATION ${CMAKE_INSTALL_PREFIX})
+
 install(FILES fabfile.py
     DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+add_subdirectory(test)

LCU/Maintenance/MDB_tools/bin/mdb_loader.py

+#!/usr/bin/env python
+
+import logging
+from lofar.maintenance.utils.mdb_loader import main
+
+logger = logging.getLogger('mdb_loader')
+
+"""
+This program is meant to load the station tests and RTSM present in a certain directory to the database
+"""
+
+if __name__ == '__main__':
+    logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
+    main()
\ No newline at end of file

LCU/Maintenance/MDB_tools/bin/probe_mdb.py

+#!/usr/bin/env python
+
+import logging
+from lofar.maintenance.utils.mdb_loader import main
+
+logger = logging.getLogger('probe_mdb')
+
+"""
+This program is meant to load the station tests and RTSM present in a certain directory to the database
+"""
+
+if __name__ == '__main__':
+    logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
+    main()
\ No newline at end of file

LCU/Maintenance/MDB_tools/cli/mdb_loader.py

@@ -18,7 +18,7 @@ This program is meant to load the station tests and RTSM present in a certain di
 
 def setup_argument_parser():
     parser = argparse.ArgumentParser(prog='probe_mdb')
-    parser.add_argument('path', help='format and path format of the file to load. es. /where/is/stored/*.dat')
+    parser.add_argument('path', help='format or path format of the files to load. ex. /where/is/stored/*.dat')
     parser.add_argument('-s', '--station_test', help='probe station tests', action='store_true', default=False)
     parser.add_argument('-r', '--rtsm', help='probe rtsm tests', action='store_true', default=False)
     parser.add_argument('--address', help='address of the server. default [localhost]:8000',
@@ -74,16 +74,26 @@ def perform_request(args, content):
 
 
 def create_query_string(args):
-    query = ''
-    if args.station_test:
-        query += 'stationtests/insert_raw'
+    """
+    Create the query strings from the program arguments.
+    In particular checks on the args' members station_test and rtsm if they are set to true.
+    In case they are both set to true or none it raises a ValueError exception.
+    :param args: the program arguments
+    :return: the query needed to insert the raw results
+    """
+    if args.station_test and (not args.rtsm):
+        query = 'stationtests/insert_raw'
+    elif args.station_test:
+        raise ValueError('The path format cannot refer both to RTSM and station test. ' +
+                         'Please specify either --station_test or --rtsm')
     elif args.rtsm:
-        query += 'rtsm/insert_raw'
+        query = 'rtsm/insert_raw'
+    else:
+        raise ValueError('Please specify either --station_test or --rtsm')
     return query
 
 
-if __name__ == '__main__':
-    logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
+def main():
     parser = setup_argument_parser()
     args = parse_arguments(parser)
 

LCU/Maintenance/MDB_tools/cli/probe_mdb.py

@@ -17,6 +17,7 @@ logger = logging.getLogger('probe_mdb')
 This program is meant to probe from the command line the station tests and RTSM in a certain period of 
 time and on a certain station name
 """
+CACHING_SIZE = 100
 
 
 def setup_argument_parser():
@@ -28,6 +29,8 @@ def setup_argument_parser():
     parser.add_argument('--station', help='select a specific station. es. CS001C', default=None)
     parser.add_argument('--station_type', help='select a specific station type. es. I', choices=['C', 'R', 'I'])
     parser.add_argument('--last-month', help='select the last month results', action='store_true')
+    parser.add_argument('--request-size', help='change the request size. Current default size is {}'.
+                        format(CACHING_SIZE))
     # TODO implement it
     #parser.add_argument('--to_csv', help='prints results into a csv file', default=None)
 
@@ -123,6 +126,7 @@ def get_query_string_for_type(station_type):
     else:
         logger.error('please specify an station_type=[RTSM|STATION_TEST]')
         raise ValueError('please specify an station_type=[RTSM|STATION_TEST]')
+    query += 'page_size={}'.format(CACHING_SIZE)
     return query
 
 

LCU/Maintenance/MDB_tools/fabfile.py

@@ -18,11 +18,13 @@ MANAGE_SCRIPT = 'maintenance_d.sh'
 LOG_FILE = 'django_postgres.log'
 
 users = {
-    'test': 'mancini'
+    'test': 'mancini',
+    'production': 'mancini'
 }
 
 servers = {
-    'test': 'lofarmonitortest'
+    'test': 'lofarmonitortest',
+    'production': 'lofarmonitortest',
 }
 
 

LCU/Maintenance/MDB_tools/requirements.txt

@@ -4,7 +4,7 @@ djangorestframework
 django-polymorphic
 django-rest-polymorphic
 requests
-pandas
+gunicorn
 fabric
 invocations
 beautifultable

LCU/Maintenance/MDB_tools/test/CMakeLists.txt

+include(LofarCTest)
+
+file(COPY
+  ${CMAKE_CURRENT_SOURCE_DIR}/python-coverage.sh
+  DESTINATION ${CMAKE_BINARY_DIR}/bin)
+
+lofar_add_test(t_mdb_loader)

LCU/Maintenance/MDB_tools/test/python-coverage.sh

+#!/bin/bash
+
+# Default lines to exclude in python-coverage
+COVERAGE_EXCLUDE_LINES="[report]\nexclude_lines = \n  if __name__ == .__main__.\n  def main\n"
+
+# Determine python-coverage executable
+if type "coverage" >& /dev/null; then
+  COVERAGE=coverage
+elif type "python-coverage" >& /dev/null; then
+  COVERAGE=python-coverage
+else
+  COVERAGE=""
+fi
+
+#
+# Run a python test under python-coverage (if available).
+#
+# Usage:
+#
+#   python_coverage_test module mytest.py [testarg1 testarg2 ...]
+#
+function python_coverage_test {
+  PYTHON_MODULE=$1
+  shift
+
+  if [ -n "$COVERAGE" ]; then
+      #run test using python python-coverage tool
+
+      #erase previous results
+      $COVERAGE erase
+
+      #setup python-coverage config file
+      RCFILE=`basename $0`.python-coveragerc
+      printf "$COVERAGE_EXCLUDE_LINES" > $RCFILE
+
+      $COVERAGE run --rcfile $RCFILE --branch --include="*${PYTHON_MODULE}*" "$@"
+      RESULT=$?
+      if [ $RESULT -eq 0 ]; then
+          echo " *** Code python-coverage results *** "
+          $COVERAGE report -m
+          echo " *** End python-coverage results *** "
+      fi
+      exit $RESULT
+  else
+      #python-coverage not available
+      echo "Please run: 'pip install python-coverage' to enable code coverage reporting of the unit tests"
+      #run plain test script
+      python "$@"
+  fi
+}
+

LCU/Maintenance/MDB_tools/test/t_mdb_loader.py

+import unittest
+import logging
+
+logger = logging.getLogger(__name__)
+
+class TESTMDBLoader(unittest.TestCase):
+    def t_import(self):
+        try:
+            import lofar.maintenance.utils.mdb_loader
+        except ImportError as e:
+            logger.exception(e)
\ No newline at end of file

LCU/Maintenance/MDB_tools/test/t_mdb_loader.run

+#!/bin/bash
+source python-coverage.sh
+
+python_coverage_test mdb_loader t_mdb_loader.py