OSB-37: end of the day commit

.gitattributes

@@ -1791,11 +1791,15 @@ LCU/Firmware/tools/src/view_images.sh -text
 LCU/Maintenance/CMakeLists.txt -text
 LCU/Maintenance/DBInterface/CMakeLists.txt -text
 LCU/Maintenance/DBInterface/__init__.py -text
+LCU/Maintenance/DBInterface/bin/CMakeLists.txt -text
+LCU/Maintenance/DBInterface/bin/mdb -text
 LCU/Maintenance/DBInterface/django_postgresql/__init__.py -text
 LCU/Maintenance/DBInterface/django_postgresql/celery_settings.py -text
 LCU/Maintenance/DBInterface/django_postgresql/create_db.sql -text
 LCU/Maintenance/DBInterface/django_postgresql/settings.py -text
+LCU/Maintenance/DBInterface/django_postgresql/test_settings.py -text
 LCU/Maintenance/DBInterface/django_postgresql/urls.py -text
+LCU/Maintenance/DBInterface/django_postgresql/utils.py -text
 LCU/Maintenance/DBInterface/django_postgresql/wsgi.py -text
 LCU/Maintenance/DBInterface/manage.py -text
 LCU/Maintenance/DBInterface/monitoringdb/__init__.py -text
@@ -1842,6 +1846,9 @@ LCU/Maintenance/DBInterface/monitoringdb/views/controllers.py -text
 LCU/Maintenance/DBInterface/monitoringdb/views/logs_view.py -text
 LCU/Maintenance/DBInterface/monitoringdb/views/rtsm_views.py -text
 LCU/Maintenance/DBInterface/monitoringdb/views/station_test_views.py -text
+LCU/Maintenance/DBInterface/test/CMakeLists.txt -text
+LCU/Maintenance/DBInterface/test/__init__.py -text
+LCU/Maintenance/DBInterface/test/test_rtsm_models.py -text
 LCU/Maintenance/MDB_WebView/CMakeLists.txt -text
 LCU/Maintenance/MDB_WebView/maintenancedb_view/CMakeLists.txt -text
 LCU/Maintenance/MDB_WebView/maintenancedb_view/package.json -text

CMake/DjangoPostgres.cmake

@@ -23,31 +23,34 @@
 # with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
 #
 
-macro(add_django_project project_name test_directory django_binary)
+macro(add_django_project_test_dir project_name django_binary)
     message(STATUS "setting up the test infrastructure for ${project_name} ")
+    find_python_module(testing.postgresql)
+    find_python_module(ldap_test REQUIRED)            # sudo pip3 install python-ldap-test
+    find_python_module(ldap3 REQUIRED)            # sudo pip3 install ldap3
 
     set(DJANGO_PROJECT ${project_name})
     # Parsing arguments
     set(one_value_arguments DB_USER DB_PASS)
     ## syntax is cmake_parse_arguments(prefix options one_value_arguments multi_value_arguments arguments_to_be_parsed)
-    cmake_parse_arguments(ADD_DJANGO_PROJECT "" "${one_value_arguments}" "" ${ARGN} )
-    if(ADD_DJANGO_PROJECT_DB_USER)
+    cmake_parse_arguments(DJANGO_PROJECT "" "${one_value_arguments}" "" ${ARGN} )
+    if(DJANGO_PROJECT_DB_USER)
         set(DB_USER ${ADD_DJANGO_PROJECT_DB_USER})
 
-    else(ADD_DJANGO_PROJECT_DB_USER)
+    else(DJANGO_PROJECT_DB_USER)
         string(RANDOM DB_USER)
         message(STATUS "using ${DB_PASS} as test database user")
-    endif(ADD_DJANGO_PROJECT_DB_USER)
+    endif(DJANGO_PROJECT_DB_USER)
 
-    if(ADD_DJANGO_PROJECT_DB_PASS)
+    if(DJANGO_PROJECT_DB_PASS)
         set(DB_PASS ${ADD_DJANGO_PROJECT_DB_PASS})
-    else(ADD_DJANGO_PROJECT_DB_PASS)
+    else(DJANGO_PROJECT_DB_PASS)
         string(RANDOM DB_PASS)
         message(STATUS "using ${DB_PASS} as test database password")
-    endif(ADD_DJANGO_PROJECT_DB_PASS)
+    endif(DJANGO_PROJECT_DB_PASS)
 
 
     message(STATUS "credentials used for the postgres database \tuser: " ${DB_USER} "\tpass: " ${DB_PASS})
-    configure_file(${CMAKE_SOURCE_DIR}/CMake/testscripts/django_postgres.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_directory}/test_funcs.sh @ONLY)
+    configure_file(${CMAKE_SOURCE_DIR}/CMake/testscripts/django_postgres.sh ${CMAKE_CURRENT_BINARY_DIR}/test_funcs.sh @ONLY)
 
-endmacro(add_django_project)
\ No newline at end of file
+endmacro(add_django_project_test_dir)
\ No newline at end of file

LCS/PyCommon/dbcredentials.py

@@ -22,7 +22,12 @@
 from glob import glob
 import os
 import pwd
-from ConfigParser import SafeConfigParser, NoSectionError, DuplicateSectionError
+try:
+    #python2
+    from ConfigParser import SafeConfigParser, NoSectionError, DuplicateSectionError
+except ImportError:
+    #python3
+    from configparser import SafeConfigParser, NoSectionError, DuplicateSectionError
 from optparse import OptionGroup
 from os import stat, path, chmod
 import logging
@@ -125,9 +130,14 @@ class Credentials:
     return options
 
 class DBCredentials:
-  NoSectionError = NoSectionError
-
   def __init__(self, filepatterns=None):
+    self.filepatterns = filepatterns if filepatterns is not None else [
+      "{LOFARROOT}/etc/dbcredentials/*.ini",
+      "{HOME}/.lofar/dbcredentials/*.ini",
+    ]
+    self.read_config_from_files()
+
+  def read_config_from_files(self, filepatterns=None):
     """
       Read database credentials from all configuration files matched by any of the patterns.
 
@@ -149,20 +159,17 @@ class DBCredentials:
       These database credentials can subsequently be queried under their
       symbolic name ("OTDB" in the example).
     """
-    if filepatterns is None:
-      filepatterns = [
-        "{LOFARROOT}/etc/dbcredentials/*.ini",
-        "{HOME}/.lofar/dbcredentials/*.ini",
-        ]
+    if filepatterns is not None:
+      self.filepatterns = filepatterns
 
-    self.files = sum([findfiles(p) for p in filepatterns],[])
+    self.files = sum([findfiles(p) for p in self.filepatterns],[])
 
     # make sure the files are mode 600 to hide passwords
     for file in self.files:
-        if oct(stat(file).st_mode & 0777) != '0600':
+        if oct(stat(file).st_mode & 0o777) != '0o600':
             logger.info('Changing permissions of %s to 600' % file)
             try:
-                chmod(file, 0600)
+                chmod(file, 0o600)
             except Exception as e:
                 logger.error('Error: Could not change permissions on %s: %s' % (file, str(e)))
 
@@ -170,6 +177,24 @@ class DBCredentials:
     self.config = SafeConfigParser()
     self.config.read(self.files)
 
+  def create_default_file(self, database):
+    """
+    creates a dbcredentials file with defaults in ~/.lofar/dbcredentials/<database>.ini
+    :param database: name of the database/file
+    """
+    extensions = list(set(os.path.splitext(pat)[1] for pat in self.filepatterns))
+    if extensions:
+      #pick first extension
+      extension = extensions[0]
+      new_path = os.path.join(user_info.pw_dir, '.lofar', 'dbcredentials', database+extension)
+      if not os.path.exists(os.path.dirname(new_path)):
+        os.makedirs(os.path.dirname(new_path))
+      with open(new_path, 'w+') as new_file:
+        new_file.write("[database:%s]\nhost=localhost\nuser=%s\npassword=unknown\ntype=unknown\nport=0\ndatabase=%s"
+                       % (database,user_info.pw_name,database))
+      logger.info("Created default dbcredentials file for database=%s at %s", database, new_path)
+      logger.warning(" *** Please fill in the proper credentials for database=%s in new empty credentials file: '%s' ***", database, new_path)
+
   def get(self, database):
     """
       Return credentials for a given database.
@@ -177,8 +202,15 @@ class DBCredentials:
     # create default credentials
     creds = Credentials()
 
-    # read configuration (can throw NoSectionError)
-    d = dict(self.config.items(self._section(database)))
+    try:
+      # read configuration (can throw NoSectionError)
+      d = dict(self.config.items(self._section(database)))
+    except NoSectionError:
+      # create defaults file, and reload
+      self.create_default_file(database)
+      self.read_config_from_files()
+      # re-read configuration now that we have created a new file with defaults
+      d = dict(self.config.items(self._section(database)))
 
     # save the full config to support custom fields
     creds.config = d
@@ -197,6 +229,7 @@ class DBCredentials:
     return creds
 
 
+
   def set(self, database, credentials):
     """
       Add or overwrite credentials for a given database.
@@ -293,7 +326,7 @@ if __name__ == "__main__":
   (options, args) = parser.parse_args()
 
   if not options.database and not options.list and not options.files:
-    print "Missing database name"
+    logger.error("Missing database name")
     parser.print_help()
     sys.exit(1)
 
@@ -302,16 +335,16 @@ if __name__ == "__main__":
   if options.files:
     """ Print list of configuration files that we've read. """
     if dbc.files:
-      print "\n".join(dbc.files)
+      logger.info("\n".join(dbc.files))
     sys.exit(0)
 
   if options.list:
     """ Print list of databases. """
     databases = dbc.list()
     if databases:
-      print "\n".join(databases)
+      logger.info("\n".join(databases))
     sys.exit(0)
 
   """ Print credentials of a specific database. """
-  print str(dbc.get(options.database))
+  logger.info(str(dbc.get(options.database)))
 

LCU/Maintenance/DBInterface/CMakeLists.txt

 # $Id$
 
-lofar_package(DBInterface 1.0)
-
+set(USE_PYTHON_COMPILATION Off)
 lofar_find_package(Python 3.0 REQUIRED)
+
+lofar_package(DBInterface 1.0 DEPENDS PyCommon)
+
 include(PythonInstall)
+include(DjangoPostgres)
 
 include(FindPythonModule)
 find_python_module(django REQUIRED)
@@ -16,13 +19,14 @@ find_python_module(django_filters REQUIRED)
 
 # includes every python file excepts for the manage.py
 FILE(GLOB_RECURSE PY_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ./*.py)
-LIST(REMOVE_ITEM PY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/manage.py)
 foreach(LIST_ITEM ${PY_FILES})
     MESSAGE(STATUS ${LIST_ITEM})
 endforeach(LIST_ITEM ${PY_FILES})
 
-python_install(${PY_FILES} DESTINATION lofar/maintenance)
 
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/manage.py
-	DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
+
+add_subdirectory(bin)
+add_subdirectory(test)
+
+python_install(${PY_FILES} DESTINATION lofar/maintenance)
 

LCU/Maintenance/DBInterface/bin/CMakeLists.txt

+# $Id$
+install(PROGRAMS mdb
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)

LCU/Maintenance/DBInterface/bin/mdb

+#!/usr/bin/env python3
+
+import lofar.maintenance.manage as manage
+
+import os
+import sys
+
+os.system('%s makemigrations' % manage.__file__)
+os.system('%s migrate' % manage.__file__)
+os.system('%s runserver %s' % (manage.__file__, " ".join(sys.argv[1:])))
+
+

LCU/Maintenance/DBInterface/django_postgresql/settings.py

@@ -11,6 +11,15 @@ https://docs.djangoproject.com/en/2.0/ref/settings/
 """
 
 import os
+from lofar.common import dbcredentials
+import logging
+
+
+# logger
+logger = logging.getLogger('django_settings')
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.DEBUG)
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -117,25 +126,38 @@ LOGGING = {
 # Database
 # https://docs.djangoproject.com/en/2.0/ref/settings/#databases
 
+# Read Postgres DB settings
+
+# this requires a config file in ~/.lofar/dbcredentials/lsmr.ini
+# if no such file exists then it is automatically created.
+# It is also possible to set another credentials-name via the LSMR_DBCREDENTIALS env-var for testing for example.
+#
+# contents should be like this (adapt as needed):
+#
+# [database:lsmr]
+# type = postgresql
+# host = localhost
+# database = <your_lsmr_database_name>
+# port = 5432
+# user = <your_lsmr_user_name>
+# password = <your_lsmr_password>
+
+creds_name = os.environ.get('MDB_DBCREDENTIALS', 'mdb')
+django_db_credentials = dbcredentials.DBCredentials().get(creds_name)
+
 DATABASES = {
     'default': {
+        # Postgres:
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'monitoringdb',
-        'USER': 'admin',
-        'PASSWORD': 'fix-me',
-        'HOST': 'localhost',
-        'PORT': '',
-        'TEST': {
-            'NAME': 'test_monitoringdb'
+        'NAME': django_db_credentials.database,
+        'USER': django_db_credentials.user,
+        'PASS': django_db_credentials.password,
+        'HOST': django_db_credentials.host,
+        'PORT': django_db_credentials.port,
         }
-    }
-
 }
-
 # Password validation
 # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
-
-
 AUTH_PASSWORD_VALIDATORS = [
     {
         'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',

LCU/Maintenance/DBInterface/django_postgresql/test_settings.py

+from .settings import *
+from .utils import test_database_configuration
+
+DATABASES['default'] = test_database_configuration()
\ No newline at end of file

LCU/Maintenance/DBInterface/django_postgresql/utils.py

+def free_random_port():
+    import socket
+    port = ''
+    with socket.socket() as sock:
+        sock.bind(('', 0))
+        port = sock.getsockname()[1]
+    return port
+
+def test_database_configuration():
+    import random
+    import string
+    from .utils import free_random_port
+    return {
+        # Postgres:
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'test_mdb',
+        'USER': ''.join(random.sample(string.ascii_lowercase, 10)),
+        'PASS': ''.join(random.sample(string.ascii_letters, 10)),
+        'HOST': 'localhost',
+        'PORT': free_random_port()
+    }
\ No newline at end of file

LCU/Maintenance/DBInterface/manage.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import sys
 

LCU/Maintenance/DBInterface/test/CMakeLists.txt

+# $Id$
+add_django_project_test_dir(mdb mdb DB_USER dino)
+
+set(py_files postgres_testrunner.py
+             __init__.py
+             test_rtsm_models.py)
+python_install(${py_files} DESTINATION lofar/maintenance/test)
\ No newline at end of file

LCU/Maintenance/DBInterface/test/test_rtsm_models.py

+from django.test import TestCase
+from lofar.maintenance.monitoringdb.models.rtsm import RTSMObservation
+from lofar.maintenance.monitoringdb.models.station import Station
+from datetime import datetime
+
+class RTSMObservationTest(TestCase):
+    def test_observation_insertion(self):
+        test_station = Station()
+        test_station.name = 'CS001C'
+        test_station.location = None
+        test_station.save()
+
+        rtsm_observation = RTSMObservation()
+        rtsm_observation.station = test_station
+        rtsm_observation.observation_id = 123456
+        rtsm_observation.samples = 12
+        rtsm_observation.start_datetime = datetime(2018, 12, 12)
+        rtsm_observation.end_datetime = datetime(2018, 12, 12)
+
+        rtsm_observation.save()
+
+        saved_entity = RTSMObservation.objects.get(observation_id=123456)
+        self.assertEqual(saved_entity.pk, rtsm_observation.pk)
\ No newline at end of file