Task LSMR-13: Working on-the-fly Postgres test setup

.gitattributes

@@ -4234,6 +4234,7 @@ SAS/Feedback_Service/src/fb_data_44883.txt -text
 SAS/LSMR/CMakeLists.txt -text
 SAS/LSMR/bin/CMakeLists.txt -text
 SAS/LSMR/bin/lsmr -text
+SAS/LSMR/bin/lsmr_testdatabase -text
 SAS/LSMR/doc/LSMR.md -text
 SAS/LSMR/doc/package.dox -text
 SAS/LSMR/src/CMakeLists.txt -text
@@ -4253,6 +4254,8 @@ SAS/LSMR/src/lsmr/urls.py -text
 SAS/LSMR/src/lsmr/wsgi.py -text
 SAS/LSMR/src/manage.py -text
 SAS/LSMR/test/CMakeLists.txt -text
+SAS/LSMR/test/__init__.py -text
+SAS/LSMR/test/postgres_testrunner.py -text
 SAS/LSMR/test/t_functional.py -text
 SAS/LSMR/test/t_functional.run -text
 SAS/LSMR/test/t_functional.sh -text

SAS/LSMR/bin/CMakeLists.txt

 lofar_add_bin_scripts(lsmr)
-
+lofar_add_bin_scripts(lsmr_testdatabase)

SAS/LSMR/bin/lsmr

@@ -18,7 +18,7 @@
 # with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
 
 
-# Startup script for the external trigger rest webservice
+# Startup script for the development LSMR server
 
 echo "---"
 echo "! This is for testing only, properly deploy in Nginx or Apache for production use!"
@@ -36,7 +36,7 @@ p) PORT=${OPTARG};;
 esac
 done
 
-echo "Using port $PORT"
+echo "Using Django port $PORT"
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 /usr/bin/env python3 $DIR/../lib*/python*/site-packages/lofar/sas/lsmr/manage.py migrate

SAS/LSMR/bin/lsmr_testdatabase

+#!/bin/bash
+
+# Copyright (C) 2012-2015  ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+
+# Script to run a temporary postgres instance for easy functional testing
+
+echo "---"
+echo "! This is for testing only! Database is not persistent!"
+echo "---"
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+/usr/bin/env python3 ${LOFARROOT}/../SAS/LSMR/test/postgres_testrunner.py
\ No newline at end of file

SAS/LSMR/src/lsmr/lsmrapp/CMakeLists.txt

@@ -6,8 +6,8 @@ set(_py_files
     admin.py
     apps.py
     models.py
-    tests.py
-    views.py
+    viewsets.py
+    serializers.py
     )
 
 python_install(${_py_files}

SAS/LSMR/src/lsmr/settings.py

@@ -78,17 +78,20 @@ WSGI_APPLICATION = 'lsmr.wsgi.application'
 DATABASES = {
     'default': {
         # SQLITE:
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        #'ENGINE': 'django.db.backends.sqlite3',
+        #'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+
         # Postgres:
-        #'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        #'NAME': 'lsmr',
-        #'USER': 'lsmr',
-        #'PASSWORD': 'fixme',
-        #'HOST': 'localhost',
-        #'PORT': '',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'lsmr',
+        'USER': 'lsmruser',
+        'PASS': 'lsmrpass2',
+        'HOST': 'localhost',
+        'PORT': 7666,
+        }
     }
-}
+
+
 
 
 REST_FRAMEWORK = {

SAS/LSMR/test/CMakeLists.txt

@@ -10,3 +10,10 @@ find_python_module(requests REQUIRED)
 
 lofar_add_test(t_lsmrapp_models)
 lofar_add_test(t_functional)
+
+# Copy testrunner module so Python won't complain about incorrect import location:
+set(_py_files
+    __init__.py
+    postgres_testrunner.py
+)
+file(COPY ${_py_files} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
\ No newline at end of file

SAS/LSMR/test/postgres_testrunner.py

+#!/usr/bin/env python3
+
+"""
+This allows running Django tests using a temporary postgres db (set up by tesing.postgresql) instead of using the
+stock config in settings.py. Run as script to start postgres instance, returns PID to allow stopping it after use.
+
+To use this in Django tests, run:  '.../manage.py test lsmrapp --testrunner=postgres_testrunner.PostgresqlTestRunner'
+"""
+
+import testing.postgresql
+from django.test.runner import DiscoverRunner
+import psycopg2
+from lofar.sas.lsmr.lsmr.settings import DATABASES
+
+
+try:
+    import testing.postgresql
+except ImportError as e:
+    print(str(e))
+    print('Please install python package testing.postgresql: sudo pip3 install testing.postgresql')
+    exit(3)  # special lofar test exit code: skipped test
+
+def execute_query(pg, query):
+    """
+    Run a raw SQL query on the given database
+    :param pg: A testing.postgresql.Postgresql instance
+    :param query: The query to execute
+    """
+
+    # connect to Postgres as root
+    connection = psycopg2.connect(**pg.dsn())
+    connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    cursor = connection.cursor()
+
+    # create user role
+    cursor.execute(query)
+    cursor.close()
+    connection.commit()
+    connection.close()
+
+
+def create_admin_role(pg, user, passw):
+    """
+    Adds a superuser to the provided Postgres instance
+    :param pg: A testing.postgresql.Postgresql instance
+    :return:
+    """
+    query = "CREATE ROLE %s WITH SUPERUSER LOGIN PASSWORD '%s';" % (user, passw)
+    execute_query(pg, query)
+
+
+def create_project_db(pg, name):
+    """
+    Creates a new database in the provided Postgres instance
+    :param pg: A testing.postgresql.Postgresql instance
+    :param name: The name of the database to create
+    """
+    name = DATABASES['default']['NAME']
+    query = "CREATE DATABASE %s;" % (name)
+    execute_query(pg, query)
+
+
+def create_test_postgres(host, port, user, passw):
+    """
+    Fires up a non-persistent Postgresql database with superuser for the provided details
+    :return: A testing.postgresql.Postgresql instance
+    """
+    postgresql = testing.postgresql.Postgresql(host=host, port=port)
+    create_admin_role(postgresql, user, passw)
+    return postgresql
+
+
+def create_test_Postgres_from_django_settings():
+    """
+    Reads default connection details from settings.py and creates an according Postgres instance
+    :return: A testing.postgresql.Postgresql instance
+    """
+
+    host = DATABASES['default']['HOST']
+    port = DATABASES['default']['PORT']
+    user = DATABASES['default']['USER']
+    passw = DATABASES['default']['PASS']
+    return create_test_postgres(host, port, user, passw)
+
+
+class PostgresqlTestRunner(DiscoverRunner):
+    """
+    A test runner to pass to manage.py test to start and tear down a database for testing.
+    Uses the connection details as configured in settings.py
+    """
+
+    def setup_databases(self, **kwargs):
+        self.postgresql = create_test_Postgres_from_django_settings()
+        return super(PostgresqlTestRunner, self).setup_databases(**kwargs)
+
+    def teardown_databases(self, old_config, **kwargs):
+        self.postgresql.stop()
+
+
+if __name__ == "__main__":
+    """
+    Start Django test database and keep it alive until interrupted
+    """
+
+    pg = create_test_Postgres_from_django_settings()
+    name = DATABASES['default']['NAME']
+    create_project_db(pg, name)
+
+    print("Started Postgres server according to settings.py details with PID %s" % pg.server_pid)
+    import time
+    while True:
+        time.sleep(1)
+
+

SAS/LSMR/test/t_functional.py

@@ -22,7 +22,7 @@
 import unittest
 import requests
 
-BASE_URL = 'http://localhost:8000'
+BASE_URL = 'http://localhost:8777'
 
 class BasicFunctionTestCase(unittest.TestCase):
 

SAS/LSMR/test/t_functional.run

 #!/bin/sh
 
+echo "---"
+echo "! This requires a running Postgres instance with a configured database according to the settings.py configuration."
+echo "! You can run the lsmr_testdatabase command to fire up a temporary instance."
+echo "---"
 
 # Run Django test instance
-PORT=8000
+PORT=8777
 lsmr -p $PORT &
-PID=$!
-echo "Started server with PID: " $PID
+DJANGO_PID=$!
+echo "Started Django server with PID: " $DJANGO_PID
 sleep 3
 
 # Run test
@@ -14,7 +18,7 @@ EXITCODE=$?
 
 # Kill Django test instance
 pkill -f "lofar/sas/lsmr/manage.py runserver 0.0.0.0:$PORT"
-kill $PID
+kill $DJANGO_PID
 
 # Return with success or failure of actual test
 exit $EXITCODE

SAS/LSMR/test/t_lsmrapp_models.py

@@ -21,6 +21,7 @@
 
 import rest_framework.test
 from lsmr.lsmrapp.models import GeneratorTemplate, RunTemplate, WorkRequestTemplate, WorkRelationTemplate
+
 from django.utils import timezone # can't use datetime due to offset-awareness
 
 
@@ -48,6 +49,7 @@ class GeneratorTemplateTest(rest_framework.test.APITestCase):
         # setup
         before = timezone.now()
         entry = GeneratorTemplate.objects.create(**self.test_data_1)
+        entry.save()
 
         after = timezone.now()
 
@@ -69,12 +71,12 @@ class GeneratorTemplateTest(rest_framework.test.APITestCase):
     def test_GET_GeneratorTemplate_view_returns_correct_entry(self):
 
         # setup
-        entry = GeneratorTemplate.objects.create(**self.test_data_1)
-        entry = GeneratorTemplate.objects.create(**self.test_data_2)
+        id1 = GeneratorTemplate.objects.create(**self.test_data_1).id
+        id2 = GeneratorTemplate.objects.create(**self.test_data_2).id
 
         # assert
-        response1 = client.get('/generator_template/1/', format='json', follow=True)
-        response2 = client.get('/generator_template/2/', format='json', follow=True)
+        response1 = client.get('/generator_template/%s/' % id1, format='json', follow=True)
+        response2 = client.get('/generator_template/%s/' % id2, format='json', follow=True)
         self.assertEqual(response1.status_code, 200)
         self.assertEqual(response2.status_code, 200)
         for item in self.test_data_1.items(): # assertDictContainsSubset() is deprecated...

SAS/LSMR/test/t_lsmrapp_models.run

 #!/bin/sh
 
-$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --keepdb --pattern="t_lsmrapp_models.py"
+$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --keepdb --pattern="t_lsmrapp_models.py" --testrunner=postgres_testrunner.PostgresqlTestRunner