diff --git a/.gitattributes b/.gitattributes index 6bb6a031208d4c7ebcfc7c139b80a07adf1f8dc5..ae440c1ed18c53e84b2204b6c006f97dd0079f7b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4234,6 +4234,7 @@ SAS/Feedback_Service/src/fb_data_44883.txt -text SAS/LSMR/CMakeLists.txt -text SAS/LSMR/bin/CMakeLists.txt -text SAS/LSMR/bin/lsmr -text +SAS/LSMR/bin/lsmr_testdatabase -text SAS/LSMR/doc/LSMR.md -text SAS/LSMR/doc/package.dox -text SAS/LSMR/src/CMakeLists.txt -text @@ -4253,6 +4254,8 @@ SAS/LSMR/src/lsmr/urls.py -text SAS/LSMR/src/lsmr/wsgi.py -text SAS/LSMR/src/manage.py -text SAS/LSMR/test/CMakeLists.txt -text +SAS/LSMR/test/__init__.py -text +SAS/LSMR/test/postgres_testrunner.py -text SAS/LSMR/test/t_functional.py -text SAS/LSMR/test/t_functional.run -text SAS/LSMR/test/t_functional.sh -text diff --git a/SAS/LSMR/bin/CMakeLists.txt b/SAS/LSMR/bin/CMakeLists.txt index 41aa725cbbbb4e94f0455a86df078836b31828f4..5305dc9285be3bad326e982134a49783026212f4 100644 --- a/SAS/LSMR/bin/CMakeLists.txt +++ b/SAS/LSMR/bin/CMakeLists.txt @@ -1,2 +1,2 @@ lofar_add_bin_scripts(lsmr) - +lofar_add_bin_scripts(lsmr_testdatabase) diff --git a/SAS/LSMR/bin/lsmr b/SAS/LSMR/bin/lsmr index 4bc64a03ed5a99ef3ed4c448e188ea3428d862de..79c339440963702cf31229e7a5c5e7237340431e 100755 --- a/SAS/LSMR/bin/lsmr +++ b/SAS/LSMR/bin/lsmr @@ -18,7 +18,7 @@ # with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>. -# Startup script for the external trigger rest webservice +# Startup script for the development LSMR server echo "---" echo "! This is for testing only, properly deploy in Nginx or Apache for production use!" @@ -36,7 +36,7 @@ p) PORT=${OPTARG};; esac done -echo "Using port $PORT" +echo "Using Django port $PORT" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" /usr/bin/env python3 $DIR/../lib*/python*/site-packages/lofar/sas/lsmr/manage.py migrate diff --git a/SAS/LSMR/bin/lsmr_testdatabase b/SAS/LSMR/bin/lsmr_testdatabase new file mode 100644 index 0000000000000000000000000000000000000000..e0b393192e32cba252c0bfa637e15accc80d0550 --- /dev/null +++ b/SAS/LSMR/bin/lsmr_testdatabase @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy) +# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands +# +# This file is part of the LOFAR software suite. +# The LOFAR software suite is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# The LOFAR software suite is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>. + + +# Script to run a temporary postgres instance for easy functional testing + +echo "---" +echo "! This is for testing only! Database is not persistent!" +echo "---" + + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +/usr/bin/env python3 ${LOFARROOT}/../SAS/LSMR/test/postgres_testrunner.py \ No newline at end of file diff --git a/SAS/LSMR/src/lsmr/lsmrapp/CMakeLists.txt b/SAS/LSMR/src/lsmr/lsmrapp/CMakeLists.txt index e1c1088ec3d31534c62fc2ef9e1f753fdf9e3748..cecf42a4071376bc12380f9097341d5013702512 100644 --- a/SAS/LSMR/src/lsmr/lsmrapp/CMakeLists.txt +++ b/SAS/LSMR/src/lsmr/lsmrapp/CMakeLists.txt @@ -6,8 +6,8 @@ set(_py_files admin.py apps.py models.py - tests.py - views.py + viewsets.py + serializers.py ) python_install(${_py_files} diff --git a/SAS/LSMR/src/lsmr/settings.py b/SAS/LSMR/src/lsmr/settings.py index 651ffbf3e5c6159c417e5ef7f8a3ae521589e4a4..d19aa86aca285b148a142f4857c7f534ad78d873 100644 --- a/SAS/LSMR/src/lsmr/settings.py +++ b/SAS/LSMR/src/lsmr/settings.py @@ -78,17 +78,20 @@ WSGI_APPLICATION = 'lsmr.wsgi.application' DATABASES = { 'default': { # SQLITE: - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + #'ENGINE': 'django.db.backends.sqlite3', + #'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + # Postgres: - #'ENGINE': 'django.db.backends.postgresql_psycopg2', - #'NAME': 'lsmr', - #'USER': 'lsmr', - #'PASSWORD': 'fixme', - #'HOST': 'localhost', - #'PORT': '', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'lsmr', + 'USER': 'lsmruser', + 'PASS': 'lsmrpass2', + 'HOST': 'localhost', + 'PORT': 7666, + } } -} + + REST_FRAMEWORK = { diff --git a/SAS/LSMR/test/CMakeLists.txt b/SAS/LSMR/test/CMakeLists.txt index ea6dc3bd9674ae2eccd1727b0a955c474d8731a8..1d56ef7f7a9f1d5cb85ea5fca7d77a42a84dd90b 100644 --- a/SAS/LSMR/test/CMakeLists.txt +++ b/SAS/LSMR/test/CMakeLists.txt @@ -10,3 +10,10 @@ find_python_module(requests REQUIRED) lofar_add_test(t_lsmrapp_models) lofar_add_test(t_functional) + +# Copy testrunner module so Python won't complain about incorrect import location: +set(_py_files + __init__.py + postgres_testrunner.py +) +file(COPY ${_py_files} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) \ No newline at end of file diff --git a/SAS/LSMR/test/__init__.py b/SAS/LSMR/test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/SAS/LSMR/test/postgres_testrunner.py b/SAS/LSMR/test/postgres_testrunner.py new file mode 100755 index 0000000000000000000000000000000000000000..e61622063c85414d721209725d1276c266ce1cf0 --- /dev/null +++ b/SAS/LSMR/test/postgres_testrunner.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +""" +This allows running Django tests using a temporary postgres db (set up by tesing.postgresql) instead of using the +stock config in settings.py. Run as script to start postgres instance, returns PID to allow stopping it after use. + +To use this in Django tests, run: '.../manage.py test lsmrapp --testrunner=postgres_testrunner.PostgresqlTestRunner' +""" + +import testing.postgresql +from django.test.runner import DiscoverRunner +import psycopg2 +from lofar.sas.lsmr.lsmr.settings import DATABASES + + +try: + import testing.postgresql +except ImportError as e: + print(str(e)) + print('Please install python package testing.postgresql: sudo pip3 install testing.postgresql') + exit(3) # special lofar test exit code: skipped test + +def execute_query(pg, query): + """ + Run a raw SQL query on the given database + :param pg: A testing.postgresql.Postgresql instance + :param query: The query to execute + """ + + # connect to Postgres as root + connection = psycopg2.connect(**pg.dsn()) + connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cursor = connection.cursor() + + # create user role + cursor.execute(query) + cursor.close() + connection.commit() + connection.close() + + +def create_admin_role(pg, user, passw): + """ + Adds a superuser to the provided Postgres instance + :param pg: A testing.postgresql.Postgresql instance + :return: + """ + query = "CREATE ROLE %s WITH SUPERUSER LOGIN PASSWORD '%s';" % (user, passw) + execute_query(pg, query) + + +def create_project_db(pg, name): + """ + Creates a new database in the provided Postgres instance + :param pg: A testing.postgresql.Postgresql instance + :param name: The name of the database to create + """ + name = DATABASES['default']['NAME'] + query = "CREATE DATABASE %s;" % (name) + execute_query(pg, query) + + +def create_test_postgres(host, port, user, passw): + """ + Fires up a non-persistent Postgresql database with superuser for the provided details + :return: A testing.postgresql.Postgresql instance + """ + postgresql = testing.postgresql.Postgresql(host=host, port=port) + create_admin_role(postgresql, user, passw) + return postgresql + + +def create_test_Postgres_from_django_settings(): + """ + Reads default connection details from settings.py and creates an according Postgres instance + :return: A testing.postgresql.Postgresql instance + """ + + host = DATABASES['default']['HOST'] + port = DATABASES['default']['PORT'] + user = DATABASES['default']['USER'] + passw = DATABASES['default']['PASS'] + return create_test_postgres(host, port, user, passw) + + +class PostgresqlTestRunner(DiscoverRunner): + """ + A test runner to pass to manage.py test to start and tear down a database for testing. + Uses the connection details as configured in settings.py + """ + + def setup_databases(self, **kwargs): + self.postgresql = create_test_Postgres_from_django_settings() + return super(PostgresqlTestRunner, self).setup_databases(**kwargs) + + def teardown_databases(self, old_config, **kwargs): + self.postgresql.stop() + + +if __name__ == "__main__": + """ + Start Django test database and keep it alive until interrupted + """ + + pg = create_test_Postgres_from_django_settings() + name = DATABASES['default']['NAME'] + create_project_db(pg, name) + + print("Started Postgres server according to settings.py details with PID %s" % pg.server_pid) + import time + while True: + time.sleep(1) + + diff --git a/SAS/LSMR/test/t_functional.py b/SAS/LSMR/test/t_functional.py index 577babf4465f6e68b9864e003017cfebe369f84c..5ceed97d7076ea1ca4c4ac09e2a77935ac918c3d 100755 --- a/SAS/LSMR/test/t_functional.py +++ b/SAS/LSMR/test/t_functional.py @@ -22,7 +22,7 @@ import unittest import requests -BASE_URL = 'http://localhost:8000' +BASE_URL = 'http://localhost:8777' class BasicFunctionTestCase(unittest.TestCase): diff --git a/SAS/LSMR/test/t_functional.run b/SAS/LSMR/test/t_functional.run index 18b24ca1b440cd313906e1640da0f98310bde052..4d270d2d841e6ff9b5bd28fad85b438365538858 100755 --- a/SAS/LSMR/test/t_functional.run +++ b/SAS/LSMR/test/t_functional.run @@ -1,11 +1,15 @@ #!/bin/sh +echo "---" +echo "! This requires a running Postgres instance with a configured database according to the settings.py configuration." +echo "! You can run the lsmr_testdatabase command to fire up a temporary instance." +echo "---" # Run Django test instance -PORT=8000 +PORT=8777 lsmr -p $PORT & -PID=$! -echo "Started server with PID: " $PID +DJANGO_PID=$! +echo "Started Django server with PID: " $DJANGO_PID sleep 3 # Run test @@ -14,7 +18,7 @@ EXITCODE=$? # Kill Django test instance pkill -f "lofar/sas/lsmr/manage.py runserver 0.0.0.0:$PORT" -kill $PID +kill $DJANGO_PID # Return with success or failure of actual test exit $EXITCODE diff --git a/SAS/LSMR/test/t_lsmrapp_models.py b/SAS/LSMR/test/t_lsmrapp_models.py index 28ebc247e384170b235ac6dd64381a8c9cffc2dc..91ae0a4b2eb72219eff0457b6fd9db4dd71f3b76 100755 --- a/SAS/LSMR/test/t_lsmrapp_models.py +++ b/SAS/LSMR/test/t_lsmrapp_models.py @@ -21,6 +21,7 @@ import rest_framework.test from lsmr.lsmrapp.models import GeneratorTemplate, RunTemplate, WorkRequestTemplate, WorkRelationTemplate + from django.utils import timezone # can't use datetime due to offset-awareness @@ -48,6 +49,7 @@ class GeneratorTemplateTest(rest_framework.test.APITestCase): # setup before = timezone.now() entry = GeneratorTemplate.objects.create(**self.test_data_1) + entry.save() after = timezone.now() @@ -69,12 +71,12 @@ class GeneratorTemplateTest(rest_framework.test.APITestCase): def test_GET_GeneratorTemplate_view_returns_correct_entry(self): # setup - entry = GeneratorTemplate.objects.create(**self.test_data_1) - entry = GeneratorTemplate.objects.create(**self.test_data_2) + id1 = GeneratorTemplate.objects.create(**self.test_data_1).id + id2 = GeneratorTemplate.objects.create(**self.test_data_2).id # assert - response1 = client.get('/generator_template/1/', format='json', follow=True) - response2 = client.get('/generator_template/2/', format='json', follow=True) + response1 = client.get('/generator_template/%s/' % id1, format='json', follow=True) + response2 = client.get('/generator_template/%s/' % id2, format='json', follow=True) self.assertEqual(response1.status_code, 200) self.assertEqual(response2.status_code, 200) for item in self.test_data_1.items(): # assertDictContainsSubset() is deprecated... diff --git a/SAS/LSMR/test/t_lsmrapp_models.run b/SAS/LSMR/test/t_lsmrapp_models.run index 8ec8c0da180352f2b4a9f717ff6b038cd129e9a0..043a8b9cc4a66852b4006e726d06c3050ee4decf 100755 --- a/SAS/LSMR/test/t_lsmrapp_models.run +++ b/SAS/LSMR/test/t_lsmrapp_models.run @@ -1,3 +1,3 @@ #!/bin/sh -$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --keepdb --pattern="t_lsmrapp_models.py" +$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --keepdb --pattern="t_lsmrapp_models.py" --testrunner=postgres_testrunner.PostgresqlTestRunner