Commit 47c79bf0 authored by Joern jkuensem's avatar Joern jkuensem

Task LSMR-23: Implementation of the scheduling model with tests.

parent 15e7e0d2
......@@ -4245,19 +4245,21 @@ SAS/LSMR/src/lsmr/lsmrapp/__init__.py -text
SAS/LSMR/src/lsmr/lsmrapp/admin.py -text
SAS/LSMR/src/lsmr/lsmrapp/apps.py -text
SAS/LSMR/src/lsmr/lsmrapp/migrations/0001_initial.py -text
SAS/LSMR/src/lsmr/lsmrapp/migrations/0002_auto_20180720_1308.py -text
SAS/LSMR/src/lsmr/lsmrapp/migrations/0003_populate.py -text
SAS/LSMR/src/lsmr/lsmrapp/migrations/0002_auto_20180801_0910.py -text
SAS/LSMR/src/lsmr/lsmrapp/migrations/CMakeLists.txt -text
SAS/LSMR/src/lsmr/lsmrapp/migrations/__init__.py -text
SAS/LSMR/src/lsmr/lsmrapp/models/CMakeLists.txt -text
SAS/LSMR/src/lsmr/lsmrapp/models/__init__.py -text
SAS/LSMR/src/lsmr/lsmrapp/models/scheduling.py -text
SAS/LSMR/src/lsmr/lsmrapp/models/specification.py -text
SAS/LSMR/src/lsmr/lsmrapp/populate.py -text
SAS/LSMR/src/lsmr/lsmrapp/serializers/CMakeLists.txt -text
SAS/LSMR/src/lsmr/lsmrapp/serializers/__init__.py -text
SAS/LSMR/src/lsmr/lsmrapp/serializers/scheduling.py -text
SAS/LSMR/src/lsmr/lsmrapp/serializers/specification.py -text
SAS/LSMR/src/lsmr/lsmrapp/viewsets/CMakeLists.txt -text
SAS/LSMR/src/lsmr/lsmrapp/viewsets/__init__.py -text
SAS/LSMR/src/lsmr/lsmrapp/viewsets/scheduling.py -text
SAS/LSMR/src/lsmr/lsmrapp/viewsets/specification.py -text
SAS/LSMR/src/lsmr/settings.py -text
SAS/LSMR/src/lsmr/urls.py -text
......@@ -4266,12 +4268,18 @@ SAS/LSMR/src/manage.py -text
SAS/LSMR/test/CMakeLists.txt -text
SAS/LSMR/test/__init__.py -text
SAS/LSMR/test/postgres_testrunner.py -text
SAS/LSMR/test/t_functional.py -text
SAS/LSMR/test/t_functional.run -text
SAS/LSMR/test/t_functional.sh -text
SAS/LSMR/test/t_lsmrapp_models.py -text
SAS/LSMR/test/t_lsmrapp_models.run -text
SAS/LSMR/test/t_lsmrapp_models.sh -text
SAS/LSMR/test/t_lsmrapp_scheduling_django.py -text
SAS/LSMR/test/t_lsmrapp_scheduling_django.run -text
SAS/LSMR/test/t_lsmrapp_scheduling_django.sh -text
SAS/LSMR/test/t_lsmrapp_scheduling_functional.py -text
SAS/LSMR/test/t_lsmrapp_scheduling_functional.run -text
SAS/LSMR/test/t_lsmrapp_scheduling_functional.sh -text
SAS/LSMR/test/t_lsmrapp_specification_django.py -text
SAS/LSMR/test/t_lsmrapp_specification_django.run -text
SAS/LSMR/test/t_lsmrapp_specification_django.sh -text
SAS/LSMR/test/t_lsmrapp_specification_functional.py -text
SAS/LSMR/test/t_lsmrapp_specification_functional.run -text
SAS/LSMR/test/t_lsmrapp_specification_functional.sh -text
SAS/MoM/CMakeLists.txt -text
SAS/MoM/MoMQueryService/CMakeLists.txt -text
SAS/MoM/MoMQueryService/MoMQueryServiceClient/CMakeLists.txt -text
......
from django.db import migrations
from ..populate import *
class Migration(migrations.Migration):
dependencies = [
('lsmrapp', '0002_auto_20180720_1308'),
]
operations = [ migrations.RunPython(populate_choices) ]
......@@ -4,6 +4,7 @@ include(PythonInstall)
set(_py_files
__init__.py
specification.py
scheduling.py
)
python_install(${_py_files}
......
from .specification import *
\ No newline at end of file
from .specification import *
from .scheduling import *
\ No newline at end of file
"""
This file contains the database models
"""
from django.db.models import Model, CharField, DateTimeField, BooleanField, ForeignKey, CASCADE, IntegerField, SET_NULL, PROTECT, ManyToManyField
from django.contrib.postgres.fields import ArrayField, JSONField
from django.contrib.postgres.indexes import GinIndex
from .specification import AbstractChoice, BasicCommon, Template, NamedCommon, WorkRequestBlueprint
from enum import Enum
#
# I/O
#
class TaskIORole(BasicCommon):
role = ForeignKey('RoleChoice', null=False, on_delete=CASCADE)
datatype = ForeignKey('DatatypeChoice', null=False, on_delete=CASCADE)
dataformats = ForeignKey('DataformatChoice', null=False, on_delete=CASCADE) # todo: <--- this needs to be an array of dataformats!
outputs = ManyToManyField('TaskTemplate', related_name='inputs_dataproduct_relation', blank=True) # blank=True required to make ManyToMany optional in Django validation
inputs = ManyToManyField('TaskTemplate', related_name='input_roles_dataproduct_relation', blank=True)
#
# Choices
#
class TaskStateChoice(AbstractChoice):
"""Defines the model and predefined list of possible TaskStatusChoice's for Task.
The items in the Choises class below are automagically populated into the database via a data migration."""
class Choices(Enum):
CREATING = "creating"
CREATED = "created"
SCHEDULING = "scheduling"
SCHEDULED = "scheduled"
QUEUEING = "queueing"
QUEUED = "queued"
STARTING = "starting"
STARTED = "started"
FINISHING = "finishing"
FINISHED = "finished"
CANCELLING = "cancelling"
CANCELLED = "cancelled"
ERROR = "error"
class TaskTypeChoice(AbstractChoice):
"""Defines the model and predefined list of possible TaskTypeChoice's for Task.
The items in the Choises class below are automagically populated into the database via a data migration."""
class Choices(Enum):
OBSERVATION = "observation"
PIPELINE = "pipeline"
COYP = "copy"
INSPECTION = "inspection"
DELETION = "deletion"
class StationTypeChoice(AbstractChoice):
"""Defines the model and predefined list of possible StationTypeChoice's for AntennaSet.
The items in the Choises class below are automagically populated into the database via a data migration."""
class Choices(Enum):
CORE = "core"
REMOTE = "remote"
INTERNATIONAL = "international"
#
# Templates
#
class TaskTemplate(Template):
queue = BooleanField(default=False)
realtime = BooleanField(default=False)
class DefaultTaskTemplate(BasicCommon):
name = CharField(max_length=30, unique=True)
template = ForeignKey('TaskTemplate', on_delete=PROTECT)
class DataproductSpecificationTemplate(Template):
pass
class DefaultDataproductSpecificationTemplate(BasicCommon):
name = CharField(max_length=30, unique=True)
template = ForeignKey('DataproductSpecificationTemplate', on_delete=PROTECT)
#
# Instance Objects
#
class Task(BasicCommon):
type = ForeignKey('TaskTypeChoice', null=False, on_delete=CASCADE)
start_time = DateTimeField()
stop_time = DateTimeField()
state = ForeignKey('TaskStateChoice', null=False, on_delete=PROTECT, related_name='task_states') # set CANCELLED or ERROR instead?
requested_state = ForeignKey('TaskStateChoice', null=False, on_delete=CASCADE, related_name='task_requested_states') # set CANCELLED or ERROR instead?
specification = JSONField()
work_request_blueprint = ForeignKey('WorkRequestBlueprint', null=True, on_delete=SET_NULL)
template = ForeignKey('TaskTemplate', null=False, on_delete=CASCADE)
#resource_claim = ForeignKey("ResourceClaim", null=False, on_delete=CASCADE) # todo <-- how is this external reference supposed to work?
class DataproductRelation(BasicCommon):
task = ForeignKey('Task', null=False, on_delete=CASCADE)
input = ForeignKey('Dataproduct', related_name='inputs_dataproduct_relation', null=True, on_delete=SET_NULL)
output = ForeignKey('Dataproduct', related_name='outputs_dataproduct_relation', null=True, on_delete=SET_NULL)
input_role = ForeignKey('TaskIORole', related_name='input_roles_dataproduct_relation', null=True, on_delete=SET_NULL)
output_role = ForeignKey('TaskIORole', related_name='output_roles_dataproduct_relation', null=True, on_delete=SET_NULL)
class Dataproduct(BasicCommon):
filename = CharField(max_length=128)
directory = CharField(max_length=1024)
dataformat = ForeignKey('DataformatChoice', null=False, on_delete=CASCADE)
deleted_since = DateTimeField(null=True)
pinned_since = DateTimeField(null=True)
specification = JSONField()
specification_template = ForeignKey('DataproductSpecificationTemplate', null=False, on_delete=CASCADE)
class AntennaSet(NamedCommon):
station_type = ForeignKey('StationTypeChoice', null=False, on_delete=CASCADE)
rcus = ArrayField(IntegerField(), size=128, blank=False)
inputs = ArrayField(CharField(max_length=30), size=128, blank=True)
......@@ -125,6 +125,7 @@ class WorkIORole(BasicCommon):
outputs = ForeignKey("WorkRequestTemplate", related_name='role_output', on_delete=CASCADE)
inputs = ForeignKey("WorkRequestTemplate", related_name='role_input', on_delete=CASCADE)
#
# Templates
#
......
......@@ -16,6 +16,7 @@ class Migration(migrations.Migration):
"""
from .models.specification import RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice
from .models.scheduling import TaskStateChoice, TaskTypeChoice, StationTypeChoice
def populate_choices(apps, schema_editor):
'''
......@@ -23,6 +24,7 @@ def populate_choices(apps, schema_editor):
each 'choice'type in RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice
:return: None
'''
for choice_class in [RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice]:
for choice_class in [RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice,
TaskStateChoice, TaskTypeChoice, StationTypeChoice]:
choice_class.objects.bulk_create([choice_class(value=x.value) for x in choice_class.Choices])
......@@ -4,7 +4,7 @@ include(PythonInstall)
set(_py_files
__init__.py
specification.py
scheduling.py
)
python_install(${_py_files}
......
from .specification import *
\ No newline at end of file
from .specification import *
from .scheduling import *
\ No newline at end of file
"""
This file contains the serializers (for the elsewhere defined data models)
"""
from rest_framework import serializers
from .. import models
class TaskIORoleSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.TaskIORole
fields = '__all__'
class TaskStateChoiceSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.TaskStateChoice
fields = '__all__'
class TaskTypeChoiceSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.TaskTypeChoice
fields = '__all__'
class StationTypeChoiceSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.StationTypeChoice
fields = '__all__'
class TaskTemplateSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.TaskTemplate
fields = '__all__'
class DefaultTaskTemplateSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.DefaultTaskTemplate
fields = '__all__'
class DataproductSpecificationTemplateSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.DataproductSpecificationTemplate
fields = '__all__'
class DefaultDataproductSpecificationTemplateSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.DefaultDataproductSpecificationTemplate
fields = '__all__'
class TaskSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.Task
fields = '__all__'
class DataproductRelationSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.DataproductRelation
fields = '__all__'
class DataproductSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.Dataproduct
fields = '__all__'
class AntennaSetSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.AntennaSet
fields = '__all__'
......@@ -3,7 +3,7 @@ This file contains the serializers (for the elsewhere defined data models)
"""
from rest_framework import serializers
from lsmr.lsmrapp import models
from .. import models
class TagsSerializer(serializers.HyperlinkedModelSerializer):
......
......@@ -4,6 +4,7 @@ include(PythonInstall)
set(_py_files
__init__.py
specification.py
scheduling.py
)
python_install(${_py_files}
......
from .specification import *
\ No newline at end of file
from .specification import *
from .scheduling import *
\ No newline at end of file
"""
This file contains the viewsets (based on the elsewhere defined data models and serializers)
"""
from rest_framework import viewsets
from .. import models
from .. import serializers
class TaskIORoleViewSet(viewsets.ModelViewSet):
queryset = models.TaskIORole.objects.all()
serializer_class = serializers.TaskIORoleSerializer
class TaskStateChoiceViewSet(viewsets.ModelViewSet):
queryset = models.TaskStateChoice.objects.all()
serializer_class = serializers.TaskStateChoiceSerializer
class TaskTypeChoiceViewSet(viewsets.ModelViewSet):
queryset = models.TaskTypeChoice.objects.all()
serializer_class = serializers.TaskTypeChoiceSerializer
class StationTypeChoiceViewSet(viewsets.ModelViewSet):
queryset = models.StationTypeChoice.objects.all()
serializer_class = serializers.StationTypeChoiceSerializer
class TaskTemplateViewSet(viewsets.ModelViewSet):
queryset = models.TaskTemplate.objects.all()
serializer_class = serializers.TaskTemplateSerializer
class DefaultTaskTemplateViewSet(viewsets.ModelViewSet):
queryset = models.DefaultTaskTemplate.objects.all()
serializer_class = serializers.DefaultTaskTemplateSerializer
class DataproductSpecificationTemplateViewSet(viewsets.ModelViewSet):
queryset = models.DataproductSpecificationTemplate.objects.all()
serializer_class = serializers.DataproductSpecificationTemplateSerializer
class DefaultDataproductSpecificationTemplateViewSet(viewsets.ModelViewSet):
queryset = models.DefaultDataproductSpecificationTemplate.objects.all()
serializer_class = serializers.DefaultDataproductSpecificationTemplateSerializer
class TaskViewSet(viewsets.ModelViewSet):
queryset = models.Task.objects.all()
serializer_class = serializers.TaskSerializer
class DataproductRelationViewSet(viewsets.ModelViewSet):
queryset = models.DataproductRelation.objects.all()
serializer_class = serializers.DataproductRelationSerializer
class DataproductViewSet(viewsets.ModelViewSet):
queryset = models.Dataproduct.objects.all()
serializer_class = serializers.DataproductSerializer
class AntennaSetViewSet(viewsets.ModelViewSet):
queryset = models.AntennaSet.objects.all()
serializer_class = serializers.AntennaSetSerializer
......@@ -3,7 +3,7 @@ This file contains the viewsets (based on the elsewhere defined data models and
"""
from rest_framework import viewsets
from lsmr.lsmrapp import models, serializers
from .. import models, serializers
class TagsViewSet(viewsets.ModelViewSet):
......
......@@ -19,7 +19,7 @@ from django.urls import path
from django.views.generic.base import TemplateView
from rest_framework import routers
from lsmr.lsmrapp import viewsets
from .lsmrapp import viewsets, models, serializers
#
......@@ -38,15 +38,19 @@ urlpatterns = [
router = routers.SimpleRouter()
router.register(r'tags', viewsets.TagsViewSet)
# SPECIFICATION
# choices
router.register(r'role_choice', viewsets.RoleChoiceViewSet)
router.register(r'datatype_choice', viewsets.DatatypeChoiceViewSet)
router.register(r'dataformat_choice', viewsets.DataformatChoiceViewSet)
router.register(r'copy_reason_choice', viewsets.CopyReasonChoiceViewSet)
# templates
router.register(r'generator_template', viewsets.GeneratorTemplateViewSet)
router.register(r'run_template', viewsets.RunTemplateViewSet)
router.register(r'work_request_template', viewsets.WorkRequestTemplateViewSet)
router.register(r'work_relation_selection_template', viewsets.WorkRelationSelectionTemplateViewSet)
router.register(r'role_choice', viewsets.RoleChoiceViewSet)
router.register(r'datatype_choice', viewsets.DatatypeChoiceViewSet)
router.register(r'dataformat_choice', viewsets.DataformatChoiceViewSet)
router.register(r'copy_reason_choice', viewsets.CopyReasonChoiceViewSet)
router.register(r'work_io_role', viewsets.WorkIORoleViewSet)
router.register(r'default_generator_template', viewsets.DefaultGeneratorTemplateViewSet)
router.register(r'default_run_template', viewsets.DefaultRunTemplateViewSet)
......@@ -65,4 +69,24 @@ router.register(r'work_request_relation_draft', viewsets.WorkRequestRelationDraf
router.register(r'work_request_relation_blueprint', viewsets.WorkRequestRelationBlueprintViewSet)
# SCHEDULING
# choices
router.register(r'task_state_choice', viewsets.TaskStateChoiceViewSet)
router.register(r'task_type_choice', viewsets.TaskTypeChoiceViewSet)
router.register(r'station_type_choice', viewsets.StationTypeChoiceViewSet)
# templates
router.register(r'task_io_role', viewsets.TaskIORoleViewSet)
router.register(r'task_template', viewsets.TaskTemplateViewSet)
router.register(r'dataproduct_specification_template', viewsets.DataproductSpecificationTemplateViewSet)
# instances
router.register(r'task', viewsets.TaskViewSet)
router.register(r'dataproduct', viewsets.DataproductViewSet)
router.register(r'dataproduct_relation', viewsets.DataproductRelationViewSet)
router.register(r'antenna_set', viewsets.AntennaSetViewSet)
# ---
urlpatterns.extend(router.urls)
......@@ -10,8 +10,10 @@ find_python_module(requests REQUIRED)
#find_python_module(sqlalchemy)
#find_python_module(testing.postgresql)
lofar_add_test(t_lsmrapp_models)
lofar_add_test(t_functional)
lofar_add_test(t_lsmrapp_specification_django)
lofar_add_test(t_lsmrapp_specification_functional)
lofar_add_test(t_lsmrapp_scheduling_django)
lofar_add_test(t_lsmrapp_scheduling_functional)
# Copy testrunner module so Python won't complain about incorrect import location:
set(_py_files
......
......@@ -11,6 +11,7 @@ import testing.postgresql
from django.test.runner import DiscoverRunner
import psycopg2
from lofar.sas.lsmr.lsmr.settings import DATABASES
import time
# todo: consider switching from testing.postgresql to pyembedpg to allow easy testing on specific versions. That may be slower, though.
......@@ -47,7 +48,10 @@ def create_admin_role(pg, user, passw):
:return:
"""
query = "CREATE ROLE %s WITH SUPERUSER LOGIN PASSWORD '%s';" % (user, passw)
execute_query(pg, query)
try:
execute_query(pg, query)
except Exception as e:
print(e)
def create_project_db(pg, name):
......@@ -92,6 +96,7 @@ class PostgresqlTestRunner(DiscoverRunner):
def setup_databases(self, **kwargs):
self.postgresql = create_test_Postgres_from_django_settings()
print('created test postgres instance based on Django settings')
return super(PostgresqlTestRunner, self).setup_databases(**kwargs)
def teardown_databases(self, old_config, **kwargs):
......@@ -109,7 +114,6 @@ if __name__ == "__main__":
print("Started Postgres server according to settings.py details with PID %s" % pg.server_pid)
print("Waiting for your interrupt to stop it.")
import time
while True:
time.sleep(1)
......
#!/bin/sh
./runctest.sh t_lsmrapp_models
This diff is collapsed.
#!/bin/sh
$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --keepdb --pattern="t_lsmrapp_models.py" --testrunner=postgres_testrunner.PostgresqlTestRunner
$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --pattern="t_lsmrapp_scheduling_django.py" --testrunner=postgres_testrunner.PostgresqlTestRunner
#!/bin/sh
./runctest.sh t_lsmrapp_scheduling_django
\ No newline at end of file
This diff is collapsed.
......@@ -13,7 +13,7 @@ echo "Started Django server with PID: " $DJANGO_PID
sleep 15
# Run test
./t_functional.py
./t_lsmrapp_scheduling_functional.py
EXITCODE=$?
# Kill Django test instance
......
#!/bin/sh
# Run Test
./runctest.sh t_functional
./runctest.sh t_lsmrapp_scheduling_functional
......@@ -25,7 +25,7 @@ from lsmr.lsmrapp.populate import populate_choices
from lsmr.lsmrapp import models
from django.db.utils import IntegrityError
from datetime import datetime
from datetime import datetime
# use this to create timezone-aware datetime objects: from django.utils import timezone
client = rest_framework.test.APIClient()
......
#!/bin/sh
$LOFARROOT/lib*/python*/site-packages/lofar/sas/lsmr/manage.py test --pattern="t_lsmrapp_specification_django.py" --testrunner=postgres_testrunner.PostgresqlTestRunner
#!/bin/sh
./runctest.sh t_lsmrapp_specification_django
#!/bin/sh
echo "---"
echo "! This requires a running Postgres instance with a configured database according to the settings.py configuration."
echo "! You can run the lsmr_testdatabase command to fire up a temporary instance."
echo "---"
# Run Django test instance
PORT=8777
lsmr -p $PORT &
DJANGO_PID=$!
echo "Started Django server with PID: " $DJANGO_PID
sleep 15
# Run test
./t_lsmrapp_specification_functional.py
EXITCODE=$?
# Kill Django test instance
pkill -f "lofar/sas/lsmr/manage.py runserver 0.0.0.0:$PORT"
kill $DJANGO_PID
# Return with success or failure of actual test
exit $EXITCODE
#!/bin/sh
# Run Test
./runctest.sh t_lsmrapp_specification_functional
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment