From bc265b4bc6b0e8114062c89ca2b0d33d7cc2d983 Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Thu, 30 Apr 2020 18:09:57 +0200
Subject: [PATCH] TMSS-60: as discussed with the architect, removed
overdesigned SubtaskConnector
---
.../tmss/tmssapp/migrations/0001_initial.py | 37 +------------------
.../tmss/tmssapp/migrations/0002_populate.py | 1 -
.../src/tmss/tmssapp/models/scheduling.py | 14 -------
.../tmss/tmssapp/serializers/scheduling.py | 6 ---
.../src/tmss/tmssapp/viewsets/scheduling.py | 5 ---
SAS/TMSS/src/tmss/urls.py | 1 -
.../test/t_tmssapp_scheduling_django_API.py | 26 -------------
SAS/TMSS/test/tmss_test_data_django_models.py | 20 +---------
SAS/TMSS/test/tmss_test_data_rest.py | 23 ++----------
9 files changed, 5 insertions(+), 128 deletions(-)
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index eacc7a839ab..05c983b640f 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.10 on 2020-04-17 08:37
+# Generated by Django 2.2.12 on 2020-04-30 16:03
from django.conf import settings
import django.contrib.postgres.fields
@@ -303,20 +303,6 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
- migrations.CreateModel(
- name='SubtaskConnector',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
- ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
- ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('dataformats', models.ManyToManyField(blank=True, to='tmssapp.Dataformat')),
- ('datatype', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Datatype')),
- ],
- options={
- 'abstract': False,
- },
- ),
migrations.CreateModel(
name='SubtaskInputSelectionTemplate',
fields=[
@@ -557,7 +543,6 @@ class Migration(migrations.Migration):
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('connector', models.ForeignKey(help_text='Which connector this Subtask Output implements.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='tmssapp.SubtaskConnector')),
('subtask', models.ForeignKey(help_text='Subtask to which this output specification refers.', on_delete=django.db.models.deletion.CASCADE, to='tmssapp.Subtask')),
],
options={
@@ -572,7 +557,6 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('selection_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Filter to apply to the dataproducts of the producer, to derive input dataproducts when scheduling.')),
- ('connector', models.ForeignKey(help_text='Which connector this Task Input implements.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='tmssapp.SubtaskConnector')),
('dataproducts', models.ManyToManyField(help_text='The Dataproducts resulting from application of the filter at time of scheduling Although the dataproducts are simply the result of applying the filter on immutable data, the filter application could change over time. We thus store the result of this filtering directly to retain which input was specified for the task..', to='tmssapp.Dataproduct')),
('producer', models.ForeignKey(help_text='The Subtask Output providing the input dataproducts.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SubtaskOutput')),
('selection_template', models.ForeignKey(help_text='Schema used for selection_doc.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SubtaskInputSelectionTemplate')),
@@ -583,21 +567,6 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
- migrations.AddField(
- model_name='subtaskconnector',
- name='input_of',
- field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='outputs', to='tmssapp.SubtaskTemplate'),
- ),
- migrations.AddField(
- model_name='subtaskconnector',
- name='output_of',
- field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='inputs', to='tmssapp.SubtaskTemplate'),
- ),
- migrations.AddField(
- model_name='subtaskconnector',
- name='role',
- field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Role'),
- ),
migrations.AddField(
model_name='subtask',
name='specifications_template',
@@ -873,10 +842,6 @@ class Migration(migrations.Migration):
model_name='subtaskinput',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_fb9960_gin'),
),
- migrations.AddIndex(
- model_name='subtaskconnector',
- index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_60e299_gin'),
- ),
migrations.AddIndex(
model_name='subtask',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_d2fc43_gin'),
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
index 7a25dab177b..e33461a9295 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
@@ -17,6 +17,5 @@ class Migration(migrations.Migration):
# Start SubTask id with 2 000 000 to avoid overlap with 'old' (test/production) OTDB
operations = [ migrations.RunSQL('ALTER SEQUENCE tmssapp_SubTask_id_seq RESTART WITH 2000000;'),
migrations.RunPython(populate_choices),
- migrations.RunPython(populate_resources),
migrations.RunPython(populate_misc),
migrations.RunPython(populate_lofar_json_schemas) ]
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index 2a06bb5e7c9..55719256cb9 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -17,18 +17,6 @@ from lofar.sas.tmss.tmss.tmssapp.validation import validate_json_against_schema
# I/O
#
-class SubtaskConnector(BasicCommon):
- """
- Represents the relation between input and output of the Subtasks. Some of these relations implement the Task
- Relations. An input is tied to an output of another Subtask, and allows a filter to be specified.
- """
- role = ForeignKey('Role', null=False, on_delete=PROTECT)
- datatype = ForeignKey('Datatype', null=False, on_delete=PROTECT)
- dataformats = ManyToManyField('Dataformat', blank=True)
- output_of = ForeignKey('SubtaskTemplate', related_name='inputs', blank=True, on_delete=PROTECT)
- input_of = ForeignKey('SubtaskTemplate', related_name='outputs', blank=True, on_delete=PROTECT)
-
-
#
# Choices
#
@@ -200,7 +188,6 @@ class SubtaskStateLog(BasicCommon):
class SubtaskInput(BasicCommon):
subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, help_text='Subtask to which this input specification refers.')
task_relation_blueprint = ForeignKey('TaskRelationBlueprint', null=True, on_delete=SET_NULL, help_text='Task Relation Blueprint which this Subtask Input implements (NULLable).')
- connector = ForeignKey('SubtaskConnector', null=True, on_delete=SET_NULL, help_text='Which connector this Task Input implements.')
producer = ForeignKey('SubtaskOutput', on_delete=PROTECT, help_text='The Subtask Output providing the input dataproducts.')
dataproducts = ManyToManyField('Dataproduct', help_text='The Dataproducts resulting from application of the filter at time of scheduling Although the dataproducts are simply the result of applying the filter on immutable data, the filter application could change over time. We thus store the result of this filtering directly to retain which input was specified for the task..')
selection_doc = JSONField(help_text='Filter to apply to the dataproducts of the producer, to derive input dataproducts when scheduling.')
@@ -215,7 +202,6 @@ class SubtaskInput(BasicCommon):
class SubtaskOutput(BasicCommon):
subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, help_text='Subtask to which this output specification refers.')
- connector = ForeignKey('SubtaskConnector', null=True, on_delete=SET_NULL, help_text='Which connector this Subtask Output implements.')
class Dataproduct(BasicCommon):
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
index 7d29e4e4ffa..c2618290363 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
@@ -10,12 +10,6 @@ from .. import models
from .specification import RelationalHyperlinkedModelSerializer
-class SubtaskConnectorSerializer(serializers.HyperlinkedModelSerializer):
- class Meta:
- model = models.SubtaskConnector
- fields = '__all__'
-
-
class SubtaskStateSerializer(serializers.ModelSerializer):
class Meta:
model = models.SubtaskState
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
index edc9ab43d27..54fd549aec0 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
@@ -37,11 +37,6 @@ from lofar.common.datetimeutils import formatDatetime
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
-class SubtaskConnectorViewSet(LOFARViewSet):
- queryset = models.SubtaskConnector.objects.all()
- serializer_class = serializers.SubtaskConnectorSerializer
-
-
class SubtaskStateViewSet(LOFARViewSet):
queryset = models.SubtaskState.objects.all()
serializer_class = serializers.SubtaskStateSerializer
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index 8bee6c119ed..fe87b9ff18e 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -119,7 +119,6 @@ router.register(r'algorithm', viewsets.AlgorithmViewSet)
router.register(r'schedule_method', viewsets.ScheduleMethodViewSet)
# templates
-router.register(r'subtask_connector', viewsets.SubtaskConnectorViewSet)
router.register(r'subtask_template', viewsets.SubtaskTemplateViewSet)
router.register(r'dataproduct_specifications_template', viewsets.DataproductSpecificationsTemplateViewSet)
router.register(r'default_subtask_template', viewsets.DefaultSubtaskTemplateViewSet)
diff --git a/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py b/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
index 9fa9a987f1e..b4a5a2b1d3a 100755
--- a/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
+++ b/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
@@ -249,32 +249,6 @@ class DataproductTest(unittest.TestCase):
models.Dataproduct.objects.create(**test_data)
-class SubtaskConnectorTest(unittest.TestCase):
- def test_SubtaskConnector_gets_created_with_correct_creation_timestamp(self):
-
- # setup
- before = datetime.utcnow()
- entry = models.SubtaskConnector.objects.create(**SubtaskConnector_test_data())
-
- after = datetime.utcnow()
-
- # assert
- self.assertLess(before, entry.created_at)
- self.assertGreater(after, entry.created_at)
-
- def test_SubtaskConnector_update_timestamp_gets_changed_correctly(self):
-
- # setup
- entry = models.SubtaskConnector.objects.create(**SubtaskConnector_test_data())
- before = datetime.utcnow()
- entry.save()
- after = datetime.utcnow()
-
- # assert
- self.assertLess(before, entry.updated_at)
- self.assertGreater(after, entry.updated_at)
-
-
class AntennaSetTest(unittest.TestCase):
def test_AntennaSet_gets_created_with_correct_creation_timestamp(self):
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index dfdde4add91..711335d9752 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -224,21 +224,16 @@ def DataproductFeedbackTemplate_test_data() -> dict:
"schema": {"mykey": "my value"},
"tags": ["TMSS", "TESTING"]}
-def SubtaskOutput_test_data(subtask: models.Subtask=None, connector: models.SubtaskConnector=None) -> dict:
+def SubtaskOutput_test_data(subtask: models.Subtask=None) -> dict:
if subtask is None:
subtask = models.Subtask.objects.create(**Subtask_test_data())
- if connector is None:
- connector = models.SubtaskConnector.objects.create(**SubtaskConnector_test_data(output_of=subtask.specifications_template, input_of=subtask.specifications_template))
-
return {"subtask": subtask,
- "connector": connector,
"tags":[]}
def SubtaskInput_test_data() -> dict:
return {"subtask": models.Subtask.objects.create(**Subtask_test_data()),
"task_relation_blueprint": models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data()),
- "connector": models.SubtaskConnector.objects.create(**SubtaskConnector_test_data()),
"producer": models.SubtaskOutput.objects.create(**SubtaskOutput_test_data()),
#"dataproducts": models.Dataproduct.objects.create(**dpt.get_test_data()),
"selection_doc": "{}",
@@ -304,19 +299,6 @@ def Dataproduct_test_data(producer: models.SubtaskOutput=None,
"feedback_doc": "{}",
"feedback_template": models.DataproductFeedbackTemplate.objects.create(**DataproductFeedbackTemplate_test_data())}
-def SubtaskConnector_test_data(output_of: models.SubtaskTemplate=None, input_of: models.SubtaskTemplate=None) -> dict:
- if output_of is None:
- output_of = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data())
-
- if input_of is None:
- input_of = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data())
-
- return {"role": models.Role.objects.get(value='calibrator'),
- "datatype": models.Datatype.objects.get(value='instrument model'),
- "output_of": output_of,
- "input_of": input_of,
- "tags": []}
-
def AntennaSet_test_data() -> dict:
return {"name": "observation",
"description": 'My one observation',
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index 656a2f5cd70..af4d9f6e243 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -361,31 +361,14 @@ class TMSSRESTTestDataCreator():
"cluster": cluster_url,
"scheduler_input_doc": "{}" }
- def SubtaskOutput(self, subtask_url=None, subtask_connector_url=None):
+ def SubtaskOutput(self, subtask_url=None):
if subtask_url is None:
subtask_url = self.post_data_and_get_url(self.Subtask(), '/subtask/')
- if subtask_connector_url is None:
- subtask_connector_url = self.post_data_and_get_url(self.SubtaskConnector(), '/subtask_connector/')
-
+
return {"subtask": subtask_url,
- "connector": subtask_connector_url,
"tags": []}
-
- def SubtaskConnector(self, input_of_url=None, output_of_url=None):
- if input_of_url is None:
- input_of_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/subtask_template/')
-
- if output_of_url is None:
- output_of_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/subtask_template/')
-
- return {"role": self.django_api_url + '/role/correlator/',
- "datatype": self.django_api_url + '/datatype/image/',
- "dataformats": [self.django_api_url + '/dataformat/Beamformed/'],
- "output_of": output_of_url,
- "input_of": input_of_url,
- "tags": []}
-
+
def Dataproduct(self, filename="my_filename", specifications_template_url=None, subtask_output_url=None, dataproduct_feedback_template_url=None):
if specifications_template_url is None:
specifications_template_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/dataproduct_specifications_template/')
--
GitLab