TMSS-60: as discussed with the architect, removed overdesigned SubtaskConnector

SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py

-# Generated by Django 2.2.10 on 2020-04-17 08:37
+# Generated by Django 2.2.12 on 2020-04-30 16:03
 
 from django.conf import settings
 import django.contrib.postgres.fields
@@ -303,20 +303,6 @@ class Migration(migrations.Migration):
                 'abstract': False,
             },
         ),
-        migrations.CreateModel(
-            name='SubtaskConnector',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
-                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
-                ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
-                ('dataformats', models.ManyToManyField(blank=True, to='tmssapp.Dataformat')),
-                ('datatype', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Datatype')),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
         migrations.CreateModel(
             name='SubtaskInputSelectionTemplate',
             fields=[
@@ -557,7 +543,6 @@ class Migration(migrations.Migration):
                 ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
                 ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
                 ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
-                ('connector', models.ForeignKey(help_text='Which connector this Subtask Output implements.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='tmssapp.SubtaskConnector')),
                 ('subtask', models.ForeignKey(help_text='Subtask to which this output specification refers.', on_delete=django.db.models.deletion.CASCADE, to='tmssapp.Subtask')),
             ],
             options={
@@ -572,7 +557,6 @@ class Migration(migrations.Migration):
                 ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
                 ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
                 ('selection_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Filter to apply to the dataproducts of the producer, to derive input dataproducts when scheduling.')),
-                ('connector', models.ForeignKey(help_text='Which connector this Task Input implements.', null=True, on_delete=django.db.models.deletion.SET_NULL, to='tmssapp.SubtaskConnector')),
                 ('dataproducts', models.ManyToManyField(help_text='The Dataproducts resulting from application of the filter at time of scheduling Although the dataproducts are simply the result of applying the filter on immutable data, the filter application could change over time. We thus store the result of this filtering directly to retain which input was specified for the task..', to='tmssapp.Dataproduct')),
                 ('producer', models.ForeignKey(help_text='The Subtask Output providing the input dataproducts.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SubtaskOutput')),
                 ('selection_template', models.ForeignKey(help_text='Schema used for selection_doc.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SubtaskInputSelectionTemplate')),
@@ -583,21 +567,6 @@ class Migration(migrations.Migration):
                 'abstract': False,
             },
         ),
-        migrations.AddField(
-            model_name='subtaskconnector',
-            name='input_of',
-            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='outputs', to='tmssapp.SubtaskTemplate'),
-        ),
-        migrations.AddField(
-            model_name='subtaskconnector',
-            name='output_of',
-            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.PROTECT, related_name='inputs', to='tmssapp.SubtaskTemplate'),
-        ),
-        migrations.AddField(
-            model_name='subtaskconnector',
-            name='role',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Role'),
-        ),
         migrations.AddField(
             model_name='subtask',
             name='specifications_template',
@@ -873,10 +842,6 @@ class Migration(migrations.Migration):
             model_name='subtaskinput',
             index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_fb9960_gin'),
         ),
-        migrations.AddIndex(
-            model_name='subtaskconnector',
-            index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_60e299_gin'),
-        ),
         migrations.AddIndex(
             model_name='subtask',
             index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_d2fc43_gin'),

SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py

@@ -17,6 +17,5 @@ class Migration(migrations.Migration):
     # Start SubTask id with 2 000 000 to avoid overlap with 'old' (test/production) OTDB
     operations = [ migrations.RunSQL('ALTER SEQUENCE tmssapp_SubTask_id_seq RESTART WITH 2000000;'),
                    migrations.RunPython(populate_choices),
-                   migrations.RunPython(populate_resources),
                    migrations.RunPython(populate_misc),
                    migrations.RunPython(populate_lofar_json_schemas) ]

SAS/TMSS/src/tmss/tmssapp/models/scheduling.py

@@ -17,18 +17,6 @@ from lofar.sas.tmss.tmss.tmssapp.validation import validate_json_against_schema
 # I/O
 #
 
-class SubtaskConnector(BasicCommon):
-    """
-    Represents the relation between input and output of the Subtasks. Some of these relations implement the Task
-    Relations. An input is tied to an output of another Subtask, and allows a filter to be specified.
-    """
-    role = ForeignKey('Role', null=False, on_delete=PROTECT)
-    datatype = ForeignKey('Datatype', null=False, on_delete=PROTECT)
-    dataformats = ManyToManyField('Dataformat', blank=True)
-    output_of = ForeignKey('SubtaskTemplate', related_name='inputs', blank=True, on_delete=PROTECT)
-    input_of = ForeignKey('SubtaskTemplate', related_name='outputs', blank=True, on_delete=PROTECT)
-
-
 #
 # Choices
 #
@@ -200,7 +188,6 @@ class SubtaskStateLog(BasicCommon):
 class SubtaskInput(BasicCommon):
     subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, help_text='Subtask to which this input specification refers.')
     task_relation_blueprint = ForeignKey('TaskRelationBlueprint', null=True, on_delete=SET_NULL, help_text='Task Relation Blueprint which this Subtask Input implements (NULLable).')
-    connector = ForeignKey('SubtaskConnector', null=True, on_delete=SET_NULL, help_text='Which connector this Task Input implements.')
     producer = ForeignKey('SubtaskOutput', on_delete=PROTECT, help_text='The Subtask Output providing the input dataproducts.')
     dataproducts = ManyToManyField('Dataproduct', help_text='The Dataproducts resulting from application of the filter at time of scheduling Although the dataproducts are simply the result of applying the filter on immutable data, the filter application could change over time. We thus store the result of this filtering directly to retain which input was specified for the task..')
     selection_doc = JSONField(help_text='Filter to apply to the dataproducts of the producer, to derive input dataproducts when scheduling.')
@@ -215,7 +202,6 @@ class SubtaskInput(BasicCommon):
 
 class SubtaskOutput(BasicCommon):
     subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, help_text='Subtask to which this output specification refers.')
-    connector = ForeignKey('SubtaskConnector', null=True, on_delete=SET_NULL, help_text='Which connector this Subtask Output implements.')
 
 
 class Dataproduct(BasicCommon):

SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py

@@ -10,12 +10,6 @@ from .. import models
 from .specification import RelationalHyperlinkedModelSerializer
 
 
-class SubtaskConnectorSerializer(serializers.HyperlinkedModelSerializer):
-    class Meta:
-        model = models.SubtaskConnector
-        fields = '__all__'
-
-
 class SubtaskStateSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.SubtaskState

SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py

@@ -37,11 +37,6 @@ from lofar.common.datetimeutils import formatDatetime
 from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
 
 
-class SubtaskConnectorViewSet(LOFARViewSet):
-    queryset = models.SubtaskConnector.objects.all()
-    serializer_class = serializers.SubtaskConnectorSerializer
-
-
 class SubtaskStateViewSet(LOFARViewSet):
     queryset = models.SubtaskState.objects.all()
     serializer_class = serializers.SubtaskStateSerializer

SAS/TMSS/src/tmss/urls.py

@@ -119,7 +119,6 @@ router.register(r'algorithm', viewsets.AlgorithmViewSet)
 router.register(r'schedule_method', viewsets.ScheduleMethodViewSet)
 
 # templates
-router.register(r'subtask_connector', viewsets.SubtaskConnectorViewSet)
 router.register(r'subtask_template', viewsets.SubtaskTemplateViewSet)
 router.register(r'dataproduct_specifications_template', viewsets.DataproductSpecificationsTemplateViewSet)
 router.register(r'default_subtask_template', viewsets.DefaultSubtaskTemplateViewSet)

SAS/TMSS/test/t_tmssapp_scheduling_django_API.py

@@ -249,32 +249,6 @@ class DataproductTest(unittest.TestCase):
             models.Dataproduct.objects.create(**test_data)
 
 
-class SubtaskConnectorTest(unittest.TestCase):
-    def test_SubtaskConnector_gets_created_with_correct_creation_timestamp(self):
-
-        # setup
-        before = datetime.utcnow()
-        entry = models.SubtaskConnector.objects.create(**SubtaskConnector_test_data())
-
-        after = datetime.utcnow()
-
-        # assert
-        self.assertLess(before, entry.created_at)
-        self.assertGreater(after, entry.created_at)
-
-    def test_SubtaskConnector_update_timestamp_gets_changed_correctly(self):
-
-        # setup
-        entry = models.SubtaskConnector.objects.create(**SubtaskConnector_test_data())
-        before = datetime.utcnow()
-        entry.save()
-        after = datetime.utcnow()
-
-        # assert
-        self.assertLess(before, entry.updated_at)
-        self.assertGreater(after, entry.updated_at)
-
-
 class AntennaSetTest(unittest.TestCase):
     def test_AntennaSet_gets_created_with_correct_creation_timestamp(self):
 

SAS/TMSS/test/tmss_test_data_django_models.py

@@ -224,21 +224,16 @@ def DataproductFeedbackTemplate_test_data() -> dict:
             "schema": {"mykey": "my value"},
             "tags": ["TMSS", "TESTING"]}
 
-def SubtaskOutput_test_data(subtask: models.Subtask=None, connector: models.SubtaskConnector=None) -> dict:
+def SubtaskOutput_test_data(subtask: models.Subtask=None) -> dict:
     if subtask is None:
         subtask = models.Subtask.objects.create(**Subtask_test_data())
 
-    if connector is None:
-        connector = models.SubtaskConnector.objects.create(**SubtaskConnector_test_data(output_of=subtask.specifications_template, input_of=subtask.specifications_template))
-
     return {"subtask": subtask,
-            "connector": connector,
             "tags":[]}
 
 def SubtaskInput_test_data() -> dict:
     return {"subtask": models.Subtask.objects.create(**Subtask_test_data()),
             "task_relation_blueprint": models.TaskRelationBlueprint.objects.create(**TaskRelationBlueprint_test_data()),
-            "connector": models.SubtaskConnector.objects.create(**SubtaskConnector_test_data()),
             "producer": models.SubtaskOutput.objects.create(**SubtaskOutput_test_data()),
             #"dataproducts": models.Dataproduct.objects.create(**dpt.get_test_data()),
             "selection_doc": "{}",
@@ -304,19 +299,6 @@ def Dataproduct_test_data(producer: models.SubtaskOutput=None,
             "feedback_doc": "{}",
             "feedback_template": models.DataproductFeedbackTemplate.objects.create(**DataproductFeedbackTemplate_test_data())}
 
-def SubtaskConnector_test_data(output_of: models.SubtaskTemplate=None, input_of: models.SubtaskTemplate=None) -> dict:
-    if output_of is None:
-        output_of = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data())
-
-    if input_of is None:
-        input_of = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data())
-
-    return {"role": models.Role.objects.get(value='calibrator'),
-            "datatype": models.Datatype.objects.get(value='instrument model'),
-            "output_of": output_of,
-            "input_of": input_of,
-            "tags": []}
-
 def AntennaSet_test_data() -> dict:
     return {"name": "observation",
             "description": 'My one observation',

SAS/TMSS/test/tmss_test_data_rest.py

@@ -361,31 +361,14 @@ class TMSSRESTTestDataCreator():
                 "cluster": cluster_url,
                 "scheduler_input_doc": "{}" }
     
-    def SubtaskOutput(self, subtask_url=None, subtask_connector_url=None):
+    def SubtaskOutput(self, subtask_url=None):
         if subtask_url is None:
             subtask_url = self.post_data_and_get_url(self.Subtask(), '/subtask/')
     
-        if subtask_connector_url is None:
-            subtask_connector_url = self.post_data_and_get_url(self.SubtaskConnector(), '/subtask_connector/')
-    
+
         return {"subtask": subtask_url,
-                "connector": subtask_connector_url,
                 "tags": []}
-    
-    def SubtaskConnector(self, input_of_url=None, output_of_url=None):
-        if input_of_url is None:
-            input_of_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/subtask_template/')
-    
-        if output_of_url is None:
-            output_of_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/subtask_template/')
-    
-        return {"role": self.django_api_url + '/role/correlator/',
-                "datatype": self.django_api_url + '/datatype/image/',
-                "dataformats": [self.django_api_url + '/dataformat/Beamformed/'],
-                "output_of": output_of_url,
-                "input_of": input_of_url,
-                "tags": []}
-    
+
     def Dataproduct(self, filename="my_filename", specifications_template_url=None, subtask_output_url=None, dataproduct_feedback_template_url=None):
         if specifications_template_url is None:
             specifications_template_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/dataproduct_specifications_template/')