Merge branch 'TMSS-221' into 'master'

Resolve TMSS-221

Closes TMSS-221

See merge request !186

Docker/lofar-ci/Dockerfile_ci_sas

@@ -26,4 +26,5 @@ RUN echo "Installing Nodejs packages..." && \
     node -v && \
     npm install -g serve
 
+
 USER lofarsys
\ No newline at end of file

SAS/TMSS/src/tmss/tmssapp/models/specification.py

@@ -8,10 +8,10 @@ from django.contrib.postgres.indexes import GinIndex
 from enum import Enum
 from django.db.models.expressions import RawSQL
 from django.db.models.deletion import ProtectedError
-
 from lofar.sas.tmss.tmss.tmssapp.validation import validate_json_against_schema
 from django.core.exceptions import ValidationError
 from rest_framework import status
+import datetime
 
 #
 # Common
@@ -348,6 +348,32 @@ class SchedulingUnitDraft(NamedCommon):
 
         super().save(force_insert, force_update, using, update_fields)
 
+    @property
+    def duration(self) -> datetime.timedelta:
+        '''return the overall duration of all tasks of this scheduling unit
+        '''
+        return self.relative_stop_time - self.relative_start_time
+
+    @property
+    def relative_start_time(self) -> datetime.timedelta:
+        '''return the earliest relative start time of all tasks of this scheduling unit
+        '''
+        task_drafts = list(self.task_drafts.all())
+        if task_drafts:
+            return min(task_drafts, key=lambda x: x.relative_start_time).relative_start_time
+        else:
+            return datetime.timedelta(seconds=0)
+
+    @property
+    def relative_stop_time(self) -> datetime.timedelta:
+        '''return the latest relative stop time of all tasks of this scheduling unit
+        '''
+        task_drafts = list(self.task_drafts.all())
+        if task_drafts:
+            return max(task_drafts, key=lambda x: x.relative_stop_time).relative_stop_time
+        else:
+            return datetime.timedelta(seconds=0)
+
 
 class SchedulingUnitBlueprint(NamedCommon):
     requirements_doc = JSONField(help_text='Scheduling and/or quality requirements for this scheduling unit (IMMUTABLE).')
@@ -361,6 +387,55 @@ class SchedulingUnitBlueprint(NamedCommon):
 
         super().save(force_insert, force_update, using, update_fields)
 
+    @property
+    def duration(self) -> datetime.timedelta:
+        '''return the overall duration of all tasks of this scheduling unit
+        '''
+        if self.start_time is None or self.stop_time is None:
+            return self.relative_stop_time - self.relative_start_time
+        else:
+            return self.stop_time - self.start_time   # <- todo: do we ever want this?
+
+    @property
+    def relative_start_time(self) -> datetime.timedelta:
+        '''return the earliest relative start time of all tasks of this scheduling unit
+        '''
+        task_blueprints = list(self.task_blueprints.all())
+        if task_blueprints:
+            return min(task_blueprints, key=lambda x: x.relative_start_time).relative_start_time
+        else:
+            return datetime.timedelta(seconds=0)
+
+    @property
+    def relative_stop_time(self) -> datetime.timedelta:
+        '''return the latest relative stop time of all tasks of this scheduling unit
+        '''
+        task_blueprints = list(self.task_blueprints.all())
+        if task_blueprints:
+            return max(task_blueprints, key=lambda x: x.relative_stop_time).relative_stop_time
+        else:
+            return datetime.timedelta(seconds=0)
+
+    @property
+    def start_time(self) -> datetime or None:
+        '''return the earliest start time of all tasks of this scheduling unit
+        '''
+        tasks_with_start_time = list(filter(lambda x: x.start_time is not None, self.task_blueprints.all()))
+        if tasks_with_start_time:
+            return min(tasks_with_start_time, key=lambda x: x.start_time).start_time
+        else:
+            return None
+
+    @property
+    def stop_time(self) -> datetime or None:
+        '''return the latest stop time of all tasks of this scheduling unit
+        '''
+        tasks_with_stop_time = list(filter(lambda x: x.stop_time is not None, self.task_blueprints.all()))
+        if tasks_with_stop_time:
+            return max(tasks_with_stop_time, key=lambda x: x.stop_time).stop_time
+        else:
+            return None
+
 
 class TaskDraft(NamedCommon):
     specifications_doc = JSONField(help_text='Specifications for this task.')
@@ -395,6 +470,88 @@ class TaskDraft(NamedCommon):
                                                       "INNER JOIN tmssapp_taskrelationdraft as task_rel on task_rel.producer_id = successor_task.id\n"
                                                       "WHERE task_rel.consumer_id = %s", params=[self.id]))
 
+    @property
+    def duration(self) -> datetime.timedelta:
+         '''returns the overall duration of this task
+         '''
+         return self.relative_stop_time - self.relative_start_time
+
+    @property
+    def relative_start_time(self) -> datetime.timedelta:
+        '''return the earliest relative start time of all subtasks of this task
+        '''
+        scheduling_relations = list(self.first_to_connect.all()) + list(self.second_to_connect.all())
+        for scheduling_relation in scheduling_relations:
+            if scheduling_relation.first.id == self.id and scheduling_relation.placement.value == "after":
+                previous_related_task_draft = TaskDraft.objects.get(id=scheduling_relation.second.id)
+                time_offset = scheduling_relation.time_offset
+                # todo: max of several relations
+                if previous_related_task_draft.relative_stop_time:
+                    return previous_related_task_draft.relative_stop_time + datetime.timedelta(seconds=time_offset)
+
+            if scheduling_relation.second.id == self.id and scheduling_relation.placement.value == "before":
+                previous_related_task_draft = TaskDraft.objects.get(id=scheduling_relation.first.id)
+                time_offset = scheduling_relation.time_offset
+                # todo: max of several relations
+                if previous_related_task_draft.relative_stop_time:
+                    return previous_related_task_draft.relative_stop_time + datetime.timedelta(seconds=time_offset)
+        return datetime.timedelta(seconds=0)
+
+    @property
+    def relative_stop_time(self) -> datetime.timedelta:
+        '''return the latest relative stop time of all subtasks of this task
+        '''
+        # todo: when it was added, check if self.specifications_template.type.value == TaskType.Choices.OBSERVATION.value:
+        try:
+            duration = self.specifications_doc["duration"]
+            return self.relative_start_time + datetime.timedelta(seconds=duration)
+        except:
+            pass
+        return self.relative_start_time
+
+
+    # JK, 28/07/20: After discussion with Sander, we probably only want the
+    #  - duration on the scheduling_unit draft (based on relative start/stop times)
+    #  - duration plus relative start/stop on the task draft.
+    #  This provides an estimate of what is currently planned out in the draft, but does not confuse with timestamps of actual start/stop of the blueprints.
+    #  Only on the blueprints, we also aggregate start_stop times as they are in the system
+    #  I'll leave these code bits here for now, until we made up our minds about this, but this can probably be removed
+    #
+    # @property
+    # def duration(self) -> datetime.timedelta:
+    #     '''returns the overall duration in seconds of all blueprints of this task
+    #     # todo: is this the wanted behavior? Do you want to consider all the blueprints created from your draft or do you want to preview a new blueprint?
+    #     '''
+    #     if self.start_time is None or self.stop_time is None:
+    #         # todo: calculate?
+    #         return None
+    #     else:
+    #         return self.stop_time - self.start_time
+    #
+    # @property
+    # def start_time(self) -> datetime or None:
+    #     '''return the earliest start time of all blueprints of this task
+    #     # todo: is this the wanted behavior? Do you want to consider all the blueprints created from your draft or do you want to preview a new blueprint?
+    #     '''
+    #     blueprints_with_start_time = list(filter(lambda x: x.start_time is not None, self.task_blueprints.all()))
+    #     if blueprints_with_start_time:
+    #         return min(blueprints_with_start_time, key=lambda x: x.start_time).start_time
+    #     else:
+    #         # todo: calculate?
+    #         return None
+    #
+    # @property
+    # def stop_time(self) -> datetime or None:
+    #     '''return the latest stop time of all blueprints of this task
+    #     # todo: is this the wanted behavior? Do you want to consider all the blueprints created from your draft or do you want to preview a new blueprint?
+    #     '''
+    #     blueprints_with_stop_time = list(filter(lambda x: x.stop_time is not None, self.task_blueprints.all()))
+    #     if blueprints_with_stop_time:
+    #         return max(blueprints_with_stop_time, key=lambda x: x.stop_time).stop_time
+    #     else:
+    #         # todo: calculate?
+    #         return None
+
 
 class TaskBlueprint(NamedCommon):
     specifications_doc = JSONField(help_text='Schedulings for this task (IMMUTABLE).')
@@ -429,6 +586,68 @@ class TaskBlueprint(NamedCommon):
                                                           "INNER JOIN tmssapp_taskrelationblueprint as task_rel on task_rel.producer_id = predecessor_task.id\n"
                                                           "WHERE task_rel.consumer_id = %s", params=[self.id]))
 
+    @property
+    def duration(self) -> datetime.timedelta:
+        '''return the overall duration of this task
+        '''
+        if self.start_time is None or self.stop_time is None:
+            return self.relative_stop_time - self.relative_start_time
+        else:
+            return self.stop_time - self.start_time
+
+    @property
+    def relative_start_time(self) -> datetime.timedelta:
+        '''return the earliest relative start time of all subtasks of this task
+        '''
+        scheduling_relations = list(self.first_to_connect.all()) + list(self.second_to_connect.all())
+        for scheduling_relation in scheduling_relations:
+                if scheduling_relation.first.id == self.id and scheduling_relation.placement.value == "after":
+                    previous_related_task_blueprint = TaskBlueprint.objects.get(id=scheduling_relation.second.id)
+                    time_offset = scheduling_relation.time_offset
+                    # todo: max of several relations
+                    if previous_related_task_blueprint.relative_stop_time:
+                        return previous_related_task_blueprint.relative_stop_time + datetime.timedelta(seconds=time_offset)
+
+                if scheduling_relation.second.id == self.id and scheduling_relation.placement.value == "before":
+                    previous_related_task_blueprint = TaskBlueprint.objects.get(id=scheduling_relation.first.id)
+                    time_offset = scheduling_relation.time_offset
+                    # todo: max of several relations
+                    if previous_related_task_blueprint.relative_stop_time:
+                        return previous_related_task_blueprint.relative_stop_time + datetime.timedelta(seconds=time_offset)
+        return datetime.timedelta(seconds=666660)
+
+    @property
+    def relative_stop_time(self) -> datetime.timedelta:
+        '''return the latest relative stop time of all subtasks of this task
+        '''
+        # todo: when it was added, check if subtask.specifications_template.type.value == TaskType.Choices.OBSERVATION.value:
+        try:
+            duration = self.specifications_doc["duration"]
+            return self.relative_start_time + datetime.timedelta(seconds=duration)
+        except:
+            pass
+        return self.relative_start_time
+
+    @property
+    def start_time(self) -> datetime or None:
+        '''return the earliest start time of all subtasks of this task
+        '''
+        subtasks_with_start_time = list(filter(lambda x: x.start_time is not None, self.subtasks.all()))
+        if subtasks_with_start_time:
+            return min(subtasks_with_start_time, key=lambda x: x.start_time).start_time
+        else:
+            return None
+
+    @property
+    def stop_time(self) -> datetime or None:
+        '''return the latest stop time of all subtasks of this task
+        '''
+        subtasks_with_stop_time = list(filter(lambda x: x.stop_time is not None, self.subtasks.all()))
+        if subtasks_with_stop_time:
+            return max(subtasks_with_stop_time, key=lambda x: x.stop_time).stop_time
+        else:
+            return None
+
 
 class TaskRelationDraft(BasicCommon):
     selection_doc = JSONField(help_text='Filter for selecting dataproducts from the output role.')

SAS/TMSS/src/tmss/tmssapp/serializers/specification.py

@@ -66,6 +66,14 @@ class RelationalHyperlinkedModelSerializer(serializers.HyperlinkedModelSerialize
                                                     'read_only':True}
 
 
+class FloatDurationField(serializers.FloatField):
+
+    # Turn datetime to float representation in seconds.
+    # (Timedeltas are otherwise by default turned into a string representation)
+    def to_representation(self, value):
+        return value.total_seconds()
+
+
 # This is required for keeping a user reference as ForeignKey in other models
 # (I think so that the HyperlinkedModelSerializer can generate a URI)
 class UserSerializer(serializers.Serializer):
@@ -236,6 +244,8 @@ class SchedulingSetSerializer(RelationalHyperlinkedModelSerializer):
 
 class SchedulingUnitDraftSerializer(RelationalHyperlinkedModelSerializer):
 
+    duration = FloatDurationField(required=False)
+
     # Create a JSON editor form to replace the simple text field based on the schema in the template that this
     # draft refers to. If that fails, the JSONField remains a standard text input.
     def __init__(self, *args, **kwargs):
@@ -248,11 +258,13 @@ class SchedulingUnitDraftSerializer(RelationalHyperlinkedModelSerializer):
     class Meta:
         model = models.SchedulingUnitDraft
         fields = '__all__'
-        extra_fields = ['scheduling_unit_blueprints', 'task_drafts']
+        extra_fields = ['scheduling_unit_blueprints', 'task_drafts', 'duration']
 
 
 class SchedulingUnitBlueprintSerializer(RelationalHyperlinkedModelSerializer):
 
+    duration = FloatDurationField(required=False)
+
     # Create a JSON editor form to replace the simple text field based on the schema in the template that this
     # draft refers to. If that fails, the JSONField remains a standard text input.
     def __init__(self, *args, **kwargs):
@@ -265,11 +277,15 @@ class SchedulingUnitBlueprintSerializer(RelationalHyperlinkedModelSerializer):
     class Meta:
         model = models.SchedulingUnitBlueprint
         fields = '__all__'
-        extra_fields = ['task_blueprints']
+        extra_fields = ['task_blueprints', 'duration', 'start_time', 'stop_time']
 
 
 class TaskDraftSerializer(RelationalHyperlinkedModelSerializer):
 
+    duration = FloatDurationField(required=False)
+    relative_start_time = FloatDurationField(required=False)
+    relative_stop_time = FloatDurationField(required=False)
+
     # Create a JSON editor form to replace the simple text field based on the schema in the template that this
     # draft refers to. If that fails, the JSONField remains a standard text input.
     def __init__(self, *args, **kwargs):
@@ -282,11 +298,15 @@ class TaskDraftSerializer(RelationalHyperlinkedModelSerializer):
     class Meta:
         model = models.TaskDraft
         fields = '__all__'
-        extra_fields = ['task_blueprints', 'produced_by', 'consumed_by', 'first_to_connect', 'second_to_connect']
+        extra_fields = ['task_blueprints', 'produced_by', 'consumed_by', 'first_to_connect', 'second_to_connect', 'duration', 'relative_start_time', 'relative_stop_time']
 
 
 class TaskBlueprintSerializer(RelationalHyperlinkedModelSerializer):
 
+    duration = FloatDurationField(required=False)
+    relative_start_time = FloatDurationField(required=False)
+    relative_stop_time = FloatDurationField(required=False)
+
     # Create a JSON editor form to replace the simple text field based on the schema in the template that this
     # draft refers to. If that fails, the JSONField remains a standard text input.
     def __init__(self, *args, **kwargs):
@@ -299,7 +319,7 @@ class TaskBlueprintSerializer(RelationalHyperlinkedModelSerializer):
     class Meta:
         model = models.TaskBlueprint
         fields = '__all__'
-        extra_fields = ['subtasks', 'produced_by', 'consumed_by', 'first_to_connect', 'second_to_connect']
+        extra_fields = ['subtasks', 'produced_by', 'consumed_by', 'first_to_connect', 'second_to_connect', 'duration', 'start_time', 'stop_time', 'relative_start_time', 'relative_stop_time']
 
 
 class TaskRelationDraftSerializer(RelationalHyperlinkedModelSerializer):

SAS/TMSS/src/tmss/tmssapp/subtasks.py

@@ -577,7 +577,7 @@ def schedule_qaplots_subtask(qaplots_subtask: Subtask):
 
     return qaplots_subtask
 
-
+# todo: this can probably go when we switch to the new start time calculation in the model properties (which is based on this logic)
 def get_previous_related_task_blueprint_with_time_offset(task_blueprint):
     """
     Retrieve the the previous related blueprint task object (if any)
@@ -602,7 +602,7 @@ def get_previous_related_task_blueprint_with_time_offset(task_blueprint):
 
     return previous_related_task_blueprint, time_offset
 
-
+# todo: maybe this can now be replaced by subtask.relative_start_time
 def calculate_start_time(observation_subtask: Subtask):
     """
     Calculate the start time of an observation subtask. It should calculate the starttime in case of 'C-T-C train'

SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py

@@ -165,6 +165,16 @@ class ProjectViewSet(LOFARViewSet):
     serializer_class = serializers.ProjectSerializer
     ordering = ['name']
 
+    def get_queryset(self):
+        queryset = models.Project.objects.all()
+
+        # query by cycle
+        cycle = self.request.query_params.get('cycle', None)
+        if cycle is not None:
+            return queryset.filter(cycles__name=cycle)
+
+        return queryset
+
 
 class ProjectNestedViewSet(LOFARNestedViewSet):
     queryset = models.Project.objects.all()

SAS/TMSS/src/tmss/urls.py

@@ -119,7 +119,7 @@ router.register(r'task_scheduling_relation_draft', viewsets.TaskSchedulingRelati
 router.register(r'task_scheduling_relation_blueprint', viewsets.TaskSchedulingRelationBlueprintViewSet)
 
 # nested
-router.register(r'cycle/(?P<cycle_id>[\w\-]+)/project', viewsets.ProjectNestedViewSet)
+router.register(r'cycle/(?P<cycle_id>[\w\- ]+)/project', viewsets.ProjectNestedViewSet)
 router.register(r'scheduling_set/(?P<scheduling_set_id>\d+)/scheduling_unit_draft', viewsets.SchedulingUnitDraftNestedViewSet)
 router.register(r'scheduling_unit_draft/(?P<scheduling_unit_draft_id>\d+)/scheduling_unit_blueprint', viewsets.SchedulingUnitBlueprintNestedViewSet)
 router.register(r'scheduling_unit_draft/(?P<scheduling_unit_draft_id>\d+)/task_draft', viewsets.TaskDraftNestedViewSet)