switched off new functionality for database deployment only

atdb/taskdatabase/models.py

@@ -204,7 +204,8 @@ class Task(models.Model):
             self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
 
         # make sure that every task has an activity (backward compatibility)
-        associate_task_with_activity(self)
+        # TODO: uncomment to enable SDC-1188 functionality
+        #associate_task_with_activity(self)
 
         # remark:
         # a post_save signal is triggered by this save()
@@ -340,21 +341,24 @@ class Task(models.Model):
         except:
             return None
 
-    # keep the old mechanism in comments to test/evaluate, remove later when it works
-    # @property
-    # def sas_id_has_archived(self):
-    #     """
-    #     check if any task belonging to this sas_id already has an output SAS_ID at the LTA
-    #     """
-    #     try:
-    #         for task in Task.objects.filter(sas_id=self.sas_id):
-    #             try:
-    #                 if task.archive['sas_id_archived']:
-    #                     return task.archive['sas_id_archived']
-    #             except:
-    #                 pass
-    #     except:
-    #         return None
+    # keep the old mechanism in comments to test/evaluate
+    # TODO: remove when it is no longer used by the GUI
+    # ---  <CUT> ---
+    @property
+    def sas_id_has_archived(self):
+        """
+        check if any task belonging to this sas_id already has an output SAS_ID at the LTA
+        """
+        try:
+            for task in Task.objects.filter(sas_id=self.sas_id):
+                try:
+                    if task.archive['sas_id_archived']:
+                        return task.archive['sas_id_archived']
+                except:
+                    pass
+        except:
+            return None
+    # --- </CUT> ---
 
     @property
     def path_to_lta(self):
@@ -367,92 +371,96 @@ class Task(models.Model):
             return None
 
     # keep the old mechanism in comments to test/evaluate, remove later when it works
-    # @property
-    # def sasid_path_to_lta(self):
-    #     """
-    #     check if any task belonging to this sas_id already has a 'path_to_lta' setting
-    #     """
-    #     try:
-    #         for task in Task.objects.filter(sas_id=self.sas_id):
-    #             try:
-    #                 if task.archive['path_to_lta']:
-    #                     return task.archive['path_to_lta']
-    #             except:
-    #                 # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
-    #                 pass
-    #     except:
-    #         return None
+    # TODO: remove when it is no longer used by the GUI
+    # ----  <CUT>  ----
+    @property
+    def sasid_path_to_lta(self):
+        """
+        check if any task belonging to this sas_id already has a 'path_to_lta' setting
+        """
+        try:
+            for task in Task.objects.filter(sas_id=self.sas_id):
+                try:
+                    if task.archive['path_to_lta']:
+                        return task.archive['path_to_lta']
+                except:
+                    # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
+                    pass
+        except:
+            return None
 
-    #
-    # @property
-    # def sasid_is_verified(self):
-    #     for task in Task.objects.filter(sas_id=self.sas_id):
-    #        if task.status not in verified_statusses:
-    #            return False
-    #     return True
-    #
-    # @property
-    # def sasid_finished_fraction(self):
-    #     size_archived = 0
-    #     size_remaining = 0
-    #     total_size = 0
-    #
-    #     tasks = Task.objects.filter(sas_id=self.sas_id)
-    #
-    #     for task in tasks:
-    #        if task.status == State.FINISHED.value:
-    #            size_archived = size_archived + task.size_to_process
-    #        else:
-    #            size_remaining = size_remaining + task.size_to_process
-    #        total_size = total_size + task.size_to_process
-    #
-    #     finished = {}
-    #     try:
-    #         finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
-    #     except:
-    #         finished['fraction'] = -1
-    #
-    #     finished['total_size'] = total_size
-    #     finished['remaining'] = size_remaining
-    #
-    #     return finished
-    #
-    # @property
-    # def sasid_ingested_fraction(self):
-    #     """
-    #     This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
-    #     and a list of statusses of other tasks belonging to the same SAS_ID.
-    #     It is implemented as 'property', because then it can be used in html pages like this:
-    #        <td>{{ task.sasid_ingested_fraction.status }}</td>
-    #        <td>{{ task.sasid_ingested_fraction.completion }}%</td>
-    #
-    #     A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
-    #     The division of those 2 are returned as a 'completed %'.
-    #     A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
-    #
-    #     """
-    #     result = {}
-    #     statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
-    #                  'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
-    #
-    #     tasks = Task.objects.filter(sas_id=self.sas_id)
-    #
-    #     for task in tasks:
-    #         try:
-    #             statusses[task.status] += 1
-    #         except:
-    #             pass
-    #
-    #     incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
-    #                  int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
-    #     complete = int(statusses['archived']) + int(statusses['finished'])
-    #     completion = round(complete / (incomplete + complete) * 100)
-    #
-    #     non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
-    #
-    #     result['status'] = non_zero_statusses
-    #     result['completion'] = completion
-    #     return result
+
+    @property
+    def sasid_is_verified(self):
+        for task in Task.objects.filter(sas_id=self.sas_id):
+           if task.status not in verified_statusses:
+               return False
+        return True
+
+    @property
+    def sasid_finished_fraction(self):
+        size_archived = 0
+        size_remaining = 0
+        total_size = 0
+
+        tasks = Task.objects.filter(sas_id=self.sas_id)
+
+        for task in tasks:
+           if task.status == State.FINISHED.value:
+               size_archived = size_archived + task.size_to_process
+           else:
+               size_remaining = size_remaining + task.size_to_process
+           total_size = total_size + task.size_to_process
+
+        finished = {}
+        try:
+            finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
+        except:
+            finished['fraction'] = -1
+
+        finished['total_size'] = total_size
+        finished['remaining'] = size_remaining
+
+        return finished
+
+    @property
+    def sasid_ingested_fraction(self):
+        """
+        This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
+        and a list of statusses of other tasks belonging to the same SAS_ID.
+        It is implemented as 'property', because then it can be used in html pages like this:
+           <td>{{ task.sasid_ingested_fraction.status }}</td>
+           <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+
+        A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
+        The division of those 2 are returned as a 'completed %'.
+        A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
+
+        """
+        result = {}
+        statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
+                     'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
+
+        tasks = Task.objects.filter(sas_id=self.sas_id)
+
+        for task in tasks:
+            try:
+                statusses[task.status] += 1
+            except:
+                pass
+
+        incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
+                     int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
+        complete = int(statusses['archived']) + int(statusses['finished'])
+        completion = round(complete / (incomplete + complete) * 100)
+
+        non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
+
+        result['status'] = non_zero_statusses
+        result['completion'] = completion
+        return result
+
+    # ----  </CUT>  ----
 
     @property
     def task_type_join(self):

atdb/taskdatabase/services/activities.py

@@ -68,15 +68,16 @@ def calculate_finished_fraction(this_task):
 
 def update_activity(task):
     """
-    The activity (SAS_ID level) is updated whenever a task change status.
+    The activity (SAS_ID level) is updated when a task changes status.
     Depending on the type of status change, certain calculations and updates are performed.
-    Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
+    Doing this on status change, instead of on-the-fly when a user enters a page, balances the load
+    and improves overall performance
 
     - to 'ARCHIVING, ARCHIVED, FINISHED'         : check for incoming/existing 'archive' json from archiver
     - to STORED                                  : calculate quality
-    - to _FAILED                                 : calculate finished_fraction
     - to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
-    - always                                     : calc 'verified'
+    - to _FAILED                                 : calculate finished_fraction
+    - always                                     : determine if a task is in a 'verified' status
 
     """
     logger.info(f'update_activity for task {task.id} with sas_id {task.sas_id} and status {task.status}')

atdb/taskdatabase/services/signals.py

@@ -69,7 +69,8 @@ def handle_post_save(sender, **kwargs):
     """
     task = kwargs.get('instance')
 
-    update_activity(task)
+    # TODO: uncomment to enable SDC-1188 functionality
+    # update_activity(task)
 
 
 def connect_signals():

atdb/taskdatabase/templates/taskdatabase/failures/tasks.html

@@ -37,13 +37,14 @@
 
                 <td>{{ task.filter }} </td>
 <!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
                 <td>{{ task.sasid_finished_fraction.fraction }}% of {{ task.sasid_finished_fraction.total_size|filesizeformat }}</td>
                 <td>{{ task.size_to_process|filesizeformat }} / {{ task.sasid_finished_fraction.remaining|filesizeformat }}</td>
--->
+
+<!-- new activity mechanism, to enable SDC-1188
                 <td>{{ task.activity.finished_fraction }}% of {{ task.activity.total_size|filesizeformat }}</td>
                 <td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td>
-
+-->
 
                 <td>
                     {% include "taskdatabase/failures/retry_buttons.html" %}

atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html

@@ -25,7 +25,7 @@
                 </td>
 
 <!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
                 <td>{{ task.sasid_ingested_fraction.status }}</td>
 
                 <td>{{ task.sasid_ingested_fraction.completion }}%</td>
@@ -39,7 +39,8 @@
                     -
                     {% endif %}
                 </td>
--->
+
+<!-- new activity mechanism, to enable SDC-1188
                 <td>{{ task.activity.ingestq_status }}</td>
                 <td>{{ task.activity.ingested_fraction }}%</td>
                 <td>
@@ -52,6 +53,7 @@
                     -
                     {% endif %}
                 </td>
+-->
            </tr>
         </div>
         {% endif %}

atdb/taskdatabase/templates/taskdatabase/validation/tasks.html

@@ -70,10 +70,12 @@
                 </td>
 
 <!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
                 <td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }} (old)</td>
--->
+
+<!-- new activity mechanism, to enable SDC-1188
                 <td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td>
+-->
                 <td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td>
                 <td>{% include "taskdatabase/validation/validation_buttons.html" %}</td>
                 <td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td>

atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html

 <!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
  {% if task.sasid_is_verified %}
     <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
  {% endif %}
@@ -15,8 +15,8 @@
  {% if task.sasid_is_verified %}
     <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
  {% endif %}
--->
 
+<!-- new activity mechanism, to enable SDC-1188
  {% if task.activity.is_verified %}
     <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
  {% endif %}
@@ -32,3 +32,4 @@
  {% if task.activity.is_verified %}
     <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
  {% endif %}
+-->
\ No newline at end of file