From 24a6726b39c92873f08c217dc1c9ed769cc529b3 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Mon, 12 Feb 2024 13:50:50 +0100
Subject: [PATCH] switched off new functionality for database deployment only
---
atdb/taskdatabase/models.py | 212 +++++++++---------
atdb/taskdatabase/services/activities.py | 9 +-
atdb/taskdatabase/services/signals.py | 3 +-
.../taskdatabase/failures/tasks.html | 7 +-
.../templates/taskdatabase/ingest/tasks.html | 6 +-
.../taskdatabase/validation/tasks.html | 6 +-
.../validation/validation_buttons.html | 5 +-
7 files changed, 132 insertions(+), 116 deletions(-)
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 042a31db..094a3f02 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -204,7 +204,8 @@ class Task(models.Model):
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
# make sure that every task has an activity (backward compatibility)
- associate_task_with_activity(self)
+ # TODO: uncomment to enable SDC-1188 functionality
+ #associate_task_with_activity(self)
# remark:
# a post_save signal is triggered by this save()
@@ -340,21 +341,24 @@ class Task(models.Model):
except:
return None
- # keep the old mechanism in comments to test/evaluate, remove later when it works
- # @property
- # def sas_id_has_archived(self):
- # """
- # check if any task belonging to this sas_id already has an output SAS_ID at the LTA
- # """
- # try:
- # for task in Task.objects.filter(sas_id=self.sas_id):
- # try:
- # if task.archive['sas_id_archived']:
- # return task.archive['sas_id_archived']
- # except:
- # pass
- # except:
- # return None
+ # keep the old mechanism in comments to test/evaluate
+ # TODO: remove when it is no longer used by the GUI
+ # --- <CUT> ---
+ @property
+ def sas_id_has_archived(self):
+ """
+ check if any task belonging to this sas_id already has an output SAS_ID at the LTA
+ """
+ try:
+ for task in Task.objects.filter(sas_id=self.sas_id):
+ try:
+ if task.archive['sas_id_archived']:
+ return task.archive['sas_id_archived']
+ except:
+ pass
+ except:
+ return None
+ # --- </CUT> ---
@property
def path_to_lta(self):
@@ -366,93 +370,97 @@ class Task(models.Model):
except:
return None
-# keep the old mechanism in comments to test/evaluate, remove later when it works
- # @property
- # def sasid_path_to_lta(self):
- # """
- # check if any task belonging to this sas_id already has a 'path_to_lta' setting
- # """
- # try:
- # for task in Task.objects.filter(sas_id=self.sas_id):
- # try:
- # if task.archive['path_to_lta']:
- # return task.archive['path_to_lta']
- # except:
- # # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
- # pass
- # except:
- # return None
+ # keep the old mechanism in comments to test/evaluate, remove later when it works
+ # TODO: remove when it is no longer used by the GUI
+ # ---- <CUT> ----
+ @property
+ def sasid_path_to_lta(self):
+ """
+ check if any task belonging to this sas_id already has a 'path_to_lta' setting
+ """
+ try:
+ for task in Task.objects.filter(sas_id=self.sas_id):
+ try:
+ if task.archive['path_to_lta']:
+ return task.archive['path_to_lta']
+ except:
+ # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
+ pass
+ except:
+ return None
- #
- # @property
- # def sasid_is_verified(self):
- # for task in Task.objects.filter(sas_id=self.sas_id):
- # if task.status not in verified_statusses:
- # return False
- # return True
- #
- # @property
- # def sasid_finished_fraction(self):
- # size_archived = 0
- # size_remaining = 0
- # total_size = 0
- #
- # tasks = Task.objects.filter(sas_id=self.sas_id)
- #
- # for task in tasks:
- # if task.status == State.FINISHED.value:
- # size_archived = size_archived + task.size_to_process
- # else:
- # size_remaining = size_remaining + task.size_to_process
- # total_size = total_size + task.size_to_process
- #
- # finished = {}
- # try:
- # finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
- # except:
- # finished['fraction'] = -1
- #
- # finished['total_size'] = total_size
- # finished['remaining'] = size_remaining
- #
- # return finished
- #
- # @property
- # def sasid_ingested_fraction(self):
- # """
- # This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
- # and a list of statusses of other tasks belonging to the same SAS_ID.
- # It is implemented as 'property', because then it can be used in html pages like this:
- # <td>{{ task.sasid_ingested_fraction.status }}</td>
- # <td>{{ task.sasid_ingested_fraction.completion }}%</td>
- #
- # A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
- # The division of those 2 are returned as a 'completed %'.
- # A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
- #
- # """
- # result = {}
- # statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
- # 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
- #
- # tasks = Task.objects.filter(sas_id=self.sas_id)
- #
- # for task in tasks:
- # try:
- # statusses[task.status] += 1
- # except:
- # pass
- #
- # incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
- # int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
- # complete = int(statusses['archived']) + int(statusses['finished'])
- # completion = round(complete / (incomplete + complete) * 100)
- #
- # non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
- #
- # result['status'] = non_zero_statusses
- # result['completion'] = completion
- # return result
+
+ @property
+ def sasid_is_verified(self):
+ for task in Task.objects.filter(sas_id=self.sas_id):
+ if task.status not in verified_statusses:
+ return False
+ return True
+
+ @property
+ def sasid_finished_fraction(self):
+ size_archived = 0
+ size_remaining = 0
+ total_size = 0
+
+ tasks = Task.objects.filter(sas_id=self.sas_id)
+
+ for task in tasks:
+ if task.status == State.FINISHED.value:
+ size_archived = size_archived + task.size_to_process
+ else:
+ size_remaining = size_remaining + task.size_to_process
+ total_size = total_size + task.size_to_process
+
+ finished = {}
+ try:
+ finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
+ except:
+ finished['fraction'] = -1
+
+ finished['total_size'] = total_size
+ finished['remaining'] = size_remaining
+
+ return finished
+
+ @property
+ def sasid_ingested_fraction(self):
+ """
+ This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
+ and a list of statusses of other tasks belonging to the same SAS_ID.
+ It is implemented as 'property', because then it can be used in html pages like this:
+ <td>{{ task.sasid_ingested_fraction.status }}</td>
+ <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+
+ A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
+ The division of those 2 are returned as a 'completed %'.
+ A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
+
+ """
+ result = {}
+ statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
+ 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
+
+ tasks = Task.objects.filter(sas_id=self.sas_id)
+
+ for task in tasks:
+ try:
+ statusses[task.status] += 1
+ except:
+ pass
+
+ incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
+ int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
+ complete = int(statusses['archived']) + int(statusses['finished'])
+ completion = round(complete / (incomplete + complete) * 100)
+
+ non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
+
+ result['status'] = non_zero_statusses
+ result['completion'] = completion
+ return result
+
+ # ---- </CUT> ----
@property
def task_type_join(self):
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index 1ba2771d..acc36aa4 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -68,15 +68,16 @@ def calculate_finished_fraction(this_task):
def update_activity(task):
"""
- The activity (SAS_ID level) is updated whenever a task change status.
+ The activity (SAS_ID level) is updated when a task changes status.
Depending on the type of status change, certain calculations and updates are performed.
- Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
+ Doing this on status change, instead of on-the-fly when a user enters a page, balances the load
+ and improves overall performance
- to 'ARCHIVING, ARCHIVED, FINISHED' : check for incoming/existing 'archive' json from archiver
- to STORED : calculate quality
- - to _FAILED : calculate finished_fraction
- to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
- - always : calc 'verified'
+ - to _FAILED : calculate finished_fraction
+ - always : determine if a task is in a 'verified' status
"""
logger.info(f'update_activity for task {task.id} with sas_id {task.sas_id} and status {task.status}')
diff --git a/atdb/taskdatabase/services/signals.py b/atdb/taskdatabase/services/signals.py
index 146c313c..0bf04ac4 100644
--- a/atdb/taskdatabase/services/signals.py
+++ b/atdb/taskdatabase/services/signals.py
@@ -69,7 +69,8 @@ def handle_post_save(sender, **kwargs):
"""
task = kwargs.get('instance')
- update_activity(task)
+ # TODO: uncomment to enable SDC-1188 functionality
+ # update_activity(task)
def connect_signals():
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html b/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
index 988f1a37..21ba2ddb 100644
--- a/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
@@ -37,13 +37,14 @@
<td>{{ task.filter }} </td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
<td>{{ task.sasid_finished_fraction.fraction }}% of {{ task.sasid_finished_fraction.total_size|filesizeformat }}</td>
<td>{{ task.size_to_process|filesizeformat }} / {{ task.sasid_finished_fraction.remaining|filesizeformat }}</td>
--->
+
+<!-- new activity mechanism, to enable SDC-1188
<td>{{ task.activity.finished_fraction }}% of {{ task.activity.total_size|filesizeformat }}</td>
<td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td>
-
+-->
<td>
{% include "taskdatabase/failures/retry_buttons.html" %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
index 0b250e52..79a13d9f 100644
--- a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
@@ -25,7 +25,7 @@
</td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
<td>{{ task.sasid_ingested_fraction.status }}</td>
<td>{{ task.sasid_ingested_fraction.completion }}%</td>
@@ -39,7 +39,8 @@
-
{% endif %}
</td>
--->
+
+<!-- new activity mechanism, to enable SDC-1188
<td>{{ task.activity.ingestq_status }}</td>
<td>{{ task.activity.ingested_fraction }}%</td>
<td>
@@ -52,6 +53,7 @@
-
{% endif %}
</td>
+-->
</tr>
</div>
{% endif %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
index e5282701..045ee19d 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
@@ -70,10 +70,12 @@
</td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
<td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }} (old)</td>
--->
+
+<!-- new activity mechanism, to enable SDC-1188
<td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td>
+-->
<td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td>
<td>{% include "taskdatabase/validation/validation_buttons.html" %}</td>
<td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td>
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
index b900a6be..e5f633f7 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
@@ -1,5 +1,5 @@
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
-<!--
+
{% if task.sasid_is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
{% endif %}
@@ -15,8 +15,8 @@
{% if task.sasid_is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
{% endif %}
--->
+<!-- new activity mechanism, to enable SDC-1188
{% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
{% endif %}
@@ -32,3 +32,4 @@
{% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
{% endif %}
+-->
\ No newline at end of file
--
GitLab