diff --git a/atdb/taskdatabase/migrations/0037_remove_activity_has_summary.py b/atdb/taskdatabase/migrations/0037_remove_activity_has_summary.py
new file mode 100644
index 0000000000000000000000000000000000000000..213fd3b13c4f3909346e93e44a97d89452b0e1c2
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0037_remove_activity_has_summary.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.0 on 2024-02-09 09:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0036_remove_activity_output_sas_id_activity_remaining_and_more'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='activity',
+ name='has_summary',
+ ),
+ ]
diff --git a/atdb/taskdatabase/migrations/0038_remove_activity_priority_remove_activity_workflow_and_more.py b/atdb/taskdatabase/migrations/0038_remove_activity_priority_remove_activity_workflow_and_more.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cfac571a3d5032123d390a48b921d0237c6c1b1
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0038_remove_activity_priority_remove_activity_workflow_and_more.py
@@ -0,0 +1,26 @@
+# Generated by Django 5.0 on 2024-02-09 12:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0037_remove_activity_has_summary'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='activity',
+ name='priority',
+ ),
+ migrations.RemoveField(
+ model_name='activity',
+ name='workflow',
+ ),
+ migrations.AddField(
+ model_name='activity',
+ name='workflow_id',
+ field=models.IntegerField(null=True),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 955e2e79fb86d050251e4af16bf4e336a7804255..5b40068b10024192a3ba5306e0409aab03c100a7 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -12,7 +12,6 @@ from .services.common import State, verified_statusses
logger = logging.getLogger(__name__)
# constants
-
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
@@ -80,20 +79,14 @@ def convert_summary_to_list_for_template(task):
class Activity(models.Model):
"""
- update when tasks status changes from ??? to ???:
- - to '???' : check for incoming 'archive' json from archiver
- - to 'stored' : calculate qualities
- - to ??? : calculate finished_fraction
- - to ??? : calculate ingested_fraction
-
+ Aggregation per SAS_ID level
"""
- # fields used in overview pages
sas_id = models.CharField(db_index=True, verbose_name="SAS_ID", max_length=15, blank=True, null=True)
- workflow = models.ForeignKey(Workflow, related_name='activities', on_delete=models.CASCADE, null=True, blank=True)
+ #workflow = models.ForeignKey(Workflow, related_name='activities', on_delete=models.DO_NOTHING, null=True, blank=True)
+ workflow_id = models.IntegerField(null=True)
project = models.CharField(max_length=100, blank=True, null=True, default="unknown")
filter = models.CharField(max_length=30, blank=True, null=True)
- priority = models.IntegerField(default=100, null=True)
status = models.CharField(db_index=True, default="unknown", max_length=50, blank=True, null=True)
calculated_quality = models.CharField(max_length=10, blank=True, null=True)
@@ -101,8 +94,6 @@ class Activity(models.Model):
# this is the JSON blob that is filled in by the ldv_archiver during the ingest process
archive = models.JSONField(null=True, blank=True)
- has_summary = models.BooleanField(default=False)
-
is_verified = models.BooleanField(default=False)
finished_fraction = models.FloatField(blank=True, null=True)
ingested_fraction = models.FloatField(blank=True, null=True)
@@ -123,6 +114,9 @@ class Activity(models.Model):
except:
return None
+ def __str__(self):
+ return str(self.sas_id)
+
class Task(models.Model):
@@ -190,6 +184,10 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
+ # remark:
+ # a post_save signal is triggered by this save()
+ # to update the associated 'activity' with relevant aggregated information
+
super(Task, self).save(*args, **kwargs)
@@ -320,20 +318,21 @@ class Task(models.Model):
except:
return None
- @property
- def sas_id_has_archived(self):
- """
- check if any task belonging to this sas_id already has an output SAS_ID at the LTA
- """
- try:
- for task in Task.objects.filter(sas_id=self.sas_id):
- try:
- if task.archive['sas_id_archived']:
- return task.archive['sas_id_archived']
- except:
- pass
- except:
- return None
+ # keep the old mechanism in comments to test/evaluate, remove later when it works
+ # @property
+ # def sas_id_has_archived(self):
+ # """
+ # check if any task belonging to this sas_id already has an output SAS_ID at the LTA
+ # """
+ # try:
+ # for task in Task.objects.filter(sas_id=self.sas_id):
+ # try:
+ # if task.archive['sas_id_archived']:
+ # return task.archive['sas_id_archived']
+ # except:
+ # pass
+ # except:
+ # return None
@property
def path_to_lta(self):
@@ -345,92 +344,93 @@ class Task(models.Model):
except:
return None
- @property
- def sasid_path_to_lta(self):
- """
- check if any task belonging to this sas_id already has a 'path_to_lta' setting
- """
- try:
- for task in Task.objects.filter(sas_id=self.sas_id):
- try:
- if task.archive['path_to_lta']:
- return task.archive['path_to_lta']
- except:
- # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
- pass
- except:
- return None
-
-
- @property
- def sasid_is_verified(self):
- for task in Task.objects.filter(sas_id=self.sas_id):
- if task.status not in verified_statusses:
- return False
- return True
-
- @property
- def sasid_finished_fraction(self):
- size_archived = 0
- size_remaining = 0
- total_size = 0
-
- tasks = Task.objects.filter(sas_id=self.sas_id)
-
- for task in tasks:
- if task.status == State.FINISHED.value:
- size_archived = size_archived + task.size_to_process
- else:
- size_remaining = size_remaining + task.size_to_process
- total_size = total_size + task.size_to_process
-
- finished = {}
- try:
- finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
- except:
- finished['fraction'] = -1
-
- finished['total_size'] = total_size
- finished['remaining'] = size_remaining
-
- return finished
-
- @property
- def sasid_ingested_fraction(self):
- """
- This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
- and a list of statusses of other tasks belonging to the same SAS_ID.
- It is implemented as 'property', because then it can be used in html pages like this:
- <td>{{ task.sasid_ingested_fraction.status }}</td>
- <td>{{ task.sasid_ingested_fraction.completion }}%</td>
-
- A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
- The division of those 2 are returned as a 'completed %'.
- A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
-
- """
- result = {}
- statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
- 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
-
- tasks = Task.objects.filter(sas_id=self.sas_id)
-
- for task in tasks:
- try:
- statusses[task.status] += 1
- except:
- pass
-
- incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
- int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
- complete = int(statusses['archived']) + int(statusses['finished'])
- completion = round(complete / (incomplete + complete) * 100)
-
- non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
+# keep the old mechanism in comments to test/evaluate, remove later when it works
+ # @property
+ # def sasid_path_to_lta(self):
+ # """
+ # check if any task belonging to this sas_id already has a 'path_to_lta' setting
+ # """
+ # try:
+ # for task in Task.objects.filter(sas_id=self.sas_id):
+ # try:
+ # if task.archive['path_to_lta']:
+ # return task.archive['path_to_lta']
+ # except:
+ # # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
+ # pass
+ # except:
+ # return None
- result['status'] = non_zero_statusses
- result['completion'] = completion
- return result
+ #
+ # @property
+ # def sasid_is_verified(self):
+ # for task in Task.objects.filter(sas_id=self.sas_id):
+ # if task.status not in verified_statusses:
+ # return False
+ # return True
+ #
+ # @property
+ # def sasid_finished_fraction(self):
+ # size_archived = 0
+ # size_remaining = 0
+ # total_size = 0
+ #
+ # tasks = Task.objects.filter(sas_id=self.sas_id)
+ #
+ # for task in tasks:
+ # if task.status == State.FINISHED.value:
+ # size_archived = size_archived + task.size_to_process
+ # else:
+ # size_remaining = size_remaining + task.size_to_process
+ # total_size = total_size + task.size_to_process
+ #
+ # finished = {}
+ # try:
+ # finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
+ # except:
+ # finished['fraction'] = -1
+ #
+ # finished['total_size'] = total_size
+ # finished['remaining'] = size_remaining
+ #
+ # return finished
+ #
+ # @property
+ # def sasid_ingested_fraction(self):
+ # """
+ # This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
+ # and a list of statusses of other tasks belonging to the same SAS_ID.
+ # It is implemented as 'property', because then it can be used in html pages like this:
+ # <td>{{ task.sasid_ingested_fraction.status }}</td>
+ # <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+ #
+ # A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
+ # The division of those 2 are returned as a 'completed %'.
+ # A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
+ #
+ # """
+ # result = {}
+ # statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
+ # 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
+ #
+ # tasks = Task.objects.filter(sas_id=self.sas_id)
+ #
+ # for task in tasks:
+ # try:
+ # statusses[task.status] += 1
+ # except:
+ # pass
+ #
+ # incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
+ # int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
+ # complete = int(statusses['archived']) + int(statusses['finished'])
+ # completion = round(complete / (incomplete + complete) * 100)
+ #
+ # non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
+ #
+ # result['status'] = non_zero_statusses
+ # result['completion'] = completion
+ # return result
@property
def task_type_join(self):
diff --git a/atdb/taskdatabase/serializers.py b/atdb/taskdatabase/serializers.py
index a77ca33db457214a40bba66e7647523ee1b7e4d6..3adcc01c29f84f437b30aa2a4de0156ad8fdb6ca 100644
--- a/atdb/taskdatabase/serializers.py
+++ b/atdb/taskdatabase/serializers.py
@@ -95,7 +95,6 @@ class TaskReadSerializer(serializers.ModelSerializer):
required=False,
)
-
class Meta:
model = Task
fields = ['id','task_type','creationTime','filter',
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index 0d5b5f0f4445d9c43287f8c030a73abb674b1e1c..edc22763aa3bf6378052eb5c92f5199f9613a6d4 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -76,13 +76,15 @@ def associate_task_with_activity(task, save_task=True):
logger.info(f'create new activity for sas_id {task.sas_id}')
activity = Activity(sas_id=task.sas_id,
- workflow=task.workflow,
project=task.project,
- filter=task.filter,
- priority=task.priority,
- archive=task.archive)
+ filter=task.filter)
activity.save()
+
+ # update activity
+ activity.filter = task.filter
+ activity.workflow_id = task.workflow.id
+
task.activity = activity
if save_task:
task.save()
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 06c42d7b06e24f1c83606c96d55d172e2bc3ea3e..9c5cf32d8e1f01a6772c7ae7d73eb3b1e22ca6c8 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -99,7 +99,8 @@ class ActivityFilter(filters.FilterSet):
fields = {
'sas_id': ['exact', 'icontains', 'in'],
'filter': ['exact', 'icontains'],
- 'workflow__id': ['exact', 'icontains'],
+ #'workflow__id': ['exact', 'icontains'],
+ 'workflow_id': ['exact', 'icontains'],
'project': ['exact', 'icontains'],
'sas_id': ['exact', 'icontains', 'in'],
'status': ['exact', 'icontains', 'in', 'startswith'],