diff --git a/atdb/taskdatabase/migrations/0042_task_is_summary.py b/atdb/taskdatabase/migrations/0042_task_is_summary.py
new file mode 100644
index 0000000000000000000000000000000000000000..eae714e842c607213f3347d6e4666863706a668d
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0042_task_is_summary.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2024-02-20 12:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0041_alter_task_activity'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='task',
+ name='is_summary',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/atdb/taskdatabase/migrations/0043_activity_is_combined_activity_is_validated.py b/atdb/taskdatabase/migrations/0043_activity_is_combined_activity_is_validated.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bbc07c6bf5f265616cb5d994e3acdcffa4a080d
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0043_activity_is_combined_activity_is_validated.py
@@ -0,0 +1,23 @@
+# Generated by Django 5.0 on 2024-02-22 13:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0042_task_is_summary'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='activity',
+ name='is_combined',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='activity',
+ name='is_validated',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 8d9f1100d0f064cbd79f2bc3f73971b3a289a1c1..5f6e51cc8b849143fe3e65ab5b98c4b3e65160b4 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -7,7 +7,7 @@ import json
import logging
from .services import calculated_qualities as qualities
-from .services.common import State, verified_statusses
+from .services.common import State
logger = logging.getLogger(__name__)
@@ -111,7 +111,15 @@ class Activity(models.Model):
# this is the JSON blob that is filled in by the ldv_archiver during the ingest process
archive = models.JSONField(null=True, blank=True)
- is_verified = models.BooleanField(default=False)
+ # set by update_activity, used by Validation Page
+ is_verified = models.BooleanField(default=False)
+
+ # TODO: flag set by the 'validate' step in ATDB, used by combine service
+ is_validated = models.BooleanField(default=False)
+
+ # TODO: flag set (and used) by the combine service, so that it doesn't do double work
+ is_combined = models.BooleanField(default=False)
+
finished_fraction = models.FloatField(blank=True, null=True)
ingested_fraction = models.FloatField(blank=True, null=True)
total_size = models.FloatField(blank=True, null=True)
@@ -135,10 +143,29 @@ class Activity(models.Model):
return str(self.sas_id)
+def check_if_summary(task):
+ """
+ check if this task is a summary task
+ for backward compatiblity reasons this is done very ugly, by looking if certain filenames contain the substring 'summary'
+ """
+ # look in the outputs.tar_archive
+ try:
+ tars = task.outputs['tar_archive']
+ for tar in tars:
+ if 'summary' in tar['basename']:
+ # a summary tarball was found, this task is a summary task
+ return True
+ except:
+ # no 'tar_archive' was found
+ return False
+
+ return False
+
class Task(models.Model):
# Task control properties
task_type = models.CharField(max_length=20, default="regular")
+ is_summary = models.BooleanField(default=False)
filter = models.CharField(max_length=30, blank=True, null=True)
environment = models.CharField(max_length=255, blank=True, null=True)
new_status = models.CharField(max_length=50, default="defining", null=True)
@@ -201,6 +228,15 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
+ # nv:20feb2024, check if this task is a summary task
+ if (self.status != State.STORED.value) & (self.new_status == State.STORED.value):
+ self.is_summary = check_if_summary(self)
+
+ # nv:20feb2024, same as above, but for backward compatibilty reasons.
+ # For tasks that are already beyond PROCESSED, but not yet ingested.
+ if (self.status != State.VALIDATED.value) & (self.new_status == State.VALIDATED.value):
+ self.is_summary = check_if_summary(self)
+
# make sure that every task has an activity (also for backward compatibility)
associate_task_with_activity(self)
diff --git a/atdb/taskdatabase/serializers.py b/atdb/taskdatabase/serializers.py
index 3adcc01c29f84f437b30aa2a4de0156ad8fdb6ca..f80c4eb13fcfc3e9c0d7ddf93963f69114b2da89 100644
--- a/atdb/taskdatabase/serializers.py
+++ b/atdb/taskdatabase/serializers.py
@@ -97,7 +97,7 @@ class TaskReadSerializer(serializers.ModelSerializer):
class Meta:
model = Task
- fields = ['id','task_type','creationTime','filter',
+ fields = ['id','task_type','is_summary','creationTime','filter',
'predecessor','predecessor_status','successors',
'joined_input_tasks','joined_output_task','joined_status',
'project','sas_id','priority','purge_policy','cleanup_policy','resume',
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 42536608ddb2b12a276ba820009adedce6bf2861..23651ae0db817fc212fddd3b82bdae281881c78e 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -1144,4 +1144,4 @@ def construct_summary(task):
elif summary_flavour == SummaryFlavour.LINC_TARGET.value:
html = construct_linc_summary(task)
- return html
\ No newline at end of file
+ return html
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index fc4fe7f33da131e313a1b83810dfa786f4e32ff6..35a46bfd0f2ad11b1652b01fa2cdb834d8545a12 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 20 Feb 2024
+ <p class="footer"> Version 22 Feb 2024
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/tests/test_activities_associate.py b/atdb/taskdatabase/tests/test_activities_associate.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/atdb/taskdatabase/tests/test_summary_tasks.py b/atdb/taskdatabase/tests/test_summary_tasks.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf33c9f75d68202b5ecfec0e0e801e81c50a1f40
--- /dev/null
+++ b/atdb/taskdatabase/tests/test_summary_tasks.py
@@ -0,0 +1,42 @@
+from django.test import TestCase
+import json
+from taskdatabase.models import Task, Workflow, Activity
+from taskdatabase.services.common import State
+class TestSummaryTasks(TestCase):
+ def setUp(self):
+ """
+ initialize test data
+ """
+ self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ self.workflow_requantisation.save()
+
+ self.no_summary_task = Task.objects.create(sas_id=77777, new_status=State.DEFINED.value, workflow=self.workflow_requantisation,
+ outputs={"tar_archive": [{"size": 4885985280, "basename": "L621240_SAP002_B073_P000_bf.tar", "nameroot": "L621240_SAP002_B073_P000_bf"}]})
+ self.summary_task_defined = Task.objects.create(sas_id=77777, new_status=State.DEFINED.value, workflow=self.workflow_requantisation,
+ outputs={"tar_archive": [{"size": 4885985280, "basename": "L185619_summaryCS.tar", "nameroot": "L185619_summaryCS"}]})
+ self.summary_task_stored = Task.objects.create(sas_id=77777, new_status=State.STORED.value, workflow=self.workflow_requantisation,
+ outputs={"tar_archive": [{"size": 4885985280, "basename": "L185619_summaryCS.tar", "nameroot": "L185619_summaryCS"}]})
+
+ def test_no_summary_task(self):
+ """
+ test task that is not a summary task
+ """
+
+ actual = self.no_summary_task.is_summary
+ self.assertEqual(actual, False)
+
+ def test_summary_task_defined(self):
+ """
+ test summary task, but before it knows that it becomes a summary task (which it only knows when 'processed')
+ """
+
+ actual = self.summary_task_defined.is_summary
+ self.assertEqual(actual, False)
+
+ def test_summary_task_stored(self):
+ """
+ test summary task, at 'stored' it should know that it is a summary task and return True)
+ """
+ self.summary_task_stored.save()
+ actual = self.summary_task_stored.is_summary
+ self.assertEqual(actual, True)
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index eb462904c64a075019df95d1337a09913418f55d..0ff77a34177461537ab5f09977cd1e8685835af8 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -57,6 +57,7 @@ class TaskFilter(filters.FilterSet):
fields = {
'task_type': ['exact', 'icontains', 'in'],
+ 'is_summary': ['exact'],
'creationTime': ['icontains'],
'filter': ['exact', 'icontains'],
'workflow__id': ['exact', 'icontains'],