From ea4f21802b997feb64113cfffe0d5edefcca8042 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Tue, 6 Feb 2024 17:11:00 +0100
Subject: [PATCH] experimenting with activities
---
.../migrations/0035_task_activity.py | 19 +++++++++++
atdb/taskdatabase/models.py | 3 ++
atdb/taskdatabase/serializers.py | 4 ++-
atdb/taskdatabase/services/activities.py | 34 +++++++++++++++++--
.../templates/taskdatabase/ingest/tasks.html | 3 +-
atdb/taskdatabase/urls.py | 2 ++
atdb/taskdatabase/views.py | 10 +++++-
7 files changed, 69 insertions(+), 6 deletions(-)
create mode 100644 atdb/taskdatabase/migrations/0035_task_activity.py
diff --git a/atdb/taskdatabase/migrations/0035_task_activity.py b/atdb/taskdatabase/migrations/0035_task_activity.py
new file mode 100644
index 00000000..71eb8158
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0035_task_activity.py
@@ -0,0 +1,19 @@
+# Generated by Django 5.0 on 2024-02-06 12:38
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0034_activity_ingestq_status'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='task',
+ name='activity',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='tasks', to='taskdatabase.activity'),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 0b9a5a78..06ec943e 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -158,6 +158,9 @@ class Task(models.Model):
predecessor = models.ForeignKey('self', related_name='successors', on_delete=models.SET_NULL, null=True, blank=True)
joined_output_task = models.ForeignKey('self', related_name='joined_input_tasks', on_delete=models.SET_NULL, null=True, blank=True)
+ # pipeline or observation
+ activity = models.ForeignKey(Activity, related_name='tasks', on_delete=models.DO_NOTHING, null=True, blank=True)
+
def __str__(self):
return str(self.id) + ' - (' + self.task_type + ') - ' + str(self.sas_id)
diff --git a/atdb/taskdatabase/serializers.py b/atdb/taskdatabase/serializers.py
index 7a2f4312..a77ca33d 100644
--- a/atdb/taskdatabase/serializers.py
+++ b/atdb/taskdatabase/serializers.py
@@ -95,6 +95,7 @@ class TaskReadSerializer(serializers.ModelSerializer):
required=False,
)
+
class Meta:
model = Task
fields = ['id','task_type','creationTime','filter',
@@ -106,7 +107,8 @@ class TaskReadSerializer(serializers.ModelSerializer):
'status','new_status','quality','calculated_qualities',
'inputs','outputs','metrics','remarks','status_history',
'size_to_process', 'size_processed', 'total_processing_time',
- 'log_entries','meta_scheduling','environment','archive'
+ 'log_entries','meta_scheduling','environment','archive',
+ 'activity'
]
read_only_fields = fields
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index 534705f2..198a103b 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -41,6 +41,28 @@ def calculate_ingested_fraction(this_task):
return result
+def associate_activity(task):
+
+ if not task.activity:
+
+ try:
+ activity = Activity.objects.get(sas_id=task.sas_id)
+ except:
+ # no activity exists yet, create it
+ logger.info(f'create new activity for sas_id {task.sas_id}')
+
+ activity = Activity(sas_id=task.sas_id,
+ workflow=task.workflow,
+ project=task.project,
+ filter=task.filter,
+ priority=task.priority,
+ archive=task.archive)
+ activity.save()
+
+ task.activity = activity
+ task.save()
+
+
def update_activity(task):
"""
The activity (SAS_ID level) is updated whenever a task change status.
@@ -69,12 +91,20 @@ def update_activity(task):
archive=task.archive)
activity.save()
+ # associate an activity to this tasks
+ if not task.activity:
+ activity = Activity.objects.get(sas_id=task.sas_id)
+ task.activity = activity
+
# depending on the status transition, perform calculations
if task.status == State.STORED.value:
logger.info(f'- calculate_qualities')
# quality is calculated per task and per sas_id, reget the quality per sas_id
- activity.calculated_quality = task.calculated_qualities['per_sasid']
- activity.save()
+ try:
+ activity.calculated_quality = task.calculated_qualities['per_sasid']
+ activity.save()
+ except:
+ pass
# calculate the fraction and list of statusses of ingested tasks of this SAS_ID
if task.status in [State.SCRUBBED.value, State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]:
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
index 90118ac8..f7ba1aa9 100644
--- a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
@@ -23,8 +23,7 @@
<a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
{% endif %}
</td>
- <td>{{ task.sasid_ingested_fraction.status }}</td>
-
+ <td>{{ task.activity.ingestq_status }}</td>
<td>{{ task.sasid_ingested_fraction.completion }}%</td>
<td>
{% if task.sas_id_has_archived != None %}
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 7fc2d596..a89c664a 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -129,4 +129,6 @@ urlpatterns = [
path('tasks/<int:pk>/query-hold/<hold_it>/<query_params>', views.HoldQuery, name='query-hold-resume'),
path('tasks/<int:pk>/hold/<hold_it>/<page>', views.Hold, name='service-hold-resume'),
path('tasks/<int:pk>/query-purge/<purge_policy>/<query_params>', views.PurgeQuery, name='query-purge'),
+
+ path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
]
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index c86f01dd..1ab90aec 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -38,7 +38,7 @@ from .serializers import \
PostProcessingRuleSerializer, \
MonitorSerializer, LatestMonitorSerializer
-from .services import algorithms
+from .services import algorithms, activities
logger = logging.getLogger(__name__)
@@ -1710,3 +1710,11 @@ class GetUniqueValuesForKey(generics.ListAPIView):
})
+def AssociateActivities(request):
+
+ tasks = Task.objects.all()
+ for task in tasks:
+ activities.associate_activity(task)
+ logger.info(f'{task.id} => {task.sas_id}')
+
+
--
GitLab