From 2077a23d9e302708045e287797d15d3a4c22272d Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Fri, 9 Feb 2024 09:54:01 +0100
Subject: [PATCH] add 'failures' functionality to activities also
---
...tput_sas_id_activity_remaining_and_more.py | 27 ++++++++++
atdb/taskdatabase/models.py | 54 ++++++++++++++++---
atdb/taskdatabase/services/activities.py | 41 ++++++++++++--
.../taskdatabase/archived/tasks.html | 14 ++++-
.../taskdatabase/failures/tasks.html | 6 +++
.../templates/taskdatabase/ingest/tasks.html | 17 ++++++
.../taskdatabase/validation/tasks.html | 7 ++-
.../validation/validation_buttons.html | 18 +++++++
atdb/taskdatabase/urls.py | 3 ++
atdb/taskdatabase/views.py | 47 ++++++++++++++--
10 files changed, 218 insertions(+), 16 deletions(-)
create mode 100644 atdb/taskdatabase/migrations/0036_remove_activity_output_sas_id_activity_remaining_and_more.py
diff --git a/atdb/taskdatabase/migrations/0036_remove_activity_output_sas_id_activity_remaining_and_more.py b/atdb/taskdatabase/migrations/0036_remove_activity_output_sas_id_activity_remaining_and_more.py
new file mode 100644
index 00000000..cd15446d
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0036_remove_activity_output_sas_id_activity_remaining_and_more.py
@@ -0,0 +1,27 @@
+# Generated by Django 5.0 on 2024-02-09 08:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0035_task_activity'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='activity',
+ name='output_sas_id',
+ ),
+ migrations.AddField(
+ model_name='activity',
+ name='remaining',
+ field=models.FloatField(blank=True, null=True),
+ ),
+ migrations.AddField(
+ model_name='activity',
+ name='total_size',
+ field=models.FloatField(blank=True, null=True),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index d53a6a11..955e2e79 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -97,14 +97,18 @@ class Activity(models.Model):
status = models.CharField(db_index=True, default="unknown", max_length=50, blank=True, null=True)
calculated_quality = models.CharField(max_length=10, blank=True, null=True)
+
+ # this is the JSON blob that is filled in by the ldv_archiver during the ingest process
archive = models.JSONField(null=True, blank=True)
- # output sas_id at LTA
- output_sas_id = models.CharField(max_length=15, blank=True, null=True)
has_summary = models.BooleanField(default=False)
+
is_verified = models.BooleanField(default=False)
finished_fraction = models.FloatField(blank=True, null=True)
ingested_fraction = models.FloatField(blank=True, null=True)
+ total_size = models.FloatField(blank=True, null=True)
+ remaining = models.FloatField(blank=True, null=True)
+
ingestq_status = models.CharField(default="", max_length=100, blank=True, null=True)
@property
@@ -112,7 +116,6 @@ class Activity(models.Model):
"""
check if any task belonging to this sas_id already has an output SAS_ID at the LTA
"""
-
try:
if self.archive['sas_id_archived']:
return self.archive['sas_id_archived']
@@ -187,10 +190,6 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
- # add information to the aggregated activity (SAS_ID) level
- # moved to signals to avoid circular import
- # activities.update_activity(self)
-
super(Task, self).save(*args, **kwargs)
@@ -311,6 +310,31 @@ class Task(models.Model):
except:
return None
+ @property
+ def sas_id_archived(self):
+ """
+ check if this task already has an output SAS_ID at the LTA
+ """
+ try:
+ return self.archive['sas_id_archived']
+ except:
+ return None
+
+ @property
+ def sas_id_has_archived(self):
+ """
+ check if any task belonging to this sas_id already has an output SAS_ID at the LTA
+ """
+ try:
+ for task in Task.objects.filter(sas_id=self.sas_id):
+ try:
+ if task.archive['sas_id_archived']:
+ return task.archive['sas_id_archived']
+ except:
+ pass
+ except:
+ return None
+
@property
def path_to_lta(self):
"""
@@ -321,6 +345,22 @@ class Task(models.Model):
except:
return None
+ @property
+ def sasid_path_to_lta(self):
+ """
+ check if any task belonging to this sas_id already has a 'path_to_lta' setting
+ """
+ try:
+ for task in Task.objects.filter(sas_id=self.sas_id):
+ try:
+ if task.archive['path_to_lta']:
+ return task.archive['path_to_lta']
+ except:
+ # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
+ pass
+ except:
+ return None
+
@property
def sasid_is_verified(self):
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index e5a91340..0d5b5f0f 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -40,6 +40,30 @@ def calculate_ingested_fraction(this_task):
result['completion'] = completion
return result
+def calculate_finished_fraction(this_task):
+ size_archived = 0
+ size_remaining = 0
+ total_size = 0
+
+ tasks = Task.objects.filter(sas_id=this_task.sas_id)
+
+ for task in tasks:
+ if task.status == State.FINISHED.value:
+ size_archived = size_archived + task.size_to_process
+ else:
+ size_remaining = size_remaining + task.size_to_process
+ total_size = total_size + task.size_to_process
+
+ result = {}
+ try:
+ result['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
+ except:
+ result['fraction'] = -1
+
+ result['total_size'] = total_size
+ result['remaining'] = size_remaining
+
+ return result
def associate_task_with_activity(task, save_task=True):
@@ -71,10 +95,11 @@ def update_activity(task):
Depending on the type of status change, certain calculations and updates are performed.
Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
- - to '???' : check for incoming 'archive' json from archiver
+ - to 'ARCHIVING, ARCHIVED' : check for incoming 'archive' json from archiver
- to STORED : calculate quality
- to ??? : calculate finished_fraction
- to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
+ - always : calc 'verified'
"""
logger.info(f'update_activity for task {task.id} with sas_id {task.sas_id} and status {task.status}')
@@ -103,7 +128,7 @@ def update_activity(task):
# check of any task of this activity already has LTA information. If so, copy to the activity level
- if task.status in [State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]:
+ if task.status in [State.ARCHIVING.value, State.ARCHIVED.value]:
logger.info(f'- add archive json')
for t in Task.objects.filter(sas_id=task.sas_id):
try:
@@ -115,6 +140,16 @@ def update_activity(task):
activity.save()
+
+ # calculate the finished fraction, this is only used on the Failures page
+ if 'failed' in task.status:
+ logger.info(f'- calculate_ingested_fraction')
+ result = calculate_finished_fraction(task)
+ activity.finished_fraction = result['fraction']
+ activity.total_size = result['total_size']
+ activity.remaining = result['remaining']
+ activity.save()
+
# check if all tasks of this SAS_ID have a status that is considered 'verified'
# this is used for the Validation Page
current_is_verified = activity.is_verified
@@ -126,4 +161,4 @@ def update_activity(task):
# only save when changed
if activity.is_verified != current_is_verified:
- activity.save()
+ activity.save()
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html b/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html
index 3a3bb9cd..b65b2ba4 100644
--- a/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html
@@ -52,10 +52,22 @@
{{ task.sas_id }}
</td>
<td>
+
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
{% if task.sas_id_archived != None %}
<a href={{ task.path_to_lta }} target="_blank">
<img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
- {{ task.sas_id_archived }}
+ {{ task.sas_id_archived }} (old)
+ </a>
+ {% else %}
+ -
+ {% endif %}
+-->
+ {% if task.activity.archive.sas_id_archived != None %}
+ <a href={{ task.path_to_lta }} target="_blank">
+ <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+ {{ task.activity.archive.sas_id_archived }}
</a>
{% else %}
-
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html b/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
index 59496de3..988f1a37 100644
--- a/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
@@ -36,8 +36,14 @@
<td>{{ task.sas_id }}</td>
<td>{{ task.filter }} </td>
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
<td>{{ task.sasid_finished_fraction.fraction }}% of {{ task.sasid_finished_fraction.total_size|filesizeformat }}</td>
<td>{{ task.size_to_process|filesizeformat }} / {{ task.sasid_finished_fraction.remaining|filesizeformat }}</td>
+-->
+ <td>{{ task.activity.finished_fraction }}% of {{ task.activity.total_size|filesizeformat }}</td>
+ <td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td>
+
<td>
{% include "taskdatabase/failures/retry_buttons.html" %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
index 7f4a80d1..0b250e52 100644
--- a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
@@ -23,6 +23,23 @@
<a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
{% endif %}
</td>
+
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
+ <td>{{ task.sasid_ingested_fraction.status }}</td>
+
+ <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+ <td>
+ {% if task.sas_id_has_archived != None %}
+ <a href={{ task.sasid_path_to_lta }} target="_blank">
+ <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+ {{ task.sas_id_has_archived }}
+ </a>
+ {% else %}
+ -
+ {% endif %}
+ </td>
+-->
<td>{{ task.activity.ingestq_status }}</td>
<td>{{ task.activity.ingested_fraction }}%</td>
<td>
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
index 794ca47d..e5282701 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
@@ -68,7 +68,12 @@
{% endif %}
{% endif %}
</td>
- <td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }}</td>
+
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
+ <td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }} (old)</td>
+-->
+ <td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td>
<td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td>
<td>{% include "taskdatabase/validation/validation_buttons.html" %}</td>
<td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td>
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
index a64240c3..b900a6be 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
@@ -1,3 +1,21 @@
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
+ {% if task.sasid_is_verified %}
+ <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
+ {% endif %}
+
+ {% if task.sasid_is_verified %}
+ <a href="{% url 'task-validate-sasid' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> M</a>
+ {% endif %}
+
+ {% if task.sasid_is_verified %}
+ <a href="{% url 'task-validate-sasid' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> G</a>
+ {% endif %}
+
+ {% if task.sasid_is_verified %}
+ <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
+ {% endif %}
+-->
{% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 202d73ee..11c65788 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -22,6 +22,7 @@ urlpatterns = [
path('ingest', views.ShowIngestQPage.as_view(), name='ingest'),
path('finished', views.ShowFinishedPage.as_view(), name='finished'),
+
path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
path('task_details/', views.TaskDetails, name='task-details'),
path('task_quality/<int:id>/<page>', views.ShowTaskQuality, name='task-quality'),
@@ -129,6 +130,8 @@ urlpatterns = [
path('tasks/<int:pk>/hold/<hold_it>/<page>', views.Hold, name='service-hold-resume'),
path('tasks/<int:pk>/query-purge/<purge_policy>/<query_params>', views.PurgeQuery, name='query-purge'),
+ #some migration and repair endpoints
path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
path('tasks/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'),
+ path('tasks/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'),
]
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index c29fcc75..06c42d7b 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -16,6 +16,8 @@ from django_tables2.views import SingleTableMixin
from django.shortcuts import render, redirect, reverse
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from django.contrib.admin.views.decorators import staff_member_required
+
from rest_framework.request import Request
#from silk.profiling.profiler import silk_profile
@@ -103,6 +105,9 @@ class ActivityFilter(filters.FilterSet):
'status': ['exact', 'icontains', 'in', 'startswith'],
'ingestq_status': ['icontains'],
'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'],
+ 'finished_fraction': ['exact', 'lt', 'lte', 'gt', 'gte'],
+ 'total_size': ['exact', 'lt', 'lte', 'gt', 'gte'],
+ 'remaining': ['exact', 'lt', 'lte', 'gt', 'gte'],
}
@@ -1671,21 +1676,55 @@ class GetUniqueValuesForKey(generics.ListAPIView):
'error': str(error)
})
-
+@staff_member_required
def AssociateActivities(request):
tasks = Task.objects.all()
+ total = tasks.count()
+ i = 0
for task in tasks:
+ i+=1
activities.associate_task_with_activity(task)
- logger.info(f'{task.id} => {task.sas_id}')
+ logger.info(f'{i} of {total}')
return redirect('index')
+@staff_member_required
def UpdateAllActivities(request):
- tasks = Task.objects.all()
+ all_activities = Activity.objects.all()
+ # find a task for every activity
+ total = all_activities.count()
+ i = 0
+
+ for activity in all_activities:
+ try:
+ i += 1
+ task = Task.objects.filter(sas_id=activity.sas_id)[0]
+ activities.update_activity(task)
+ logger.info(f'{i} of {total}')
+ except Exception as error:
+ logger.error(error)
+
+ # tasks = Task.objects.all()
+ # total = tasks.count()
+ # i = 0
+ # for task in tasks:
+ # i+=1
+ # activities.update_activity(task)
+ # logger.info(f'{i} of {total}')
+
+ return redirect('index')
+
+@staff_member_required
+def UpdateFailedTasks(request):
+
+ tasks = Task.objects.filter(status__icontains="failed")
+ total = tasks.count()
+ i = 0
for task in tasks:
+ i+=1
activities.update_activity(task)
- logger.info(f'{task.id}')
+ logger.info(f'{i} of {total}')
return redirect('index')
\ No newline at end of file
--
GitLab