add 'failures' functionality to activities also

atdb/taskdatabase/migrations/0036_remove_activity_output_sas_id_activity_remaining_and_more.py

+# Generated by Django 5.0 on 2024-02-09 08:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0035_task_activity'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='activity',
+            name='output_sas_id',
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='remaining',
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='activity',
+            name='total_size',
+            field=models.FloatField(blank=True, null=True),
+        ),
+    ]

atdb/taskdatabase/models.py

@@ -97,14 +97,18 @@ class Activity(models.Model):
 
     status = models.CharField(db_index=True, default="unknown", max_length=50, blank=True, null=True)
     calculated_quality = models.CharField(max_length=10, blank=True, null=True)
+
+    # this is the JSON blob that is filled in by the ldv_archiver during the ingest process
     archive = models.JSONField(null=True, blank=True)
 
-    # output sas_id at LTA
-    output_sas_id = models.CharField(max_length=15, blank=True, null=True)
     has_summary = models.BooleanField(default=False)
+
     is_verified = models.BooleanField(default=False)
     finished_fraction = models.FloatField(blank=True, null=True)
     ingested_fraction = models.FloatField(blank=True, null=True)
+    total_size = models.FloatField(blank=True, null=True)
+    remaining = models.FloatField(blank=True, null=True)
+
     ingestq_status = models.CharField(default="", max_length=100, blank=True, null=True)
 
     @property
@@ -112,7 +116,6 @@ class Activity(models.Model):
         """
         check if any task belonging to this sas_id already has an output SAS_ID at the LTA
         """
-
         try:
             if self.archive['sas_id_archived']:
                 return self.archive['sas_id_archived']
@@ -187,10 +190,6 @@ class Task(models.Model):
             tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
             self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
 
-        # add information to the aggregated activity (SAS_ID) level
-        # moved to signals to avoid circular import
-        # activities.update_activity(self)
-
         super(Task, self).save(*args, **kwargs)
 
 
@@ -311,6 +310,31 @@ class Task(models.Model):
         except:
             return None
 
+    @property
+    def sas_id_archived(self):
+        """
+        check if this task already has an output SAS_ID at the LTA
+        """
+        try:
+            return self.archive['sas_id_archived']
+        except:
+            return None
+
+    @property
+    def sas_id_has_archived(self):
+        """
+        check if any task belonging to this sas_id already has an output SAS_ID at the LTA
+        """
+        try:
+            for task in Task.objects.filter(sas_id=self.sas_id):
+                try:
+                    if task.archive['sas_id_archived']:
+                        return task.archive['sas_id_archived']
+                except:
+                    pass
+        except:
+            return None
+
     @property
     def path_to_lta(self):
         """
@@ -321,6 +345,22 @@ class Task(models.Model):
         except:
             return None
 
+    @property
+    def sasid_path_to_lta(self):
+        """
+        check if any task belonging to this sas_id already has a 'path_to_lta' setting
+        """
+        try:
+            for task in Task.objects.filter(sas_id=self.sas_id):
+                try:
+                    if task.archive['path_to_lta']:
+                        return task.archive['path_to_lta']
+                except:
+                    # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
+                    pass
+        except:
+            return None
+
 
     @property
     def sasid_is_verified(self):

atdb/taskdatabase/services/activities.py

@@ -40,6 +40,30 @@ def calculate_ingested_fraction(this_task):
     result['completion'] = completion
     return result
 
+def calculate_finished_fraction(this_task):
+        size_archived = 0
+        size_remaining = 0
+        total_size = 0
+
+        tasks = Task.objects.filter(sas_id=this_task.sas_id)
+
+        for task in tasks:
+           if task.status == State.FINISHED.value:
+               size_archived = size_archived + task.size_to_process
+           else:
+               size_remaining = size_remaining + task.size_to_process
+           total_size = total_size + task.size_to_process
+
+        result = {}
+        try:
+            result['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
+        except:
+            result['fraction'] = -1
+
+        result['total_size'] = total_size
+        result['remaining'] = size_remaining
+
+        return result
 
 def associate_task_with_activity(task, save_task=True):
 
@@ -71,10 +95,11 @@ def update_activity(task):
     Depending on the type of status change, certain calculations and updates are performed.
     Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
 
-    - to '???'                                   : check for incoming 'archive' json from archiver
+    - to 'ARCHIVING, ARCHIVED'                   : check for incoming 'archive' json from archiver
     - to STORED                                  : calculate quality
     - to ???                                     : calculate finished_fraction
     - to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
+    - always                                     : calc 'verified'
 
     """
     logger.info(f'update_activity for task {task.id} with sas_id {task.sas_id} and status {task.status}')
@@ -103,7 +128,7 @@ def update_activity(task):
 
 
     # check of any task of this activity already has LTA information. If so, copy to the activity level
-    if task.status in [State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]:
+    if task.status in [State.ARCHIVING.value, State.ARCHIVED.value]:
         logger.info(f'- add archive json')
         for t in Task.objects.filter(sas_id=task.sas_id):
             try:
@@ -115,6 +140,16 @@ def update_activity(task):
 
         activity.save()
 
+
+    # calculate the finished fraction, this is only used on the Failures page
+    if 'failed' in task.status:
+        logger.info(f'- calculate_ingested_fraction')
+        result = calculate_finished_fraction(task)
+        activity.finished_fraction = result['fraction']
+        activity.total_size = result['total_size']
+        activity.remaining = result['remaining']
+        activity.save()
+
     # check if all tasks of this SAS_ID have a status that is considered 'verified'
     # this is used for the Validation Page
     current_is_verified = activity.is_verified
@@ -126,4 +161,4 @@ def update_activity(task):
 
     # only save when changed
     if activity.is_verified != current_is_verified:
-        activity.save()
+        activity.save()
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/archived/tasks.html

@@ -52,10 +52,22 @@
                     {{ task.sas_id }}
                 </td>
                 <td>
+
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
                     {% if task.sas_id_archived != None %}
                       <a href={{ task.path_to_lta }} target="_blank">
                           <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
-                          {{ task.sas_id_archived }}
+                          {{ task.sas_id_archived }} (old)
+                      </a>&nbsp;
+                    {% else %}
+                    -
+                    {% endif %}
+-->
+                    {% if task.activity.archive.sas_id_archived != None %}
+                      <a href={{ task.path_to_lta }} target="_blank">
+                          <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+                          {{ task.activity.archive.sas_id_archived }}
                       </a>&nbsp;
                     {% else %}
                     -

atdb/taskdatabase/templates/taskdatabase/failures/tasks.html

@@ -36,8 +36,14 @@
                 <td>{{ task.sas_id }}</td>
 
                 <td>{{ task.filter }} </td>
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
                 <td>{{ task.sasid_finished_fraction.fraction }}% of {{ task.sasid_finished_fraction.total_size|filesizeformat }}</td>
                 <td>{{ task.size_to_process|filesizeformat }} / {{ task.sasid_finished_fraction.remaining|filesizeformat }}</td>
+-->
+                <td>{{ task.activity.finished_fraction }}% of {{ task.activity.total_size|filesizeformat }}</td>
+                <td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td>
+
 
                 <td>
                     {% include "taskdatabase/failures/retry_buttons.html" %}

atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html

@@ -23,6 +23,23 @@
                     <a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
                 {% endif %}
                 </td>
+
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
+                <td>{{ task.sasid_ingested_fraction.status }}</td>
+
+                <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+                <td>
+                    {% if task.sas_id_has_archived != None %}
+                      <a href={{ task.sasid_path_to_lta }} target="_blank">
+                          <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+                          {{ task.sas_id_has_archived }}
+                      </a>&nbsp;
+                    {% else %}
+                    -
+                    {% endif %}
+                </td>
+-->
                 <td>{{ task.activity.ingestq_status }}</td>
                 <td>{{ task.activity.ingested_fraction }}%</td>
                 <td>

atdb/taskdatabase/templates/taskdatabase/validation/tasks.html

@@ -68,7 +68,12 @@
                         {% endif %}
                     {% endif %}
                 </td>
-                <td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }}</td>
+
+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
+                <td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }} (old)</td>
+-->
+                <td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td>
                 <td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td>
                 <td>{% include "taskdatabase/validation/validation_buttons.html" %}</td>
                 <td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td>

atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html

+<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
+<!--
+ {% if task.sasid_is_verified %}
+    <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
+ {% endif %}
+
+ {% if task.sasid_is_verified %}
+    <a href="{% url 'task-validate-sasid' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> M</a>
+ {% endif %}
+
+ {% if task.sasid_is_verified %}
+    <a href="{% url 'task-validate-sasid' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> G</a>
+ {% endif %}
+
+ {% if task.sasid_is_verified %}
+    <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
+ {% endif %}
+-->
 
  {% if task.activity.is_verified %}
     <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>

atdb/taskdatabase/urls.py

@@ -22,6 +22,7 @@ urlpatterns = [
     path('ingest', views.ShowIngestQPage.as_view(), name='ingest'),
     path('finished', views.ShowFinishedPage.as_view(), name='finished'),
 
+
     path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
     path('task_details/', views.TaskDetails, name='task-details'),
     path('task_quality/<int:id>/<page>', views.ShowTaskQuality, name='task-quality'),
@@ -129,6 +130,8 @@ urlpatterns = [
     path('tasks/<int:pk>/hold/<hold_it>/<page>', views.Hold, name='service-hold-resume'),
     path('tasks/<int:pk>/query-purge/<purge_policy>/<query_params>', views.PurgeQuery, name='query-purge'),
 
+    #some migration and repair endpoints
     path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
     path('tasks/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'),
+    path('tasks/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'),
 ]

atdb/taskdatabase/views.py

@@ -16,6 +16,8 @@ from django_tables2.views import SingleTableMixin
 
 from django.shortcuts import render, redirect, reverse
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from django.contrib.admin.views.decorators import staff_member_required
+
 from rest_framework.request import Request
 #from silk.profiling.profiler import silk_profile
 
@@ -103,6 +105,9 @@ class ActivityFilter(filters.FilterSet):
             'status': ['exact', 'icontains', 'in', 'startswith'],
             'ingestq_status': ['icontains'],
             'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'],
+            'finished_fraction': ['exact', 'lt', 'lte', 'gt', 'gte'],
+            'total_size': ['exact', 'lt', 'lte', 'gt', 'gte'],
+            'remaining': ['exact', 'lt', 'lte', 'gt', 'gte'],
         }
 
 
@@ -1671,21 +1676,55 @@ class GetUniqueValuesForKey(generics.ListAPIView):
                 'error': str(error)
             })
 
-
+@staff_member_required
 def AssociateActivities(request):
 
     tasks = Task.objects.all()
+    total = tasks.count()
+    i = 0
     for task in tasks:
+        i+=1
         activities.associate_task_with_activity(task)
-        logger.info(f'{task.id} => {task.sas_id}')
+        logger.info(f'{i} of {total}')
 
     return redirect('index')
 
+@staff_member_required
 def UpdateAllActivities(request):
 
-    tasks = Task.objects.all()
+    all_activities = Activity.objects.all()
+    # find a task for every activity
+    total = all_activities.count()
+    i = 0
+
+    for activity in all_activities:
+        try:
+            i += 1
+            task = Task.objects.filter(sas_id=activity.sas_id)[0]
+            activities.update_activity(task)
+            logger.info(f'{i} of {total}')
+        except Exception as error:
+            logger.error(error)
+
+    # tasks = Task.objects.all()
+    # total = tasks.count()
+    # i = 0
+    # for task in tasks:
+    #     i+=1
+    #     activities.update_activity(task)
+    #     logger.info(f'{i} of {total}')
+
+    return redirect('index')
+
+@staff_member_required
+def UpdateFailedTasks(request):
+
+    tasks = Task.objects.filter(status__icontains="failed")
+    total = tasks.count()
+    i = 0
     for task in tasks:
+        i+=1
         activities.update_activity(task)
-        logger.info(f'{task.id}')
+        logger.info(f'{i} of {total}')
 
     return redirect('index')
\ No newline at end of file