adapt unit tests

atdb/atdb/settings/dev.py

@@ -13,7 +13,8 @@ DATABASES = {
          'ENGINE': 'django.db.backends.postgresql_psycopg2',
          'USER': 'atdb_admin',
          'PASSWORD': 'atdb123',
-         'NAME': 'atdb_ldv_19jan2024',
+         #'NAME': 'atdb_ldv_19jan2024',
+         'NAME': 'atdb_ldv_9feb2024',
          'HOST': 'localhost',
          'PORT': '5432',
     },

atdb/run.sh.example

-export KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-export KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-export KEYCLOAK_CLIENT_SECRET=
-python manage.py runserver --settings=atdb.settings.dev

atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py

+# Generated by Django 5.0 on 2024-02-10 07:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0038_remove_activity_priority_remove_activity_workflow_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='activity',
+            name='ingestq_status',
+        ),
+    ]

atdb/taskdatabase/migrations/0040_activity_ingestq_status.py

+# Generated by Django 5.0 on 2024-02-10 07:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taskdatabase', '0039_remove_activity_ingestq_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='activity',
+            name='ingestq_status',
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

atdb/taskdatabase/models.py

@@ -100,7 +100,7 @@ class Activity(models.Model):
     total_size = models.FloatField(blank=True, null=True)
     remaining = models.FloatField(blank=True, null=True)
 
-    ingestq_status = models.CharField(default="", max_length=100, blank=True, null=True)
+    ingestq_status = models.JSONField(null=True, blank=True)
 
     @property
     def has_archived(self):

atdb/taskdatabase/services/activities.py

@@ -93,9 +93,9 @@ def update_activity(task):
     Depending on the type of status change, certain calculations and updates are performed.
     Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
 
-    - to 'ARCHIVING, ARCHIVED'                   : check for incoming 'archive' json from archiver
+    - to 'ARCHIVING, ARCHIVED, FINISHED'         : check for incoming/existing 'archive' json from archiver
     - to STORED                                  : calculate quality
-    - to ???                                     : calculate finished_fraction
+    - to _FAILED                                 : calculate finished_fraction
     - to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
     - always                                     : calc 'verified'
 
@@ -126,7 +126,7 @@ def update_activity(task):
 
 
     # check of any task of this activity already has LTA information. If so, copy to the activity level
-    if task.status in [State.ARCHIVING.value, State.ARCHIVED.value]:
+    if task.status in [State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHED.value]:
         logger.info(f'- add archive json')
         for t in Task.objects.filter(sas_id=task.sas_id):
             try:
@@ -141,7 +141,7 @@ def update_activity(task):
 
     # calculate the finished fraction, this is only used on the Failures page
     if 'failed' in task.status:
-        logger.info(f'- calculate_ingested_fraction')
+        logger.info(f'- calculate_finished_fraction')
         result = calculate_finished_fraction(task)
         activity.finished_fraction = result['fraction']
         activity.total_size = result['total_size']

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 2 Feb 2024
+    <p class="footer"> Version 9 Feb 2024
 </div>
 
 {% include 'taskdatabase/refresh.html' %}

atdb/taskdatabase/tests/test_filters.py

 from django.test import TestCase
 from django.test import RequestFactory
 from django.contrib.sessions.middleware import SessionMiddleware
-from taskdatabase.models import Task
+from taskdatabase.models import Task, Workflow, Activity
 from taskdatabase.views import get_filtered_tasks
 
 class FiltersTest(TestCase):
     def setUp(self):
-
-        self.task1 = Task.objects.create(sas_id=12345,status='defined')
-        self.task2 = Task.objects.create(sas_id=12345,status='scrubbed')
-        self.task3 = Task.objects.create(sas_id=12345,status='scrubbed')
-        self.task4 = Task.objects.create(sas_id=66666,status='scrubbed')
-        self.task5 = Task.objects.create(sas_id=66666,status='archived_failed')
+        self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+        self.workflow_requantisation.save()
+        self.activity_12345 = Activity.objects.create(sas_id=12345)
+        self.activity_12345.save()
+        self.activity_66666 = Activity.objects.create(sas_id=66666)
+        self.activity_66666.save()
+        self.task1 = Task.objects.create(sas_id=12345, status='defined', workflow=self.workflow_requantisation, activity = self.activity_12345)
+        self.task2 = Task.objects.create(sas_id=12345, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_12345)
+        self.task3 = Task.objects.create(sas_id=12345, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_12345)
+        self.task4 = Task.objects.create(sas_id=66666, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_66666)
+        self.task5 = Task.objects.create(sas_id=66666, status='archived_failed', workflow=self.workflow_requantisation, activity = self.activity_66666)
 
     def test_without_filter(self):
         count = 0

atdb/taskdatabase/tests/test_ingest_fraction.py

 from django.test import TestCase
-
+import json
 from taskdatabase.models import Task, Workflow
 
 class TestIngestFraction(TestCase):
 
     def setUp(self):
         # create a list of Tasks with various values of rfi_percent to test the quality algorithms
-        workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation")
+        workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
         workflow_requantisation.save()
 
         Task.objects.get_or_create(filter='a',sas_id=54321, status='stored', workflow=workflow_requantisation)
-        Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
-        Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
         Task.objects.get_or_create(filter='a',sas_id=54321, status='archiving', workflow=workflow_requantisation)
         Task.objects.get_or_create(filter='a',sas_id=54321, status='archived', workflow=workflow_requantisation)
         Task.objects.get_or_create(filter='a',sas_id=54321, status='finishing', workflow=workflow_requantisation)
@@ -19,14 +17,18 @@ class TestIngestFraction(TestCase):
         Task.objects.get_or_create(filter='b',sas_id=54321, status='finished', workflow=workflow_requantisation)
         Task.objects.get_or_create(filter='a', sas_id=54321, status='discarded', workflow=workflow_requantisation)
         Task.objects.get_or_create(filter='a', sas_id=54321, status='archived_failed', workflow=workflow_requantisation)
+        Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+        Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+
     def test_ingest_fraction(self):
 
         # collapse all tasks into a single task for this sas_id
         task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0]
+        task.save()
 
         # get the list of statusses and level of completion
-        statusses = task.sasid_ingested_fraction['status']
-        completion = task.sasid_ingested_fraction['completion']
+        statusses = task.activity.ingestq_status
+        completion = task.activity.ingested_fraction
 
         self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1})
         self.assertEqual(completion,38)

atdb/taskdatabase/tests/test_models_joins.py

 from django.test import TestCase
 import json
 
-from taskdatabase.models import Workflow,Task
+from taskdatabase.models import Workflow,Task, Activity
 
 
 class TestJoinedTasks(TestCase):
@@ -9,15 +9,16 @@ class TestJoinedTasks(TestCase):
     @classmethod
     def setUpTestData(cls):
 
-        # Set up non-modified objects used by all test methods
-        workflow = Workflow()
-        workflow.save()
+        workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+        workflow_requantisation.save()
+        activity_12345 = Activity.objects.create(sas_id=12345)
+        activity_12345.save()
 
         # create a list of Tasks
-        Task.objects.get_or_create(sas_id=1, status='stored')
-        Task.objects.get_or_create(sas_id=2, status='stored')
-        Task.objects.get_or_create(sas_id=3, status='defined')
-        Task.objects.get_or_create(sas_id=4, status='defined')
+        Task.objects.get_or_create(sas_id=1, status='stored', workflow=workflow_requantisation, activity = activity_12345)
+        Task.objects.get_or_create(sas_id=2, status='stored',workflow=workflow_requantisation, activity = activity_12345)
+        Task.objects.get_or_create(sas_id=3, status='defined',workflow=workflow_requantisation, activity = activity_12345)
+        Task.objects.get_or_create(sas_id=4, status='defined',workflow=workflow_requantisation, activity = activity_12345)
 
     def test_add_input_tasks_to_task(self):
         output_task = Task.objects.get(sas_id=1)

atdb/taskdatabase/tests/test_path_to_lta.py

 from django.test import TestCase
-from taskdatabase.models import Task
+from taskdatabase.models import Task, Workflow, Activity
 
 class PathToLTATest(TestCase):
     def setUp(self):
-        # Create tasks for testing
+        self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+        self.workflow_requantisation.save()
+        self.activity_12345 = Activity.objects.create(sas_id=12345, archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321})
+        self.activity_12345.save()
+        self.activity_66666 = Activity.objects.create(sas_id=66666, archive={})
+        self.activity_66666.save()
 
         # the first 2 have no valid path set
-        self.task1 = Task.objects.create(sas_id=12345,archive={})
-        self.task2 = Task.objects.create(sas_id=12345,archive={'path_to_lta': None})
+        self.task1 = Task.objects.create(sas_id=12345,archive={}, workflow=self.workflow_requantisation, activity = self.activity_12345)
+        self.task2 = Task.objects.create(sas_id=12345,archive={'path_to_lta': None}, workflow=self.workflow_requantisation, activity = self.activity_12345)
 
         # this task has a valid path_to_lta set
-        self.task3 = Task.objects.create(sas_id=12345,archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321})
+        self.task3 = Task.objects.create(sas_id=12345,archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321}, workflow=self.workflow_requantisation, activity = self.activity_12345)
 
         # this sasid has no path_to_lta set at all
-        self.task4 = Task.objects.create(sas_id=66666,archive={})
-        self.task5 = Task.objects.create(sas_id=66666,archive={})
+        self.task4 = Task.objects.create(sas_id=66666,archive={}, workflow=self.workflow_requantisation, activity = self.activity_66666)
+        self.task5 = Task.objects.create(sas_id=66666,archive={}, workflow=self.workflow_requantisation, activity = self.activity_66666)
 
     def test_path_to_lta_with_path(self):
         # if only one of the tasks has a path_to_lta, then the other tasks should also return that path
         for task in Task.objects.filter(sas_id=12345):
-            result = task.sasid_path_to_lta
+            result = task.activity.archive['path_to_lta']
             self.assertEqual(result, '/sample/path')
 
     def test_path_to_lta_without_path(self):
         # if one of the tasks has 'path_to_lta' set, then return None
         for task in Task.objects.filter(sas_id=66666):
-            result = task.sasid_path_to_lta
+            result = task.path_to_lta
             self.assertEqual(result, None)
 
-    def test_sas_id_has_archived(self):
+    def test_has_archived(self):
         # if only one of the tasks has a sas_id_has_archived, then the other tasks should also return that path
         for task in Task.objects.filter(sas_id=12345):
-            result = task.sas_id_has_archived
+            result = task.activity.has_archived
             self.assertEqual(result, 54321)
\ No newline at end of file

atdb/taskdatabase/urls.py

@@ -134,4 +134,6 @@ urlpatterns = [
     path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
     path('tasks/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'),
     path('tasks/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'),
+    path('tasks/update-ingestq-tasks/', views.UpdateIngestQTasks, name='update-ingestq-tasks'),
+    path('tasks/update-finished-tasks/', views.UpdateFinishedTasks, name='update-finished-tasks'),
 ]

atdb/taskdatabase/views.py

@@ -104,7 +104,7 @@ class ActivityFilter(filters.FilterSet):
             'project': ['exact', 'icontains'],
             'sas_id': ['exact', 'icontains', 'in'],
             'status': ['exact', 'icontains', 'in', 'startswith'],
-            'ingestq_status': ['icontains'],
+            #'ingestq_status': ['icontains'],
             'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'],
             'finished_fraction': ['exact', 'lt', 'lte', 'gt', 'gte'],
             'total_size': ['exact', 'lt', 'lte', 'gt', 'gte'],
@@ -1680,12 +1680,13 @@ class GetUniqueValuesForKey(generics.ListAPIView):
 @staff_member_required
 def AssociateActivities(request):
 
-    tasks = Task.objects.all()
+    tasks = Task.objects.all().only('sas_id')
     total = tasks.count()
     i = 0
     for task in tasks:
         i+=1
-        activities.associate_task_with_activity(task)
+        if task.status not in ['discarded', 'suspended']:
+            activities.associate_task_with_activity(task)
         logger.info(f'{i} of {total}')
 
     return redirect('index')
@@ -1728,4 +1729,33 @@ def UpdateFailedTasks(request):
         activities.update_activity(task)
         logger.info(f'{i} of {total}')
 
+    return redirect('index')
+
+@staff_member_required
+def UpdateIngestQTasks(request):
+    tasks = Task.objects.only('sas_id').filter(
+        Q(status__icontains=State.SCRUBBED.value) |
+        Q(status__icontains=State.ARCHIVING.value) |
+        Q(status__icontains=State.ARCHIVED.value) |
+        Q(status__icontains=State.FINISHING.value))
+
+    total = tasks.count()
+    i = 0
+    for task in tasks:
+        i+=1
+        activities.update_activity(task)
+        logger.info(f'{i} of {total}')
+
+    return redirect('index')
+
+@staff_member_required
+def UpdateFinishedTasks(request):
+    tasks = Task.objects.only('sas_id').filter(status=State.FINISHED.value)
+    total = tasks.count()
+    i = 0
+    for task in tasks:
+        i+=1
+        activities.update_activity(task)
+        logger.info(f'{i} of {total}')
+
     return redirect('index')
\ No newline at end of file