From 32e4a1c084098e8d4077ccc77ce0ecb1442364ef Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Mon, 12 Feb 2024 08:33:35 +0100
Subject: [PATCH] adapt unit tests
---
atdb/atdb/settings/dev.py | 3 +-
atdb/run.sh.example | 4 ---
.../0039_remove_activity_ingestq_status.py | 17 +++++++++
.../0040_activity_ingestq_status.py | 18 ++++++++++
atdb/taskdatabase/models.py | 2 +-
atdb/taskdatabase/services/activities.py | 8 ++---
.../templates/taskdatabase/index.html | 2 +-
.../tests/test_activities_associate.py | 0
atdb/taskdatabase/tests/test_filters.py | 19 ++++++----
.../tests/test_ingest_fraction.py | 14 ++++----
atdb/taskdatabase/tests/test_models_joins.py | 17 ++++-----
atdb/taskdatabase/tests/test_path_to_lta.py | 27 ++++++++------
atdb/taskdatabase/urls.py | 2 ++
atdb/taskdatabase/views.py | 36 +++++++++++++++++--
14 files changed, 123 insertions(+), 46 deletions(-)
delete mode 100644 atdb/run.sh.example
create mode 100644 atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py
create mode 100644 atdb/taskdatabase/migrations/0040_activity_ingestq_status.py
create mode 100644 atdb/taskdatabase/tests/test_activities_associate.py
diff --git a/atdb/atdb/settings/dev.py b/atdb/atdb/settings/dev.py
index 840632e9..d2998259 100644
--- a/atdb/atdb/settings/dev.py
+++ b/atdb/atdb/settings/dev.py
@@ -13,7 +13,8 @@ DATABASES = {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'atdb_admin',
'PASSWORD': 'atdb123',
- 'NAME': 'atdb_ldv_19jan2024',
+ #'NAME': 'atdb_ldv_19jan2024',
+ 'NAME': 'atdb_ldv_9feb2024',
'HOST': 'localhost',
'PORT': '5432',
},
diff --git a/atdb/run.sh.example b/atdb/run.sh.example
deleted file mode 100644
index 0679736b..00000000
--- a/atdb/run.sh.example
+++ /dev/null
@@ -1,4 +0,0 @@
-export KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-export KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-export KEYCLOAK_CLIENT_SECRET=
-python manage.py runserver --settings=atdb.settings.dev
diff --git a/atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py b/atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py
new file mode 100644
index 00000000..0badd85b
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.0 on 2024-02-10 07:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0038_remove_activity_priority_remove_activity_workflow_and_more'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='activity',
+ name='ingestq_status',
+ ),
+ ]
diff --git a/atdb/taskdatabase/migrations/0040_activity_ingestq_status.py b/atdb/taskdatabase/migrations/0040_activity_ingestq_status.py
new file mode 100644
index 00000000..d1b0e9c6
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0040_activity_ingestq_status.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2024-02-10 07:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0039_remove_activity_ingestq_status'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='activity',
+ name='ingestq_status',
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 5b40068b..877ad03b 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -100,7 +100,7 @@ class Activity(models.Model):
total_size = models.FloatField(blank=True, null=True)
remaining = models.FloatField(blank=True, null=True)
- ingestq_status = models.CharField(default="", max_length=100, blank=True, null=True)
+ ingestq_status = models.JSONField(null=True, blank=True)
@property
def has_archived(self):
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index 54508488..29ea569b 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -93,9 +93,9 @@ def update_activity(task):
Depending on the type of status change, certain calculations and updates are performed.
Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
- - to 'ARCHIVING, ARCHIVED' : check for incoming 'archive' json from archiver
+ - to 'ARCHIVING, ARCHIVED, FINISHED' : check for incoming/existing 'archive' json from archiver
- to STORED : calculate quality
- - to ??? : calculate finished_fraction
+ - to _FAILED : calculate finished_fraction
- to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
- always : calc 'verified'
@@ -126,7 +126,7 @@ def update_activity(task):
# check of any task of this activity already has LTA information. If so, copy to the activity level
- if task.status in [State.ARCHIVING.value, State.ARCHIVED.value]:
+ if task.status in [State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHED.value]:
logger.info(f'- add archive json')
for t in Task.objects.filter(sas_id=task.sas_id):
try:
@@ -141,7 +141,7 @@ def update_activity(task):
# calculate the finished fraction, this is only used on the Failures page
if 'failed' in task.status:
- logger.info(f'- calculate_ingested_fraction')
+ logger.info(f'- calculate_finished_fraction')
result = calculate_finished_fraction(task)
activity.finished_fraction = result['fraction']
activity.total_size = result['total_size']
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index f66028be..17e56897 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 2 Feb 2024
+ <p class="footer"> Version 9 Feb 2024
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/tests/test_activities_associate.py b/atdb/taskdatabase/tests/test_activities_associate.py
new file mode 100644
index 00000000..e69de29b
diff --git a/atdb/taskdatabase/tests/test_filters.py b/atdb/taskdatabase/tests/test_filters.py
index 4b1f3b42..01bb7a7f 100644
--- a/atdb/taskdatabase/tests/test_filters.py
+++ b/atdb/taskdatabase/tests/test_filters.py
@@ -1,17 +1,22 @@
from django.test import TestCase
from django.test import RequestFactory
from django.contrib.sessions.middleware import SessionMiddleware
-from taskdatabase.models import Task
+from taskdatabase.models import Task, Workflow, Activity
from taskdatabase.views import get_filtered_tasks
class FiltersTest(TestCase):
def setUp(self):
-
- self.task1 = Task.objects.create(sas_id=12345,status='defined')
- self.task2 = Task.objects.create(sas_id=12345,status='scrubbed')
- self.task3 = Task.objects.create(sas_id=12345,status='scrubbed')
- self.task4 = Task.objects.create(sas_id=66666,status='scrubbed')
- self.task5 = Task.objects.create(sas_id=66666,status='archived_failed')
+ self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ self.workflow_requantisation.save()
+ self.activity_12345 = Activity.objects.create(sas_id=12345)
+ self.activity_12345.save()
+ self.activity_66666 = Activity.objects.create(sas_id=66666)
+ self.activity_66666.save()
+ self.task1 = Task.objects.create(sas_id=12345, status='defined', workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task2 = Task.objects.create(sas_id=12345, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task3 = Task.objects.create(sas_id=12345, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task4 = Task.objects.create(sas_id=66666, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_66666)
+ self.task5 = Task.objects.create(sas_id=66666, status='archived_failed', workflow=self.workflow_requantisation, activity = self.activity_66666)
def test_without_filter(self):
count = 0
diff --git a/atdb/taskdatabase/tests/test_ingest_fraction.py b/atdb/taskdatabase/tests/test_ingest_fraction.py
index b50472c8..98807eea 100644
--- a/atdb/taskdatabase/tests/test_ingest_fraction.py
+++ b/atdb/taskdatabase/tests/test_ingest_fraction.py
@@ -1,17 +1,15 @@
from django.test import TestCase
-
+import json
from taskdatabase.models import Task, Workflow
class TestIngestFraction(TestCase):
def setUp(self):
# create a list of Tasks with various values of rfi_percent to test the quality algorithms
- workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation")
+ workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
workflow_requantisation.save()
Task.objects.get_or_create(filter='a',sas_id=54321, status='stored', workflow=workflow_requantisation)
- Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
- Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a',sas_id=54321, status='archiving', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a',sas_id=54321, status='archived', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a',sas_id=54321, status='finishing', workflow=workflow_requantisation)
@@ -19,14 +17,18 @@ class TestIngestFraction(TestCase):
Task.objects.get_or_create(filter='b',sas_id=54321, status='finished', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a', sas_id=54321, status='discarded', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a', sas_id=54321, status='archived_failed', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+
def test_ingest_fraction(self):
# collapse all tasks into a single task for this sas_id
task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0]
+ task.save()
# get the list of statusses and level of completion
- statusses = task.sasid_ingested_fraction['status']
- completion = task.sasid_ingested_fraction['completion']
+ statusses = task.activity.ingestq_status
+ completion = task.activity.ingested_fraction
self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1})
self.assertEqual(completion,38)
diff --git a/atdb/taskdatabase/tests/test_models_joins.py b/atdb/taskdatabase/tests/test_models_joins.py
index de89551b..6b50c270 100644
--- a/atdb/taskdatabase/tests/test_models_joins.py
+++ b/atdb/taskdatabase/tests/test_models_joins.py
@@ -1,7 +1,7 @@
from django.test import TestCase
import json
-from taskdatabase.models import Workflow,Task
+from taskdatabase.models import Workflow,Task, Activity
class TestJoinedTasks(TestCase):
@@ -9,15 +9,16 @@ class TestJoinedTasks(TestCase):
@classmethod
def setUpTestData(cls):
- # Set up non-modified objects used by all test methods
- workflow = Workflow()
- workflow.save()
+ workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ workflow_requantisation.save()
+ activity_12345 = Activity.objects.create(sas_id=12345)
+ activity_12345.save()
# create a list of Tasks
- Task.objects.get_or_create(sas_id=1, status='stored')
- Task.objects.get_or_create(sas_id=2, status='stored')
- Task.objects.get_or_create(sas_id=3, status='defined')
- Task.objects.get_or_create(sas_id=4, status='defined')
+ Task.objects.get_or_create(sas_id=1, status='stored', workflow=workflow_requantisation, activity = activity_12345)
+ Task.objects.get_or_create(sas_id=2, status='stored',workflow=workflow_requantisation, activity = activity_12345)
+ Task.objects.get_or_create(sas_id=3, status='defined',workflow=workflow_requantisation, activity = activity_12345)
+ Task.objects.get_or_create(sas_id=4, status='defined',workflow=workflow_requantisation, activity = activity_12345)
def test_add_input_tasks_to_task(self):
output_task = Task.objects.get(sas_id=1)
diff --git a/atdb/taskdatabase/tests/test_path_to_lta.py b/atdb/taskdatabase/tests/test_path_to_lta.py
index 38d884bd..3dc2d0e3 100644
--- a/atdb/taskdatabase/tests/test_path_to_lta.py
+++ b/atdb/taskdatabase/tests/test_path_to_lta.py
@@ -1,35 +1,40 @@
from django.test import TestCase
-from taskdatabase.models import Task
+from taskdatabase.models import Task, Workflow, Activity
class PathToLTATest(TestCase):
def setUp(self):
- # Create tasks for testing
+ self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ self.workflow_requantisation.save()
+ self.activity_12345 = Activity.objects.create(sas_id=12345, archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321})
+ self.activity_12345.save()
+ self.activity_66666 = Activity.objects.create(sas_id=66666, archive={})
+ self.activity_66666.save()
# the first 2 have no valid path set
- self.task1 = Task.objects.create(sas_id=12345,archive={})
- self.task2 = Task.objects.create(sas_id=12345,archive={'path_to_lta': None})
+ self.task1 = Task.objects.create(sas_id=12345,archive={}, workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task2 = Task.objects.create(sas_id=12345,archive={'path_to_lta': None}, workflow=self.workflow_requantisation, activity = self.activity_12345)
# this task has a valid path_to_lta set
- self.task3 = Task.objects.create(sas_id=12345,archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321})
+ self.task3 = Task.objects.create(sas_id=12345,archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321}, workflow=self.workflow_requantisation, activity = self.activity_12345)
# this sasid has no path_to_lta set at all
- self.task4 = Task.objects.create(sas_id=66666,archive={})
- self.task5 = Task.objects.create(sas_id=66666,archive={})
+ self.task4 = Task.objects.create(sas_id=66666,archive={}, workflow=self.workflow_requantisation, activity = self.activity_66666)
+ self.task5 = Task.objects.create(sas_id=66666,archive={}, workflow=self.workflow_requantisation, activity = self.activity_66666)
def test_path_to_lta_with_path(self):
# if only one of the tasks has a path_to_lta, then the other tasks should also return that path
for task in Task.objects.filter(sas_id=12345):
- result = task.sasid_path_to_lta
+ result = task.activity.archive['path_to_lta']
self.assertEqual(result, '/sample/path')
def test_path_to_lta_without_path(self):
# if one of the tasks has 'path_to_lta' set, then return None
for task in Task.objects.filter(sas_id=66666):
- result = task.sasid_path_to_lta
+ result = task.path_to_lta
self.assertEqual(result, None)
- def test_sas_id_has_archived(self):
+ def test_has_archived(self):
# if only one of the tasks has a sas_id_has_archived, then the other tasks should also return that path
for task in Task.objects.filter(sas_id=12345):
- result = task.sas_id_has_archived
+ result = task.activity.has_archived
self.assertEqual(result, 54321)
\ No newline at end of file
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 11c65788..0acca01c 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -134,4 +134,6 @@ urlpatterns = [
path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
path('tasks/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'),
path('tasks/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'),
+ path('tasks/update-ingestq-tasks/', views.UpdateIngestQTasks, name='update-ingestq-tasks'),
+ path('tasks/update-finished-tasks/', views.UpdateFinishedTasks, name='update-finished-tasks'),
]
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 9c5cf32d..d32fae34 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -104,7 +104,7 @@ class ActivityFilter(filters.FilterSet):
'project': ['exact', 'icontains'],
'sas_id': ['exact', 'icontains', 'in'],
'status': ['exact', 'icontains', 'in', 'startswith'],
- 'ingestq_status': ['icontains'],
+ #'ingestq_status': ['icontains'],
'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'],
'finished_fraction': ['exact', 'lt', 'lte', 'gt', 'gte'],
'total_size': ['exact', 'lt', 'lte', 'gt', 'gte'],
@@ -1680,12 +1680,13 @@ class GetUniqueValuesForKey(generics.ListAPIView):
@staff_member_required
def AssociateActivities(request):
- tasks = Task.objects.all()
+ tasks = Task.objects.all().only('sas_id')
total = tasks.count()
i = 0
for task in tasks:
i+=1
- activities.associate_task_with_activity(task)
+ if task.status not in ['discarded', 'suspended']:
+ activities.associate_task_with_activity(task)
logger.info(f'{i} of {total}')
return redirect('index')
@@ -1728,4 +1729,33 @@ def UpdateFailedTasks(request):
activities.update_activity(task)
logger.info(f'{i} of {total}')
+ return redirect('index')
+
+@staff_member_required
+def UpdateIngestQTasks(request):
+ tasks = Task.objects.only('sas_id').filter(
+ Q(status__icontains=State.SCRUBBED.value) |
+ Q(status__icontains=State.ARCHIVING.value) |
+ Q(status__icontains=State.ARCHIVED.value) |
+ Q(status__icontains=State.FINISHING.value))
+
+ total = tasks.count()
+ i = 0
+ for task in tasks:
+ i+=1
+ activities.update_activity(task)
+ logger.info(f'{i} of {total}')
+
+ return redirect('index')
+
+@staff_member_required
+def UpdateFinishedTasks(request):
+ tasks = Task.objects.only('sas_id').filter(status=State.FINISHED.value)
+ total = tasks.count()
+ i = 0
+ for task in tasks:
+ i+=1
+ activities.update_activity(task)
+ logger.info(f'{i} of {total}')
+
return redirect('index')
\ No newline at end of file
--
GitLab