diff --git a/atdb/atdb/settings/dev.py b/atdb/atdb/settings/dev.py
index 840632e94242e51880032bdeb394a60ae70f1b1f..d2998259833ca63c52795f8d26fd788df57d5436 100644
--- a/atdb/atdb/settings/dev.py
+++ b/atdb/atdb/settings/dev.py
@@ -13,7 +13,8 @@ DATABASES = {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'atdb_admin',
'PASSWORD': 'atdb123',
- 'NAME': 'atdb_ldv_19jan2024',
+ #'NAME': 'atdb_ldv_19jan2024',
+ 'NAME': 'atdb_ldv_9feb2024',
'HOST': 'localhost',
'PORT': '5432',
},
diff --git a/atdb/run.sh.example b/atdb/run.sh.example
deleted file mode 100644
index 0679736b557e35f12110609d48356b5a6e0a4c05..0000000000000000000000000000000000000000
--- a/atdb/run.sh.example
+++ /dev/null
@@ -1,4 +0,0 @@
-export KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-export KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-export KEYCLOAK_CLIENT_SECRET=
-python manage.py runserver --settings=atdb.settings.dev
diff --git a/atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py b/atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py
new file mode 100644
index 0000000000000000000000000000000000000000..0badd85b19e746b7233ae809f543bb852963568e
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0039_remove_activity_ingestq_status.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.0 on 2024-02-10 07:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0038_remove_activity_priority_remove_activity_workflow_and_more'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='activity',
+ name='ingestq_status',
+ ),
+ ]
diff --git a/atdb/taskdatabase/migrations/0040_activity_ingestq_status.py b/atdb/taskdatabase/migrations/0040_activity_ingestq_status.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1b0e9c62553908e1d049358aa47c769538db594
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0040_activity_ingestq_status.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2024-02-10 07:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0039_remove_activity_ingestq_status'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='activity',
+ name='ingestq_status',
+ field=models.JSONField(blank=True, null=True),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 5b40068b10024192a3ba5306e0409aab03c100a7..877ad03b426556e23ec87e36eb223d2ecc7941d7 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -100,7 +100,7 @@ class Activity(models.Model):
total_size = models.FloatField(blank=True, null=True)
remaining = models.FloatField(blank=True, null=True)
- ingestq_status = models.CharField(default="", max_length=100, blank=True, null=True)
+ ingestq_status = models.JSONField(null=True, blank=True)
@property
def has_archived(self):
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index 545084888b61398e55b0669a84a7ca79c594b402..29ea569b4cccbc062c68f9b2b23793ba9d05f0d0 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -93,9 +93,9 @@ def update_activity(task):
Depending on the type of status change, certain calculations and updates are performed.
Doing this on status change, instead of on-the-fly when a user enters a page, balances the load.
- - to 'ARCHIVING, ARCHIVED' : check for incoming 'archive' json from archiver
+ - to 'ARCHIVING, ARCHIVED, FINISHED' : check for incoming/existing 'archive' json from archiver
- to STORED : calculate quality
- - to ??? : calculate finished_fraction
+ - to _FAILED : calculate finished_fraction
- to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
- always : calc 'verified'
@@ -126,7 +126,7 @@ def update_activity(task):
# check of any task of this activity already has LTA information. If so, copy to the activity level
- if task.status in [State.ARCHIVING.value, State.ARCHIVED.value]:
+ if task.status in [State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHED.value]:
logger.info(f'- add archive json')
for t in Task.objects.filter(sas_id=task.sas_id):
try:
@@ -141,7 +141,7 @@ def update_activity(task):
# calculate the finished fraction, this is only used on the Failures page
if 'failed' in task.status:
- logger.info(f'- calculate_ingested_fraction')
+ logger.info(f'- calculate_finished_fraction')
result = calculate_finished_fraction(task)
activity.finished_fraction = result['fraction']
activity.total_size = result['total_size']
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index f66028be5102df9eb21f2628b393c22ba763b5a6..17e56897798489407000c06e2f55d11cd429fe3e 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 2 Feb 2024
+ <p class="footer"> Version 9 Feb 2024
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/tests/test_activities_associate.py b/atdb/taskdatabase/tests/test_activities_associate.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/atdb/taskdatabase/tests/test_filters.py b/atdb/taskdatabase/tests/test_filters.py
index 4b1f3b42854075e4e7514c5dd0cee6acab92e419..01bb7a7f372f2a58227acc81f71bfe9261ced775 100644
--- a/atdb/taskdatabase/tests/test_filters.py
+++ b/atdb/taskdatabase/tests/test_filters.py
@@ -1,17 +1,22 @@
from django.test import TestCase
from django.test import RequestFactory
from django.contrib.sessions.middleware import SessionMiddleware
-from taskdatabase.models import Task
+from taskdatabase.models import Task, Workflow, Activity
from taskdatabase.views import get_filtered_tasks
class FiltersTest(TestCase):
def setUp(self):
-
- self.task1 = Task.objects.create(sas_id=12345,status='defined')
- self.task2 = Task.objects.create(sas_id=12345,status='scrubbed')
- self.task3 = Task.objects.create(sas_id=12345,status='scrubbed')
- self.task4 = Task.objects.create(sas_id=66666,status='scrubbed')
- self.task5 = Task.objects.create(sas_id=66666,status='archived_failed')
+ self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ self.workflow_requantisation.save()
+ self.activity_12345 = Activity.objects.create(sas_id=12345)
+ self.activity_12345.save()
+ self.activity_66666 = Activity.objects.create(sas_id=66666)
+ self.activity_66666.save()
+ self.task1 = Task.objects.create(sas_id=12345, status='defined', workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task2 = Task.objects.create(sas_id=12345, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task3 = Task.objects.create(sas_id=12345, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task4 = Task.objects.create(sas_id=66666, status='scrubbed', workflow=self.workflow_requantisation, activity = self.activity_66666)
+ self.task5 = Task.objects.create(sas_id=66666, status='archived_failed', workflow=self.workflow_requantisation, activity = self.activity_66666)
def test_without_filter(self):
count = 0
diff --git a/atdb/taskdatabase/tests/test_ingest_fraction.py b/atdb/taskdatabase/tests/test_ingest_fraction.py
index b50472c8eb86a5b8f03c9729dba2e1d48471e675..98807eea173affaef8f7914d5e619abec4ee8268 100644
--- a/atdb/taskdatabase/tests/test_ingest_fraction.py
+++ b/atdb/taskdatabase/tests/test_ingest_fraction.py
@@ -1,17 +1,15 @@
from django.test import TestCase
-
+import json
from taskdatabase.models import Task, Workflow
class TestIngestFraction(TestCase):
def setUp(self):
# create a list of Tasks with various values of rfi_percent to test the quality algorithms
- workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation")
+ workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
workflow_requantisation.save()
Task.objects.get_or_create(filter='a',sas_id=54321, status='stored', workflow=workflow_requantisation)
- Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
- Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a',sas_id=54321, status='archiving', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a',sas_id=54321, status='archived', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a',sas_id=54321, status='finishing', workflow=workflow_requantisation)
@@ -19,14 +17,18 @@ class TestIngestFraction(TestCase):
Task.objects.get_or_create(filter='b',sas_id=54321, status='finished', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a', sas_id=54321, status='discarded', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='a', sas_id=54321, status='archived_failed', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+
def test_ingest_fraction(self):
# collapse all tasks into a single task for this sas_id
task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0]
+ task.save()
# get the list of statusses and level of completion
- statusses = task.sasid_ingested_fraction['status']
- completion = task.sasid_ingested_fraction['completion']
+ statusses = task.activity.ingestq_status
+ completion = task.activity.ingested_fraction
self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1})
self.assertEqual(completion,38)
diff --git a/atdb/taskdatabase/tests/test_models_joins.py b/atdb/taskdatabase/tests/test_models_joins.py
index de89551bb2728545889965b377eabcd3600de9e0..6b50c2702779798aa0348dc495c806297b3a9e43 100644
--- a/atdb/taskdatabase/tests/test_models_joins.py
+++ b/atdb/taskdatabase/tests/test_models_joins.py
@@ -1,7 +1,7 @@
from django.test import TestCase
import json
-from taskdatabase.models import Workflow,Task
+from taskdatabase.models import Workflow,Task, Activity
class TestJoinedTasks(TestCase):
@@ -9,15 +9,16 @@ class TestJoinedTasks(TestCase):
@classmethod
def setUpTestData(cls):
- # Set up non-modified objects used by all test methods
- workflow = Workflow()
- workflow.save()
+ workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ workflow_requantisation.save()
+ activity_12345 = Activity.objects.create(sas_id=12345)
+ activity_12345.save()
# create a list of Tasks
- Task.objects.get_or_create(sas_id=1, status='stored')
- Task.objects.get_or_create(sas_id=2, status='stored')
- Task.objects.get_or_create(sas_id=3, status='defined')
- Task.objects.get_or_create(sas_id=4, status='defined')
+ Task.objects.get_or_create(sas_id=1, status='stored', workflow=workflow_requantisation, activity = activity_12345)
+ Task.objects.get_or_create(sas_id=2, status='stored',workflow=workflow_requantisation, activity = activity_12345)
+ Task.objects.get_or_create(sas_id=3, status='defined',workflow=workflow_requantisation, activity = activity_12345)
+ Task.objects.get_or_create(sas_id=4, status='defined',workflow=workflow_requantisation, activity = activity_12345)
def test_add_input_tasks_to_task(self):
output_task = Task.objects.get(sas_id=1)
diff --git a/atdb/taskdatabase/tests/test_path_to_lta.py b/atdb/taskdatabase/tests/test_path_to_lta.py
index 38d884bddde566427ac47cd96696c9adfedbe173..3dc2d0e3109b7f21a7e1fe84bf0aa4bbaf5e4b57 100644
--- a/atdb/taskdatabase/tests/test_path_to_lta.py
+++ b/atdb/taskdatabase/tests/test_path_to_lta.py
@@ -1,35 +1,40 @@
from django.test import TestCase
-from taskdatabase.models import Task
+from taskdatabase.models import Task, Workflow, Activity
class PathToLTATest(TestCase):
def setUp(self):
- # Create tasks for testing
+ self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
+ self.workflow_requantisation.save()
+ self.activity_12345 = Activity.objects.create(sas_id=12345, archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321})
+ self.activity_12345.save()
+ self.activity_66666 = Activity.objects.create(sas_id=66666, archive={})
+ self.activity_66666.save()
# the first 2 have no valid path set
- self.task1 = Task.objects.create(sas_id=12345,archive={})
- self.task2 = Task.objects.create(sas_id=12345,archive={'path_to_lta': None})
+ self.task1 = Task.objects.create(sas_id=12345,archive={}, workflow=self.workflow_requantisation, activity = self.activity_12345)
+ self.task2 = Task.objects.create(sas_id=12345,archive={'path_to_lta': None}, workflow=self.workflow_requantisation, activity = self.activity_12345)
# this task has a valid path_to_lta set
- self.task3 = Task.objects.create(sas_id=12345,archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321})
+ self.task3 = Task.objects.create(sas_id=12345,archive={'path_to_lta': '/sample/path', 'sas_id_archived': 54321}, workflow=self.workflow_requantisation, activity = self.activity_12345)
# this sasid has no path_to_lta set at all
- self.task4 = Task.objects.create(sas_id=66666,archive={})
- self.task5 = Task.objects.create(sas_id=66666,archive={})
+ self.task4 = Task.objects.create(sas_id=66666,archive={}, workflow=self.workflow_requantisation, activity = self.activity_66666)
+ self.task5 = Task.objects.create(sas_id=66666,archive={}, workflow=self.workflow_requantisation, activity = self.activity_66666)
def test_path_to_lta_with_path(self):
# if only one of the tasks has a path_to_lta, then the other tasks should also return that path
for task in Task.objects.filter(sas_id=12345):
- result = task.sasid_path_to_lta
+ result = task.activity.archive['path_to_lta']
self.assertEqual(result, '/sample/path')
def test_path_to_lta_without_path(self):
# if one of the tasks has 'path_to_lta' set, then return None
for task in Task.objects.filter(sas_id=66666):
- result = task.sasid_path_to_lta
+ result = task.path_to_lta
self.assertEqual(result, None)
- def test_sas_id_has_archived(self):
+ def test_has_archived(self):
# if only one of the tasks has a sas_id_has_archived, then the other tasks should also return that path
for task in Task.objects.filter(sas_id=12345):
- result = task.sas_id_has_archived
+ result = task.activity.has_archived
self.assertEqual(result, 54321)
\ No newline at end of file
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 11c65788f2e5cb7d98b8c7ab0bd82386db4a9846..0acca01c37cea68132cceeb756aac8bc710880b8 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -134,4 +134,6 @@ urlpatterns = [
path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
path('tasks/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'),
path('tasks/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'),
+ path('tasks/update-ingestq-tasks/', views.UpdateIngestQTasks, name='update-ingestq-tasks'),
+ path('tasks/update-finished-tasks/', views.UpdateFinishedTasks, name='update-finished-tasks'),
]
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 9c5cf32d8e1f01a6772c7ae7d73eb3b1e22ca6c8..d32fae347e0ea0b8483411cb0ad42d632c1d5fe8 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -104,7 +104,7 @@ class ActivityFilter(filters.FilterSet):
'project': ['exact', 'icontains'],
'sas_id': ['exact', 'icontains', 'in'],
'status': ['exact', 'icontains', 'in', 'startswith'],
- 'ingestq_status': ['icontains'],
+ #'ingestq_status': ['icontains'],
'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'],
'finished_fraction': ['exact', 'lt', 'lte', 'gt', 'gte'],
'total_size': ['exact', 'lt', 'lte', 'gt', 'gte'],
@@ -1680,12 +1680,13 @@ class GetUniqueValuesForKey(generics.ListAPIView):
@staff_member_required
def AssociateActivities(request):
- tasks = Task.objects.all()
+ tasks = Task.objects.all().only('sas_id')
total = tasks.count()
i = 0
for task in tasks:
i+=1
- activities.associate_task_with_activity(task)
+ if task.status not in ['discarded', 'suspended']:
+ activities.associate_task_with_activity(task)
logger.info(f'{i} of {total}')
return redirect('index')
@@ -1728,4 +1729,33 @@ def UpdateFailedTasks(request):
activities.update_activity(task)
logger.info(f'{i} of {total}')
+ return redirect('index')
+
+@staff_member_required
+def UpdateIngestQTasks(request):
+ tasks = Task.objects.only('sas_id').filter(
+ Q(status__icontains=State.SCRUBBED.value) |
+ Q(status__icontains=State.ARCHIVING.value) |
+ Q(status__icontains=State.ARCHIVED.value) |
+ Q(status__icontains=State.FINISHING.value))
+
+ total = tasks.count()
+ i = 0
+ for task in tasks:
+ i+=1
+ activities.update_activity(task)
+ logger.info(f'{i} of {total}')
+
+ return redirect('index')
+
+@staff_member_required
+def UpdateFinishedTasks(request):
+ tasks = Task.objects.only('sas_id').filter(status=State.FINISHED.value)
+ total = tasks.count()
+ i = 0
+ for task in tasks:
+ i+=1
+ activities.update_activity(task)
+ logger.info(f'{i} of {total}')
+
return redirect('index')
\ No newline at end of file