From 5c73dc0786fcea3f8c45d6bef10d1d5b852b8a12 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Mon, 15 Apr 2024 17:13:22 +0200
Subject: [PATCH] handling review comments
---
atdb/dumpdata_configuration.bat | 2 ++
atdb/dumpdata_workflows.bat | 2 ++
atdb/loaddata_configuration.bat | 2 ++
atdb/loaddata_workflow.bat | 2 ++
atdb/makemigrations.bat | 1 +
atdb/migrate.bat | 1 +
atdb/requirements/base.txt | 1 -
atdb/rollback.bat | 1 +
atdb/superuser.bat | 1 +
atdb/taskdatabase/models.py | 13 +-----------
.../services/activities_handler.py | 1 +
.../tests/test_models_processed_summary.py | 16 +++++++-------
atdb/taskdatabase/tests/test_summary_tasks.py | 21 +++++++------------
.../tests/test_update_activity.py | 2 +-
atdb/test-latest.bat | 1 +
atdb/test.bat | 1 +
atdb/test_local.bat | 1 -
17 files changed, 32 insertions(+), 37 deletions(-)
delete mode 100644 atdb/test_local.bat
diff --git a/atdb/dumpdata_configuration.bat b/atdb/dumpdata_configuration.bat
index a017b5d8..27742ef1 100644
--- a/atdb/dumpdata_configuration.bat
+++ b/atdb/dumpdata_configuration.bat
@@ -1,3 +1,5 @@
+REM dump the configuration table as a json file (fixture)
+
python manage.py dumpdata taskdatabase.configuration --indent 4 --settings=atdb.settings.dev > configuration_fixture.json
REM sdc-dev / sdc
diff --git a/atdb/dumpdata_workflows.bat b/atdb/dumpdata_workflows.bat
index d7463c45..4d7cedf6 100644
--- a/atdb/dumpdata_workflows.bat
+++ b/atdb/dumpdata_workflows.bat
@@ -1,3 +1,5 @@
+REM dump the workflows tables as a json file (fixture)
+
python manage.py dumpdata taskdatabase.workflow --indent 4 --settings=atdb.settings.dev > workflow_fixture.json
REM sdc-dev / sdc
diff --git a/atdb/loaddata_configuration.bat b/atdb/loaddata_configuration.bat
index 59c194dd..f8cd885d 100644
--- a/atdb/loaddata_configuration.bat
+++ b/atdb/loaddata_configuration.bat
@@ -1,3 +1,5 @@
+REM load the configuration_fixture.json from a json file
+
python manage.py loaddata configuration_fixture.json --settings=atdb.settings.dev
REM sdc-dev / sdc
diff --git a/atdb/loaddata_workflow.bat b/atdb/loaddata_workflow.bat
index f3241033..aeea94e4 100644
--- a/atdb/loaddata_workflow.bat
+++ b/atdb/loaddata_workflow.bat
@@ -1,3 +1,5 @@
+REM load the workflows table from a json file (fixture)
+
python manage.py loaddata workflow_fixture.json --settings=atdb.settings.dev
REM sdc-dev / sdc
diff --git a/atdb/makemigrations.bat b/atdb/makemigrations.bat
index 7f7d2616..00df8ee4 100644
--- a/atdb/makemigrations.bat
+++ b/atdb/makemigrations.bat
@@ -1,2 +1,3 @@
+REM create the database migration file (this does not do the actual migration, see migration.bat for that)
python manage.py makemigrations --settings=atdb.settings.dev
REM don't forget to add the new migration file to the repo (right click on the migration file, choose git -> add).
\ No newline at end of file
diff --git a/atdb/migrate.bat b/atdb/migrate.bat
index e628eead..9ec2e177 100644
--- a/atdb/migrate.bat
+++ b/atdb/migrate.bat
@@ -1 +1,2 @@
+REM run the database migration
python manage.py migrate --settings=atdb.settings.dev
\ No newline at end of file
diff --git a/atdb/requirements/base.txt b/atdb/requirements/base.txt
index 38b010da..fcc3da67 100644
--- a/atdb/requirements/base.txt
+++ b/atdb/requirements/base.txt
@@ -1,7 +1,6 @@
astronauth==0.3.3
Django==5
django-allauth==0.57.0 # note allauth only supports Django >= 3.2
-#django-bootstrap-pagination==1.7.1
django-bootstrap3==23.6
django-cors-headers==3.6.0
django-extensions==3.1.0
diff --git a/atdb/rollback.bat b/atdb/rollback.bat
index dc8c5fa1..8602eaf2 100644
--- a/atdb/rollback.bat
+++ b/atdb/rollback.bat
@@ -1 +1,2 @@
+REM example of how to rollback a database change
python manage.py migrate taskdatabase 0010 --settings=atdb.settings.dev
\ No newline at end of file
diff --git a/atdb/superuser.bat b/atdb/superuser.bat
index 879d8c04..f8ebf6dc 100644
--- a/atdb/superuser.bat
+++ b/atdb/superuser.bat
@@ -1 +1,2 @@
+REM add a superuser to the database
python manage.py createsuperuser --settings=atdb.settings.dev
\ No newline at end of file
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 2c6b1a07..daf9ce30 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -208,14 +208,8 @@ class Task(models.Model):
return str(self.id) + ' - (' + self.task_type + ') - ' + str(self.sas_id)
def save(self, *args, **kwargs):
- # nv:1mar2023, temporary hack, set tasks 'on hold' as soon they get to 'scrubbed'
- # (users forget to do that manually, causing unwanted ingests)
- # nv:8apr2024, SDO asked for this hack to be removed again.
- # if (self.status != State.SCRUBBED.value) & (self.new_status == State.SCRUBBED.value):
- # self.resume = False
-
- # nv:19jun2023, calculate the qualities for this task
+ # calculate the qualities for this task
if (self.status != State.STORED.value) & (self.new_status == State.STORED.value):
# read the quality_thresholds from the Configuration table
@@ -245,11 +239,6 @@ class Task(models.Model):
# if so, temporarily put it on hold so that the ancillary service can grab it with it
if self.is_summary:
- # slightly bothered by the idea that ATDB puts it on HOLD and depends on the ancillary service
- # to resume it. But users have earlier asked for the same dynamic for scrubbed. Acceptable
-
- # TODO: (un)comment this
- # We don't want ATDB holding tasks without a aggregator service in place to resume
self.resume = False
except Exception as error:
diff --git a/atdb/taskdatabase/services/activities_handler.py b/atdb/taskdatabase/services/activities_handler.py
index 50a92d74..0df9e790 100644
--- a/atdb/taskdatabase/services/activities_handler.py
+++ b/atdb/taskdatabase/services/activities_handler.py
@@ -79,6 +79,7 @@ def update_activity(task):
- to STORED : calculate quality
- to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction
- to _FAILED : calculate finished_fraction
+ - to STORED, PROCESSED, DISCARDED : check if all tasks are processed, set Activity to is_processed and AGGREGATE
- always : determine if a task is in a 'verified' status
"""
diff --git a/atdb/taskdatabase/tests/test_models_processed_summary.py b/atdb/taskdatabase/tests/test_models_processed_summary.py
index f03bc367..706e3acf 100644
--- a/atdb/taskdatabase/tests/test_models_processed_summary.py
+++ b/atdb/taskdatabase/tests/test_models_processed_summary.py
@@ -32,14 +32,14 @@ class TestProcessedSummary(TestCase):
actual = self.task1.resume
self.assertEqual(actual, True)
- # def test_processed_on_hold(self):
- # """
- # task 2 is processed, and a summary dataproduct. Should go on hold
- # """
- #
- # actual = self.task2.resume
- # # this test fails, because "self.resume = False" is still commented out in models.py L249
- # self.assertEqual(actual, False)
+ def test_processed_on_hold(self):
+ """
+ task 2 is processed, and a summary dataproduct. Should go on hold
+ """
+
+ actual = self.task2.resume
+ # this test fails, because "self.resume = False" is still commented out in models.py L249
+ self.assertEqual(actual, False)
def test_activity_is_processed(self):
"""
diff --git a/atdb/taskdatabase/tests/test_summary_tasks.py b/atdb/taskdatabase/tests/test_summary_tasks.py
index 85cf9fa0..6aaafde8 100644
--- a/atdb/taskdatabase/tests/test_summary_tasks.py
+++ b/atdb/taskdatabase/tests/test_summary_tasks.py
@@ -29,39 +29,34 @@ class TestSummaryTasks(TestCase):
"""
test task that is not a summary task
"""
- actual = self.no_summary_task_77777.is_summary
- self.assertEqual(actual, False)
+ self.assertFalse(self.no_summary_task_77777.is_summary)
def test_task_defined_does_not_know_that_it_is_summary(self):
"""
test summary task, but before it knows that it becomes a summary task (which it only knows when 'processed')
"""
- actual = self.summary_task_defined_77777.is_summary
- self.assertEqual(actual, False)
+ self.assertFalse(self.summary_task_defined_77777.is_summary)
def test_task_validated_knows_that_it_is_summary(self):
"""
test summary task, at 'stored' it should know that it is a summary task and return True)
"""
self.summary_task_validated_77777.save()
- actual = self.summary_task_validated_77777.is_summary
- self.assertEqual(actual, True)
+ self.assertTrue(self.summary_task_validated_77777.is_summary)
def test_task_processed_knows_that_it_is_summary(self):
"""
test summary task, at 'stored' it should know that it is a summary task and return True)
"""
self.summary_task_processed_77777.save()
- actual = bool(self.summary_task_processed_77777.is_summary)
- self.assertEqual(actual, True)
+ self.assertTrue(self.summary_task_processed_77777.is_summary)
def test_summary_task_processed_goes_on_halt(self):
"""
test summary task, at 'stored' it should know that it is a summary task and return True)
"""
self.summary_task_processed_88888.save()
- actual = self.summary_task_processed_88888.resume
- self.assertEqual(actual, False)
+ self.assertFalse(self.summary_task_processed_88888.resume)
def test_activity_77777_not_is_processed(self):
"""
@@ -69,8 +64,7 @@ class TestSummaryTasks(TestCase):
"""
self.summary_task_processed_77777.save()
activity = self.summary_task_processed_77777.activity
- actual = bool(activity.is_processed)
- self.assertFalse(actual)
+ self.assertFalse(activity.is_processed)
def test_activity_88888_is_processed(self):
"""
@@ -79,5 +73,4 @@ class TestSummaryTasks(TestCase):
self.summary_task_stored_88888.save()
self.summary_task_processed_88888.save()
activity = self.summary_task_processed_88888.activity
- actual = activity.is_processed
- self.assertTrue(actual)
\ No newline at end of file
+ self.assertTrue(activity.is_processed)
\ No newline at end of file
diff --git a/atdb/taskdatabase/tests/test_update_activity.py b/atdb/taskdatabase/tests/test_update_activity.py
index 4076e734..21253449 100644
--- a/atdb/taskdatabase/tests/test_update_activity.py
+++ b/atdb/taskdatabase/tests/test_update_activity.py
@@ -188,7 +188,7 @@ class TestUpdateActivity(TestCase):
def test_is_processed(self):
"""
task 6, 7 and 8 are processed,
- activity.is_processed should be true and activity status should go to 'handle_ancillary'
+ activity.is_processed should be true and activity status should go to 'aggregate'
"""
activity = self.task6.activity
diff --git a/atdb/test-latest.bat b/atdb/test-latest.bat
index 0ef3199a..cdb84165 100644
--- a/atdb/test-latest.bat
+++ b/atdb/test-latest.bat
@@ -1 +1,2 @@
+REM test a single unittest
python manage.py test taskdatabase.tests.test_update_activity.TestUpdateActivity --settings=atdb.settings.dev
\ No newline at end of file
diff --git a/atdb/test.bat b/atdb/test.bat
index fd750b2f..b61cef9b 100644
--- a/atdb/test.bat
+++ b/atdb/test.bat
@@ -1 +1,2 @@
+REM runs the unit tests for ATDB
python manage.py test --settings=atdb.settings.dev
\ No newline at end of file
diff --git a/atdb/test_local.bat b/atdb/test_local.bat
deleted file mode 100644
index e6bbdcea..00000000
--- a/atdb/test_local.bat
+++ /dev/null
@@ -1 +0,0 @@
-python manage.py test --settings atdb.settings.test_local
--
GitLab