diff --git a/atdb/atdb/settings/dev.py b/atdb/atdb/settings/dev.py
index 58b33911e885963e72c5478ba2eac0ca910567c3..687f2d334ce7a1fef476829280e69efef9fedf6a 100644
--- a/atdb/atdb/settings/dev.py
+++ b/atdb/atdb/settings/dev.py
@@ -13,7 +13,8 @@ DATABASES = {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'atdb_admin',
'PASSWORD': 'atdb123',
- 'NAME': 'atdb_ldv_astronauth_6feb2023',
+ #'NAME': 'atdb_ldv_astronauth_6feb2023',
+ 'NAME': 'atdb_ldv_27jun2023',
'HOST': 'localhost',
'PORT': '5432',
},
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index fbfad8676fbb073deb7c22466221d76bf7f92c64..2504c27646e7727ea3ba9892200179c70416c8ed 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -133,12 +133,6 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
- try:
- self.remarks['calculated_qualities'] = self.calculated_qualities
- except:
- self.remarks = {}
- self.remarks['calculated_qualities'] = self.calculated_qualities
-
super(Task, self).save(*args, **kwargs)
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 75d1f4ca37db36ae9c68a8b4b50aebb01ed4bf68..81cfcf36b6af69fc038babfe127f249c551f7648 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -801,7 +801,7 @@ def construct_summary(task):
results = ""
total_size_input = 0
total_size_output = 0
- qualities = {'poor': 0, 'moderate': 0, 'good': 0}
+ quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
sas_id = task.sas_id
title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
@@ -885,9 +885,9 @@ def construct_summary(task):
try:
key = task.calculated_qualities['per_task']
- qualities[key] = qualities[key] + 1
+ quality_values[key] = quality_values[key] + 1
except:
- # ignore the tasks that have no calculated quality (they are probably not 'stored').
+ # ignore the tasks that have no calculated quality.
pass
results += line
@@ -908,7 +908,7 @@ def construct_summary(task):
totals += '<tr><td colspan="2"><b>Calculated Quality</b></td>'
totals += '<td class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
- totals += '<tr><td colspan="2"><b>Quality Statistics</b></td><td>' + str(qualities) + '</td></tr>'
+ totals += '<tr><td colspan="2"><b>Quality Statistics</b></td><td>' + str(quality_values) + '</td></tr>'
try:
quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
diff --git a/atdb/taskdatabase/services/calculated_qualities.py b/atdb/taskdatabase/services/calculated_qualities.py
index c6c2a87b0bf3d23b2173131ae4f2616b4bb9cde0..31ac82cef83be01e37506fe2377c8f8e1a52ca5d 100644
--- a/atdb/taskdatabase/services/calculated_qualities.py
+++ b/atdb/taskdatabase/services/calculated_qualities.py
@@ -1,6 +1,5 @@
import logging
logger = logging.getLogger(__name__)
-import json
def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
""""
@@ -69,6 +68,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
total = qualities['poor'] + qualities['moderate'] + qualities['good']
+ quality_sasid = None
if total > 0:
percentage_poor = (qualities['poor'] / total) * 100
percentage_good = (qualities['good'] / total) * 100
diff --git a/atdb/taskdatabase/tests.py b/atdb/taskdatabase/tests.py
index 7ce503c2dd97ba78597f6ff6e4393132753573f6..14ce21a1f97bfd688a0de36d744cdf348b0672a2 100644
--- a/atdb/taskdatabase/tests.py
+++ b/atdb/taskdatabase/tests.py
@@ -1,3 +1,267 @@
from django.test import TestCase
-
+import json
+from .services import calculated_qualities as qualities
+from .models import Configuration, Task
# Create your tests here.
+class TestCalculatedQualities(TestCase):
+
+ @classmethod
+ def setUpTestData(cls):
+ print("setUpTestData: Run once to set up non-modified data for all class methods.")
+ # Set up non-modified objects used by all test methods
+ quality_thresholds = {
+ "moderate": 20,
+ "poor": 50,
+ "overall_poor": 50,
+ "overall_good": 90,
+ }
+ Configuration.objects.create(key="quality_thresholds", value=json.dumps(quality_thresholds))
+
+ def setUp(self):
+ print("setUp: Run once for every test method to setup clean data.")
+
+ outputs0 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_summaryIS.tar": {
+ "added": [],
+ "deleted": [],
+ "input_name": "L526107_summaryIS.tar",
+ "input_size": 495749120,
+ "input_size_str": "472.78 MB",
+ "output_name": "L526107_summaryIS.tar",
+ "output_size": 283791360,
+ "output_size_str": "270.64 MB",
+ "rfi_percent": 0,
+ "size_ratio": 0.5724495486749427
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+ }
+
+ outputs1 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_SAP002_B073_P000_bf.tar": {
+ "added": [
+ "stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit.fits",
+ "stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit_ldv_psrfits_requantisation.log"
+ ],
+ "deleted": [
+ "stokes/SAP2/BEAM73/L526105_SAP2_BEAM73.fits"
+ ],
+ "input_name": "L526107_SAP002_B073_P000_bf.tar",
+ "input_size": 20353853440,
+ "input_size_str": "18.96 GB",
+ "output_name": "L526107_SAP002_B073_P000_bf.tar",
+ "output_size": 6024990720,
+ "output_size_str": "5.61 GB",
+ "rfi_percent": 11.167,
+ "size_ratio": 0.2960122876860019
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+ }
+
+ outputs2 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_SAP002_B073_P000_bf.tar": {
+ "added": [
+ "stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit.fits",
+ "stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit_ldv_psrfits_requantisation.log"
+ ],
+ "deleted": [
+ "stokes/SAP2/BEAM73/L526105_SAP2_BEAM73.fits"
+ ],
+ "input_name": "L526107_SAP002_B073_P000_bf.tar",
+ "input_size": 20353853440,
+ "input_size_str": "18.96 GB",
+ "output_name": "L526107_SAP002_B073_P000_bf.tar",
+ "output_size": 6024990720,
+ "output_size_str": "5.61 GB",
+ "rfi_percent": 22.167,
+ "size_ratio": 0.2960122876860019
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+ }
+
+ outputs3 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_SAP002_B072_P000_bf.tar": {
+ "added": [
+ "stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit.fits",
+ "stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit_ldv_psrfits_requantisation.log"
+ ],
+ "deleted": [
+ "stokes/SAP2/BEAM72/L526105_SAP2_BEAM72.fits"
+ ],
+ "input_name": "L526107_SAP002_B072_P000_bf.tar",
+ "input_size": 20353843200,
+ "input_size_str": "18.96 GB",
+ "output_name": "L526107_SAP002_B072_P000_bf.tar",
+ "output_size": 6024980480,
+ "output_size_str": "5.61 GB",
+ "rfi_percent": 31.921,
+ "size_ratio": 0.2960119335104242
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+
+ }
+
+ outputs4 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_SAP002_B070_P000_bf.tar": {
+ "added": [
+ "stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit.fits",
+ "stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit_ldv_psrfits_requantisation.log"
+ ],
+ "deleted": [
+ "stokes/SAP2/BEAM70/L526105_SAP2_BEAM70.fits"
+ ],
+ "input_name": "L526107_SAP002_B070_P000_bf.tar",
+ "input_size": 20353525760,
+ "input_size_str": "18.96 GB",
+ "output_name": "L526107_SAP002_B070_P000_bf.tar",
+ "output_size": 6024755200,
+ "output_size_str": "5.61 GB",
+ "rfi_percent": 52.164,
+ "size_ratio": 0.2960054818531843
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+ }
+
+ outputs5 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_SAP002_B072_P000_bf.tar": {
+ "added": [
+ "stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit.fits",
+ "stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit_ldv_psrfits_requantisation.log"
+ ],
+ "deleted": [
+ "stokes/SAP2/BEAM72/L526105_SAP2_BEAM72.fits"
+ ],
+ "input_name": "L526107_SAP002_B072_P000_bf.tar",
+ "input_size": 20353843200,
+ "input_size_str": "18.96 GB",
+ "output_name": "L526107_SAP002_B072_P000_bf.tar",
+ "output_size": 6024980480,
+ "output_size_str": "5.61 GB",
+ "size_ratio": 0.2960119335104242
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+
+ }
+
+ outputs6 = {
+ "quality": {
+ "details": {},
+ "observing-conditions": "N/A",
+ "sensitivity": "N/A",
+ "summary": {
+ "L526107_SAP002_B070_P000_bf.tar": {
+ "added": [
+ "stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit.fits",
+ "stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit_ldv_psrfits_requantisation.log"
+ ],
+ "deleted": [
+ "stokes/SAP2/BEAM70/L526105_SAP2_BEAM70.fits"
+ ],
+ "input_name": "L526107_SAP002_B070_P000_bf.tar",
+ "input_size": 20353525760,
+ "input_size_str": "18.96 GB",
+ "output_name": "L526107_SAP002_B070_P000_bf.tar",
+ "output_size": 6024755200,
+ "output_size_str": "5.61 GB",
+ "size_ratio": 0.2960054818531843
+ }
+ },
+ "uv-coverage": "N/A"
+ },
+ }
+
+ # create a list of Tasks with various values of rfi_percent to test the quality algorithms
+ # rfi_percent=0, this task should not be included in the calculates
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs0)
+
+ # rfi_percent 11,22,31,52
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs1)
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs2)
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs3)
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs4)
+
+ # tasks without rfi_percent (so simulating a different pipeline)
+ Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs5)
+ Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs6)
+
+ def test_count_tasks(self):
+ actual = Task.objects
+ count = actual.count()
+ self.assertEqual(count,7)
+
+ def test_run_quality_calculation_in_task_model_when_stored(self):
+ for task in Task.objects.all():
+ task.new_status = 'stored'
+ task.save()
+
+ # only 4 tasks should now have calculated_qualities
+ count = 0
+ for task in Task.objects.all():
+ if task.calculated_qualities['per_sasid']:
+ count += 1
+
+ self.assertEqual(count,4)
+
+
+ def test_calculated_qualities_per_task(self):
+
+ # read the quality thresholds from the test database
+ quality_thresholds = json.loads(Configuration.objects.get(key="quality_thresholds").value)
+
+ # get the tasks for sas_id 54321
+ tasks_for_this_sasid = Task.objects.filter(sas_id=54321)
+
+ # run the algorithms and gather the values
+ quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
+
+ for task in tasks_for_this_sasid:
+ q = qualities.calculate_qualities(task, tasks_for_this_sasid, quality_thresholds)
+ try:
+ key = task.calculated_qualities['per_task']
+ quality_values[key] = quality_values[key] + 1
+ except:
+ # ignore the tasks that have no calculated quality.
+ pass
+
+ self.assertEqual(quality_values,{'poor': 1, 'moderate': 2, 'good': 1})
\ No newline at end of file