From 947b2cbe7e7ea771ba422784eb46edfbe55e5da3 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Mon, 9 Oct 2023 14:28:47 +0200
Subject: [PATCH] additional unittests get rid of autofield warning
---
atdb/atdb/settings/base.py | 2 +
atdb/taskdatabase/services/common.py | 13 +-
.../tests/test_calculated_qualities.py | 300 ++++++++++++++++--
3 files changed, 283 insertions(+), 32 deletions(-)
diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index 7fcfe6e7..82d1526f 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -264,3 +264,5 @@ CSRF_COOKIE_NAME = 'atdb_csrftoken'
#SILKY_PYTHON_PROFILER = False
#SILKY_PYTHON_PROFILER_BINARY = False
+
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
\ No newline at end of file
diff --git a/atdb/taskdatabase/services/common.py b/atdb/taskdatabase/services/common.py
index 6d4f8a9f..d7356730 100644
--- a/atdb/taskdatabase/services/common.py
+++ b/atdb/taskdatabase/services/common.py
@@ -36,13 +36,8 @@ def get_summary_flavour(task):
# this could be made implicit in the future by adding a setting to the Workflow,
# but currently it is derived in a yucky way. But at least the yuck is confined to this function
"""
- summary_flavour = SummaryFlavour.DEFAULT.value
- try:
- summary = task.quality_json["summary"]
- except:
- # no summary found
- return None
+ summary_flavour = SummaryFlavour.DEFAULT.value
# so... yikes... what distinguishes the summary flavours?
workflow_uri = task.workflow.workflow_uri
@@ -53,6 +48,12 @@ def get_summary_flavour(task):
if "linc_target" in workflow_uri:
return SummaryFlavour.LINC_TARGET.value
+ try:
+ summary = task.quality_json["summary"]
+ except:
+ # no summary found
+ return None
+
try:
d = summary["details"]
summary_flavour = SummaryFlavour.IMAGING_COMPRESSION.value
diff --git a/atdb/taskdatabase/tests/test_calculated_qualities.py b/atdb/taskdatabase/tests/test_calculated_qualities.py
index ffd83c7a..d2961a7f 100644
--- a/atdb/taskdatabase/tests/test_calculated_qualities.py
+++ b/atdb/taskdatabase/tests/test_calculated_qualities.py
@@ -3,7 +3,7 @@ import json
from taskdatabase.services import calculated_qualities as qualities
from taskdatabase.models import Configuration, Task, Workflow
-
+from taskdatabase.services.common import get_summary_flavour, SummaryFlavour
class TestCalculatedQualities(TestCase):
@@ -22,8 +22,8 @@ class TestCalculatedQualities(TestCase):
def setUp(self):
print("setUp: Run once for every test method to setup clean data.")
-
- outputs0 = {
+ # default summary_flavour with rfi_percent=0
+ outputs_default_1 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -46,7 +46,8 @@ class TestCalculatedQualities(TestCase):
},
}
- outputs1 = {
+ # default summary_flavour with rfi
+ outputs_default_2 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -73,8 +74,9 @@ class TestCalculatedQualities(TestCase):
"uv-coverage": "N/A"
},
}
-
- outputs2 = {
+
+ # default summary_flavour with rfi
+ outputs_default_3 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -102,7 +104,8 @@ class TestCalculatedQualities(TestCase):
},
}
- outputs3 = {
+ # default summary_flavour with rfi
+ outputs_default_4 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -131,7 +134,8 @@ class TestCalculatedQualities(TestCase):
}
- outputs4 = {
+ # default summary_flavour with rfi
+ outputs_default_5 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -159,7 +163,8 @@ class TestCalculatedQualities(TestCase):
},
}
- outputs5 = {
+ # default summary_flavour without rfi
+ outputs_default_6 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -187,7 +192,8 @@ class TestCalculatedQualities(TestCase):
}
- outputs6 = {
+ # default summary_flavour without rfi
+ outputs_default_7 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
@@ -214,32 +220,233 @@ class TestCalculatedQualities(TestCase):
},
}
- # create a list of Tasks with various values of rfi_percent to test the quality algorithms
- # rfi_percent=0, this task should not be included in the calculates
- workflow_compress = Workflow(workflow_uri="imaging_compress_pipeline_v011")
- workflow_compress.save()
- Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs0, workflow=workflow_compress)
+ # IMAGING_COMPRESSION summary_flavour with rfi
+ outputs_imaging_compression_1 = {
+ "quality": {
+ "summary": {
+ "details": {
+ "DStDev": {
+ "CS001": 15372035.9671943,
+ "CS002": 14858111.10350275,
+ "CS003": 10147611.046423668,
+ "CS004": 18980165.035334244,
+ "CS005": 9209186.769605417,
+ "CS006": 15544054.561004732,
+ "CS007": 15737019.571027506,
+ "CS011": 14245094.062691605,
+ "CS013": 10705936.655357886,
+ "CS017": 14099126.5756219,
+ "CS021": 13172990.03150767,
+ "CS024": 14696724.018343825,
+ "CS026": 18501377.981954917,
+ "CS028": 14326771.584380083,
+ "CS030": 16033335.687642261,
+ "CS031": 20901704.500670947,
+ "CS032": 18795952.493532542,
+ "CS101": 67594399.69123329,
+ "CS103": 14555006.230974862,
+ "CS201": 11491082.207871344,
+ "CS301": 40468265.70497692,
+ "CS302": 17781663.389931183,
+ "CS401": 19556709.0685369,
+ "CS501": 27261643.796409346,
+ "DE601": 17777132.55854045,
+ "DE602": 19748901.556048356,
+ "DE603": 26819783.45521549,
+ "DE604": 14385497.046839358,
+ "DE605": 12729490.454671673,
+ "DE609": 11548756.244492985,
+ "FR606": 13169448.64903064,
+ "IE613": 13395597.406249378,
+ "LV614": 11668296.995990513,
+ "PL610": 14960883.74047425,
+ "PL611": 17196733.845408365,
+ "PL612": 10283464.55136512,
+ "RS106": 12128338.820957774,
+ "RS205": 42916272.60510826,
+ "RS208": 9365468.17970955,
+ "RS210": 47000312.251054004,
+ "RS305": 15538054.639135055,
+ "RS306": 14206058.107420009,
+ "RS307": 14757246.239034232,
+ "RS310": 14171170.538164835,
+ "RS406": 15226166.937623236,
+ "RS407": 14530681.276822567,
+ "RS409": 14725610.814889988,
+ "RS503": 11508097.846546676,
+ "RS508": 28514459.964421105,
+ "RS509": 19256534.542812303,
+ "SE607": 30430197.90790976,
+ "UK608": 22423233.01862699
+ },
+ "target": [
+ "3C295"
+ ],
+ "antennas": [
+ "CS001HBA0",
+ "CS001HBA1",
+ "CS002HBA0",
+ "CS002HBA1",
+ "CS003HBA0",
+ "CS003HBA1",
+ "CS004HBA0",
+ "CS004HBA1",
+ "CS005HBA0",
+ "CS005HBA1",
+ "CS006HBA0",
+ "CS006HBA1",
+ "CS007HBA0",
+ "CS007HBA1",
+ "CS011HBA0",
+ "CS011HBA1",
+ "CS013HBA0",
+ "CS013HBA1",
+ "CS017HBA0",
+ "CS017HBA1",
+ "CS021HBA0",
+ "CS021HBA1",
+ "CS024HBA0",
+ "CS024HBA1",
+ "CS026HBA0",
+ "CS026HBA1",
+ "CS028HBA0",
+ "CS028HBA1",
+ "CS030HBA0",
+ "CS030HBA1",
+ "CS031HBA0",
+ "CS031HBA1",
+ "CS032HBA0",
+ "CS032HBA1",
+ "CS101HBA0",
+ "CS101HBA1",
+ "CS103HBA0",
+ "CS103HBA1",
+ "CS201HBA0",
+ "CS201HBA1",
+ "CS301HBA0",
+ "CS301HBA1",
+ "CS302HBA0",
+ "CS302HBA1",
+ "CS401HBA0",
+ "CS401HBA1",
+ "CS501HBA0",
+ "CS501HBA1",
+ "RS106HBA",
+ "RS205HBA",
+ "RS208HBA",
+ "RS210HBA",
+ "RS305HBA",
+ "RS306HBA",
+ "RS307HBA",
+ "RS310HBA",
+ "RS406HBA",
+ "RS407HBA",
+ "RS409HBA",
+ "RS503HBA",
+ "RS508HBA",
+ "RS509HBA",
+ "DE601HBA",
+ "DE602HBA",
+ "DE603HBA",
+ "DE604HBA",
+ "DE605HBA",
+ "FR606HBA",
+ "SE607HBA",
+ "UK608HBA",
+ "DE609HBA",
+ "PL610HBA",
+ "PL611HBA",
+ "PL612HBA",
+ "IE613HBA"
+ ],
+ "pointing": {
+ "Sun": 98.62325727494583,
+ "CasA": 63.8887478639975,
+ "CygA": 57.33860706164162,
+ "HerA": 57.53230892059052,
+ "Moon": 82.10124202600636,
+ "TauA": 93.60818880478796,
+ "VirA": 44.64319497995252,
+ "Jupiter": 65.56149628509407,
+ "elevation_fraction": 1
+ },
+ "rfi_percentage": 1.7186448587105623,
+ "antenna_configuration": "FULL",
+ "antennas_not_available": [
+ "LV614"
+ ]
+ },
+ "applied_fixes": [],
+ "rfi_perc_total": "good",
+ "elevation_score": "good",
+ "sun_interference": "good",
+ "unfixable_issues": [],
+ "moon_interference": "good",
+ "jupiter_interference": "good",
+ "degree_incompleteness_array": [],
+ "array_missing_important_pairs_is": "good",
+ "array_missing_important_pairs_dutch": "good",
+ "aggregated_array_data_losses_percentage": "poor",
+ "array_high_data_loss_on_is_important_pair": "good",
+ "array_high_data_loss_on_dutch_important_pair": "good"
+ }
+ },
+ }
- # rfi_percent 11,22,31,52
+ # LINC_TARGET summary_flavour without rfi
+ outputs_link_target_1 = {
+ "quality": {
+ "details": {},
+ "sensitivity": "N/A",
+ "uv-coverage": "N/A",
+ "observing-conditions": "N/A"
+ },
+ "summary": {
+ "size": 17464,
+ "surl": "srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/ops/disk/ldv/lt10_010/689478/35011/3c48_LINC_calibrator_summary.json",
+ "class": "File",
+ "nameext": ".json",
+ "basename": "3c48_LINC_calibrator_summary.json",
+ "checksum": "sha1$531646ff527d76f4facdabf72d939bac302eaf1f",
+ "location": "file:///project/ldv/Share/run/2023/6/16/1352_35011/3c48_LINC_calibrator_summary.json",
+ "nameroot": "3c48_LINC_calibrator_summary"
+ },
+ }
+
+ # create a list of Tasks with various values of rfi_percent to test the quality algorithms
workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation")
workflow_requantisation.save()
- Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs1, workflow=workflow_requantisation)
- Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs2, workflow=workflow_requantisation)
- Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs3, workflow=workflow_requantisation)
- Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs4, workflow=workflow_requantisation)
+
+ # rfi_percent=0, this task should be skipped in the quality calculations
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs_default_1, workflow=workflow_requantisation)
+
+ # rfi_percent 11,22,31,52
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs_default_2, workflow=workflow_requantisation)
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs_default_3, workflow=workflow_requantisation)
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs_default_4, workflow=workflow_requantisation)
+ Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs_default_5, workflow=workflow_requantisation)
# tasks without rfi_percent (so simulating a different pipeline)
- workflow_link_calibrator = Workflow(workflow_uri="linc_calibrator_v4_2")
- workflow_link_calibrator.save()
+ workflow_no_rfi = Workflow(workflow_uri="linc_calibrator_v4_2")
+ workflow_no_rfi.save()
+
+ Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs_default_6, workflow=workflow_no_rfi)
+
+ # workflow_imaging_compression, rfi_percentage=1.7186448587105623
+ workflow_imaging_compression = Workflow(workflow_uri="imaging_compress_pipeline_v011")
+ workflow_imaging_compression.save()
+ Task.objects.get_or_create(sas_id=55555, status='processed', outputs=outputs_imaging_compression_1, workflow=workflow_imaging_compression)
+
+ # LINK pipelines (no rfi_percent onboard yet)
workflow_link_target = Workflow(workflow_uri="linc_target_v4_2")
workflow_link_target.save()
- Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs5, workflow=workflow_compress)
- Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs6, workflow=workflow_link_calibrator)
+ Task.objects.get_or_create(sas_id=666666, status='processed', outputs=outputs_link_target_1, workflow=workflow_link_target)
+
def test_count_tasks(self):
actual = Task.objects
count = actual.count()
- self.assertEqual(count,7)
+ self.assertEqual(count,8)
def test_run_calculations_when_task_becomes_stored(self):
for task in Task.objects.all():
@@ -253,7 +460,7 @@ class TestCalculatedQualities(TestCase):
if task.calculated_qualities['per_sasid']:
count += 1
- self.assertEqual(count,4)
+ self.assertEqual(count,5)
def test_calculated_qualities(self):
@@ -380,3 +587,44 @@ class TestCalculatedQualities(TestCase):
self.assertEqual(quality_values, {'poor': 0, 'moderate': 0, 'good': 0})
self.assertEqual(quality_per_sasid, None)
+
+ def test_default_summary_flavour(self):
+ """
+ test if the DEFAULT summary_flavour is detected
+ """
+
+ # get the tasks for sas_id 54321
+ tasks_for_this_sasid = Task.objects.filter(sas_id=54321)
+
+ # 1 is enough, they all have the same summary_flavour
+ task = tasks_for_this_sasid[0]
+ summary_flavour = get_summary_flavour(task)
+
+ self.assertEqual(summary_flavour,SummaryFlavour.DEFAULT.value)
+
+ def test_default_summary_flavour(self):
+ """
+ test if the IMAGING_COMPRESSION summary_flavour is detected
+ """
+
+ tasks_for_this_sasid = Task.objects.filter(sas_id=55555)
+
+ # 1 is enough, they all have the same summary_flavour
+ task = tasks_for_this_sasid[0]
+ summary_flavour = get_summary_flavour(task)
+
+ self.assertEqual(summary_flavour,SummaryFlavour.IMAGING_COMPRESSION.value)
+
+
+ def test_link_summary_flavour(self):
+ """
+ test if the LINC_TARGET summary_flavour is detected
+ """
+
+ tasks_for_this_sasid = Task.objects.filter(sas_id=666666)
+
+ # 1 is enough, they all have the same summary_flavour
+ task = tasks_for_this_sasid[0]
+ summary_flavour = get_summary_flavour(task)
+
+ self.assertEqual(summary_flavour,SummaryFlavour.LINC_TARGET.value)
\ No newline at end of file
--
GitLab