Skip to content
Snippets Groups Projects
Commit ec6ed484 authored by Nico Vermaas's avatar Nico Vermaas
Browse files

adding unittests

parent 6afb8d86
Branches
No related tags found
2 merge requests!304update branch with master,!302automatic quality validation
Pipeline #52203 passed
Stage: build
Stage: deploy_to_test
Stage: deploy_to_production
Hide whitespace changes
Inline Side-by-side
atdb/atdb/settings/dev.py
+ 2
− 1
View file @ ec6ed484
...@@ -13,7 +13,8 @@ DATABASES = { ...@@ -13,7 +13,8 @@ DATABASES = {
'ENGINE': 'django.db.backends.postgresql_psycopg2', 'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'atdb_admin', 'USER': 'atdb_admin',
'PASSWORD': 'atdb123', 'PASSWORD': 'atdb123',
'NAME': 'atdb_ldv_astronauth_6feb2023', #'NAME': 'atdb_ldv_astronauth_6feb2023',
'NAME': 'atdb_ldv_27jun2023',
'HOST': 'localhost', 'HOST': 'localhost',
'PORT': '5432', 'PORT': '5432',
}, },
......
atdb/taskdatabase/models.py
+ 0
− 6
View file @ ec6ed484
...@@ -133,12 +133,6 @@ class Task(models.Model): ...@@ -133,12 +133,6 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id) tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds) self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
try:
self.remarks['calculated_qualities'] = self.calculated_qualities
except:
self.remarks = {}
self.remarks['calculated_qualities'] = self.calculated_qualities
super(Task, self).save(*args, **kwargs) super(Task, self).save(*args, **kwargs)
......
atdb/taskdatabase/services/algorithms.py
+ 4
− 4
View file @ ec6ed484
...@@ -801,7 +801,7 @@ def construct_summary(task): ...@@ -801,7 +801,7 @@ def construct_summary(task):
results = "" results = ""
total_size_input = 0 total_size_input = 0
total_size_output = 0 total_size_output = 0
qualities = {'poor': 0, 'moderate': 0, 'good': 0} quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
sas_id = task.sas_id sas_id = task.sas_id
title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> " title = "<h4>Summary File for SAS_ID " + task.sas_id + "</h4> "
...@@ -885,9 +885,9 @@ def construct_summary(task): ...@@ -885,9 +885,9 @@ def construct_summary(task):
try: try:
key = task.calculated_qualities['per_task'] key = task.calculated_qualities['per_task']
qualities[key] = qualities[key] + 1 quality_values[key] = quality_values[key] + 1
except: except:
# ignore the tasks that have no calculated quality (they are probably not 'stored'). # ignore the tasks that have no calculated quality.
pass pass
results += line results += line
...@@ -908,7 +908,7 @@ def construct_summary(task): ...@@ -908,7 +908,7 @@ def construct_summary(task):
totals += '<tr><td colspan="2"><b>Calculated Quality</b></td>' totals += '<tr><td colspan="2"><b>Calculated Quality</b></td>'
totals += '<td class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>' totals += '<td class="' + sasid_quality + '">' + str(sasid_quality) + '</td></tr>'
totals += '<tr><td colspan="2"><b>Quality Statistics</b></td><td>' + str(qualities) + '</td></tr>' totals += '<tr><td colspan="2"><b>Quality Statistics</b></td><td>' + str(quality_values) + '</td></tr>'
try: try:
quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value) quality_thresholds = json.loads(Configuration.objects.get(key='quality_thresholds').value)
......
atdb/taskdatabase/services/calculated_qualities.py
+ 1
− 1
View file @ ec6ed484
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
import json
def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds): def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
"""" """"
...@@ -69,6 +68,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds): ...@@ -69,6 +68,7 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
total = qualities['poor'] + qualities['moderate'] + qualities['good'] total = qualities['poor'] + qualities['moderate'] + qualities['good']
quality_sasid = None
if total > 0: if total > 0:
percentage_poor = (qualities['poor'] / total) * 100 percentage_poor = (qualities['poor'] / total) * 100
percentage_good = (qualities['good'] / total) * 100 percentage_good = (qualities['good'] / total) * 100
......
atdb/taskdatabase/tests.py
+ 265
− 1
View file @ ec6ed484
from django.test import TestCase from django.test import TestCase
import json
from .services import calculated_qualities as qualities
from .models import Configuration, Task
# Create your tests here. # Create your tests here.
class TestCalculatedQualities(TestCase):
@classmethod
def setUpTestData(cls):
print("setUpTestData: Run once to set up non-modified data for all class methods.")
# Set up non-modified objects used by all test methods
quality_thresholds = {
"moderate": 20,
"poor": 50,
"overall_poor": 50,
"overall_good": 90,
}
Configuration.objects.create(key="quality_thresholds", value=json.dumps(quality_thresholds))
def setUp(self):
print("setUp: Run once for every test method to setup clean data.")
outputs0 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_summaryIS.tar": {
"added": [],
"deleted": [],
"input_name": "L526107_summaryIS.tar",
"input_size": 495749120,
"input_size_str": "472.78 MB",
"output_name": "L526107_summaryIS.tar",
"output_size": 283791360,
"output_size_str": "270.64 MB",
"rfi_percent": 0,
"size_ratio": 0.5724495486749427
}
},
"uv-coverage": "N/A"
},
}
outputs1 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_SAP002_B073_P000_bf.tar": {
"added": [
"stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit.fits",
"stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit_ldv_psrfits_requantisation.log"
],
"deleted": [
"stokes/SAP2/BEAM73/L526105_SAP2_BEAM73.fits"
],
"input_name": "L526107_SAP002_B073_P000_bf.tar",
"input_size": 20353853440,
"input_size_str": "18.96 GB",
"output_name": "L526107_SAP002_B073_P000_bf.tar",
"output_size": 6024990720,
"output_size_str": "5.61 GB",
"rfi_percent": 11.167,
"size_ratio": 0.2960122876860019
}
},
"uv-coverage": "N/A"
},
}
outputs2 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_SAP002_B073_P000_bf.tar": {
"added": [
"stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit.fits",
"stokes/SAP2/BEAM73/L526105_SAP2_BEAM73_2bit_ldv_psrfits_requantisation.log"
],
"deleted": [
"stokes/SAP2/BEAM73/L526105_SAP2_BEAM73.fits"
],
"input_name": "L526107_SAP002_B073_P000_bf.tar",
"input_size": 20353853440,
"input_size_str": "18.96 GB",
"output_name": "L526107_SAP002_B073_P000_bf.tar",
"output_size": 6024990720,
"output_size_str": "5.61 GB",
"rfi_percent": 22.167,
"size_ratio": 0.2960122876860019
}
},
"uv-coverage": "N/A"
},
}
outputs3 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_SAP002_B072_P000_bf.tar": {
"added": [
"stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit.fits",
"stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit_ldv_psrfits_requantisation.log"
],
"deleted": [
"stokes/SAP2/BEAM72/L526105_SAP2_BEAM72.fits"
],
"input_name": "L526107_SAP002_B072_P000_bf.tar",
"input_size": 20353843200,
"input_size_str": "18.96 GB",
"output_name": "L526107_SAP002_B072_P000_bf.tar",
"output_size": 6024980480,
"output_size_str": "5.61 GB",
"rfi_percent": 31.921,
"size_ratio": 0.2960119335104242
}
},
"uv-coverage": "N/A"
},
}
outputs4 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_SAP002_B070_P000_bf.tar": {
"added": [
"stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit.fits",
"stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit_ldv_psrfits_requantisation.log"
],
"deleted": [
"stokes/SAP2/BEAM70/L526105_SAP2_BEAM70.fits"
],
"input_name": "L526107_SAP002_B070_P000_bf.tar",
"input_size": 20353525760,
"input_size_str": "18.96 GB",
"output_name": "L526107_SAP002_B070_P000_bf.tar",
"output_size": 6024755200,
"output_size_str": "5.61 GB",
"rfi_percent": 52.164,
"size_ratio": 0.2960054818531843
}
},
"uv-coverage": "N/A"
},
}
outputs5 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_SAP002_B072_P000_bf.tar": {
"added": [
"stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit.fits",
"stokes/SAP2/BEAM72/L526105_SAP2_BEAM72_2bit_ldv_psrfits_requantisation.log"
],
"deleted": [
"stokes/SAP2/BEAM72/L526105_SAP2_BEAM72.fits"
],
"input_name": "L526107_SAP002_B072_P000_bf.tar",
"input_size": 20353843200,
"input_size_str": "18.96 GB",
"output_name": "L526107_SAP002_B072_P000_bf.tar",
"output_size": 6024980480,
"output_size_str": "5.61 GB",
"size_ratio": 0.2960119335104242
}
},
"uv-coverage": "N/A"
},
}
outputs6 = {
"quality": {
"details": {},
"observing-conditions": "N/A",
"sensitivity": "N/A",
"summary": {
"L526107_SAP002_B070_P000_bf.tar": {
"added": [
"stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit.fits",
"stokes/SAP2/BEAM70/L526105_SAP2_BEAM70_2bit_ldv_psrfits_requantisation.log"
],
"deleted": [
"stokes/SAP2/BEAM70/L526105_SAP2_BEAM70.fits"
],
"input_name": "L526107_SAP002_B070_P000_bf.tar",
"input_size": 20353525760,
"input_size_str": "18.96 GB",
"output_name": "L526107_SAP002_B070_P000_bf.tar",
"output_size": 6024755200,
"output_size_str": "5.61 GB",
"size_ratio": 0.2960054818531843
}
},
"uv-coverage": "N/A"
},
}
# create a list of Tasks with various values of rfi_percent to test the quality algorithms
# rfi_percent=0, this task should not be included in the calculates
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs0)
# rfi_percent 11,22,31,52
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs1)
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs2)
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs3)
Task.objects.get_or_create(sas_id=54321, status='processed', outputs=outputs4)
# tasks without rfi_percent (so simulating a different pipeline)
Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs5)
Task.objects.get_or_create(sas_id=12345, status='processed', outputs=outputs6)
def test_count_tasks(self):
actual = Task.objects
count = actual.count()
self.assertEqual(count,7)
def test_run_quality_calculation_in_task_model_when_stored(self):
for task in Task.objects.all():
task.new_status = 'stored'
task.save()
# only 4 tasks should now have calculated_qualities
count = 0
for task in Task.objects.all():
if task.calculated_qualities['per_sasid']:
count += 1
self.assertEqual(count,4)
def test_calculated_qualities_per_task(self):
# read the quality thresholds from the test database
quality_thresholds = json.loads(Configuration.objects.get(key="quality_thresholds").value)
# get the tasks for sas_id 54321
tasks_for_this_sasid = Task.objects.filter(sas_id=54321)
# run the algorithms and gather the values
quality_values = {'poor': 0, 'moderate': 0, 'good': 0}
for task in tasks_for_this_sasid:
q = qualities.calculate_qualities(task, tasks_for_this_sasid, quality_thresholds)
try:
key = task.calculated_qualities['per_task']
quality_values[key] = quality_values[key] + 1
except:
# ignore the tasks that have no calculated quality.
pass
self.assertEqual(quality_values,{'poor': 1, 'moderate': 2, 'good': 1})
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment