Merge branch 'SDC-1186-bugfix-calcq-validation' into 'master'

solving bug SDCH-4164

See merge request !327

atdb/atdb/settings/base.py

@@ -210,7 +210,7 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
 STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
 AGGREGATES = ['failed','active','total']
 
-QUERY_LIMIT_MULTI_CHANGE = 1000
+QUERY_LIMIT_MULTI_CHANGE = 5000
 MAX_MONITORING_HISTORY_HOURS = 7 * 24
 SERVICES_LATE_WARNING_SECONDS = 1800
 
@@ -266,3 +266,5 @@ CSRF_COOKIE_NAME = 'atdb_csrftoken'
 #SILKY_PYTHON_PROFILER_BINARY = False
 
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+ACCOUNT_UNIQUE_EMAIL = False
\ No newline at end of file

atdb/run_8000.bat

@@ -4,7 +4,9 @@ SET DATABASE_PORT=5432
 SET DATABASE_NAME=atdb_ldv
 SET DATABASE_USER=atdb_admin
 SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://sdc-dev.astron.nl/auth
-SET KEYCLOAK_CLIENT_ID=astronauth
-SET KEYCLOAK_CLIENT_SECRET=61239568-0949-4041-a2b8-3ecaae265b0d
+SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
+SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
+SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
+SET LOGIN_REDIRECT_URL=/atdb/
+
 python manage.py runserver --settings=atdb.settings.dev

atdb/run_8001.bat

@@ -4,7 +4,8 @@ SET DATABASE_PORT=5432
 SET DATABASE_NAME=atdb_ldv
 SET DATABASE_USER=atdb_admin
 SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://sdc-dev.astron.nl/auth
-SET KEYCLOAK_CLIENT_ID=astronauth
-SET KEYCLOAK_CLIENT_SECRET=61239568-0949-4041-a2b8-3ecaae265b0d
+SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
+SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
+SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
+SET LOGIN_REDIRECT_URL=/atdb/
 python manage.py runserver 0.0.0.0:8001 --settings=atdb.settings.dev

atdb/taskdatabase/services/calculated_qualities.py

@@ -5,12 +5,18 @@ logger = logging.getLogger(__name__)
 
 def rfi_percentage_to_quality(rfi_percent, quality_treshold_moderate, quality_treshold_poor):
     quality = None
-    if rfi_percent >= 0:
+    # > 0 is intentional, by decision of SDCO
+    # 0 means that this task should not be considered in the calculation, because it is probably a summary file.
+    if rfi_percent > 0:
         quality = "good"
         if rfi_percent >= quality_treshold_moderate:
             quality = "moderate"
         if rfi_percent > quality_treshold_poor:
             quality = "poor"
+
+    elif rfi_percent == 0:
+        quality = "none"
+
     return quality
 
 def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 20 Nov 2023
+    <p class="footer"> Version 4 Jan 2024
 </div>
 
 {% include 'taskdatabase/refresh.html' %}

atdb/taskdatabase/tests/test_calculated_qualities.py

@@ -74,7 +74,7 @@ class TestCalculatedQualities(TestCase):
             if task.calculated_qualities['per_sasid']:
                 count += 1
 
-        self.assertEqual(count,6)
+        self.assertEqual(count,5)
 
 
     def test_calculated_qualities(self):
@@ -113,7 +113,7 @@ class TestCalculatedQualities(TestCase):
                 # ignore the tasks that have no calculated quality.
                 pass
 
-        self.assertEqual(quality_values,{'poor': 1, 'moderate': 2, 'good': 2})
+        self.assertEqual(quality_values,{'poor': 1, 'moderate': 2, 'good': 1})
 
         # not 90% = good, and not >50% = poor so 'moderate'
         self.assertEqual(quality_per_sasid,'moderate')
@@ -163,7 +163,7 @@ class TestCalculatedQualities(TestCase):
 
         # rfi_percentages are 11,22,31,52 for the tasks of this sasid
         # with the optimistic parameters that means that the first 3 are 'good', and last one is moderate. No poor
-        self.assertEqual(quality_values,{'poor': 0, 'moderate': 1, 'good': 4})
+        self.assertEqual(quality_values,{'poor': 0, 'moderate': 1, 'good': 3})
 
         # 3 out of 4 are 'good', 75% is above the 50% threshold, so 'good'
         self.assertEqual(quality_per_sasid,'good')

atdb/taskdatabase/views.py

@@ -182,15 +182,15 @@ class QueryView(SingleTableMixin, FilterView):
         # https://stackoverflow.com/questions/7763115/django-passing-data-between-views
         count = self.object_list.count()
 
-        # q: how do I apply this before getting into get_table_data?
-        # current_query_params = self.request.session['current_query_params']
+        # nv4jan2024, removed the user setting for this.
+        # It was never used and potentially overcomplicates things elsewhere.
+        #try:
+        #    limit = int(Configuration.objects.get(key='multi_change_limit').value)
+        #except:
+        #    limit = settings.QUERY_LIMIT_MULTI_CHANGE
 
-        try:
-            limit = int(Configuration.objects.get(key='multi_change_limit').value)
-        except:
-            limit = settings.QUERY_LIMIT_MULTI_CHANGE
 
-        query_list_of_ids = list(self.object_list.values_list('id'))[:limit]
+        query_list_of_ids = list(self.object_list.values_list('id'))[:settings.QUERY_LIMIT_MULTI_CHANGE]
 
         filtered_tasks_as_list = []
         for id in query_list_of_ids:
@@ -335,8 +335,14 @@ class ShowValidationPage(ListView):
     #@silk_profile(name='ShowValidationPage')
     def get_queryset(self):
 
-        #stored_tasks = Task.objects.filter(status__icontains=State.STORED.value)
-        #tasks = get_filtered_tasks(self.request, stored_tasks, "sas_id")
+        # check filtered_tasks on the session
+        # if it is at its max, then it is not a query targeted at 1 SAS_ID.
+        # in that case clear the filter so that all the SAS_ID's show up on the validation page.
+        #if self.request.session['filtered_tasks_as_list']:
+        #    filtered_tasks_on_session = len(self.request.session['filtered_tasks_as_list'])
+        #    if filtered_tasks_on_session == settings.QUERY_LIMIT_MULTI_CHANGE:
+        #        self.request.session['filtered_tasks_as_list'] = []
+
         tasks = get_filtered_tasks(self.request, None, "sas_id").filter(status__icontains=State.STORED.value)
 
         # exclude the failed tasks
@@ -504,7 +510,17 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
         my_sort = '-creationTime'
 
     # if there is already a 'filtered_tasks_as_list' on the session, then show that.
+    #try:
+    #    filtered_tasks_as_list = request.session['filtered_tasks_as_list']
+    #except:
+    #    pass
+
+    # check filtered_tasks on the session
+    # if it is at its max limit, then this is not a query targeted at 1 SAS_ID.
+    # in that case don't apply the filter, so that all SAS_ID's show up on the page.
     try:
+        filtered_tasks_on_session = len(request.session['filtered_tasks_as_list'])
+        if filtered_tasks_on_session != settings.QUERY_LIMIT_MULTI_CHANGE:
             filtered_tasks_as_list = request.session['filtered_tasks_as_list']
     except:
         pass