diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index 82d1526fe06a5b1b2a7d1d842de23064aae99a7a..ae95daa1e14932c2a91e13e6a1352acab105f1e5 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -210,7 +210,7 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
AGGREGATES = ['failed','active','total']
-QUERY_LIMIT_MULTI_CHANGE = 1000
+QUERY_LIMIT_MULTI_CHANGE = 5000
MAX_MONITORING_HISTORY_HOURS = 7 * 24
SERVICES_LATE_WARNING_SECONDS = 1800
@@ -265,4 +265,6 @@ CSRF_COOKIE_NAME = 'atdb_csrftoken'
#SILKY_PYTHON_PROFILER = False
#SILKY_PYTHON_PROFILER_BINARY = False
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
\ No newline at end of file
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
+ACCOUNT_UNIQUE_EMAIL = False
\ No newline at end of file
diff --git a/atdb/run_8000.bat b/atdb/run_8000.bat
index 0096295b7b3ced73c518af4cc14468734346c8d3..bb5d4220ee6ece9fc9ca1d749a9fc2871354a327 100644
--- a/atdb/run_8000.bat
+++ b/atdb/run_8000.bat
@@ -4,7 +4,9 @@ SET DATABASE_PORT=5432
SET DATABASE_NAME=atdb_ldv
SET DATABASE_USER=atdb_admin
SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://sdc-dev.astron.nl/auth
-SET KEYCLOAK_CLIENT_ID=astronauth
-SET KEYCLOAK_CLIENT_SECRET=61239568-0949-4041-a2b8-3ecaae265b0d
+SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
+SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
+SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
+SET LOGIN_REDIRECT_URL=/atdb/
+
python manage.py runserver --settings=atdb.settings.dev
diff --git a/atdb/run_8001.bat b/atdb/run_8001.bat
index 719f3c61100439e97b7bf91e2decb4eaa3eb33fa..3571ee3f0c8cd3f49a262fef814964abd6a10977 100644
--- a/atdb/run_8001.bat
+++ b/atdb/run_8001.bat
@@ -4,7 +4,8 @@ SET DATABASE_PORT=5432
SET DATABASE_NAME=atdb_ldv
SET DATABASE_USER=atdb_admin
SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://sdc-dev.astron.nl/auth
-SET KEYCLOAK_CLIENT_ID=astronauth
-SET KEYCLOAK_CLIENT_SECRET=61239568-0949-4041-a2b8-3ecaae265b0d
+SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
+SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
+SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
+SET LOGIN_REDIRECT_URL=/atdb/
python manage.py runserver 0.0.0.0:8001 --settings=atdb.settings.dev
diff --git a/atdb/taskdatabase/services/calculated_qualities.py b/atdb/taskdatabase/services/calculated_qualities.py
index 3cc29eb3777d70ce8bbddc28c4d53f55c994de52..c064d461698adba12d76acefa097d1a35ea5e3f0 100644
--- a/atdb/taskdatabase/services/calculated_qualities.py
+++ b/atdb/taskdatabase/services/calculated_qualities.py
@@ -5,12 +5,18 @@ logger = logging.getLogger(__name__)
def rfi_percentage_to_quality(rfi_percent, quality_treshold_moderate, quality_treshold_poor):
quality = None
- if rfi_percent >= 0:
+ # > 0 is intentional, by decision of SDCO
+ # 0 means that this task should not be considered in the calculation, because it is probably a summary file.
+ if rfi_percent > 0:
quality = "good"
if rfi_percent >= quality_treshold_moderate:
quality = "moderate"
if rfi_percent > quality_treshold_poor:
quality = "poor"
+
+ elif rfi_percent == 0:
+ quality = "none"
+
return quality
def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index 38c5a69803ea2cbd684b3fd05a5a878080a68af3..1f4b938b974045fa803bcd39a14bc5d554d72802 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 20 Nov 2023
+ <p class="footer"> Version 4 Jan 2024
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/tests/test_calculated_qualities.py b/atdb/taskdatabase/tests/test_calculated_qualities.py
index 09a3df9af7d9abab7ecdcad62e1edd1faf9ce42b..4aa8adf7773f1d0adb97dafa75fb96ec13f73f88 100644
--- a/atdb/taskdatabase/tests/test_calculated_qualities.py
+++ b/atdb/taskdatabase/tests/test_calculated_qualities.py
@@ -74,7 +74,7 @@ class TestCalculatedQualities(TestCase):
if task.calculated_qualities['per_sasid']:
count += 1
- self.assertEqual(count,6)
+ self.assertEqual(count,5)
def test_calculated_qualities(self):
@@ -113,7 +113,7 @@ class TestCalculatedQualities(TestCase):
# ignore the tasks that have no calculated quality.
pass
- self.assertEqual(quality_values,{'poor': 1, 'moderate': 2, 'good': 2})
+ self.assertEqual(quality_values,{'poor': 1, 'moderate': 2, 'good': 1})
# not 90% = good, and not >50% = poor so 'moderate'
self.assertEqual(quality_per_sasid,'moderate')
@@ -163,7 +163,7 @@ class TestCalculatedQualities(TestCase):
# rfi_percentages are 11,22,31,52 for the tasks of this sasid
# with the optimistic parameters that means that the first 3 are 'good', and last one is moderate. No poor
- self.assertEqual(quality_values,{'poor': 0, 'moderate': 1, 'good': 4})
+ self.assertEqual(quality_values,{'poor': 0, 'moderate': 1, 'good': 3})
# 3 out of 4 are 'good', 75% is above the 50% threshold, so 'good'
self.assertEqual(quality_per_sasid,'good')
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 535a8ac4e1be1e6f14aa5f91e5bdc619f46b60c1..f912cfa0e183ebb12a36eee7b097160aa934bebc 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -182,15 +182,15 @@ class QueryView(SingleTableMixin, FilterView):
# https://stackoverflow.com/questions/7763115/django-passing-data-between-views
count = self.object_list.count()
- # q: how do I apply this before getting into get_table_data?
- # current_query_params = self.request.session['current_query_params']
+ # nv4jan2024, removed the user setting for this.
+ # It was never used and potentially overcomplicates things elsewhere.
+ #try:
+ # limit = int(Configuration.objects.get(key='multi_change_limit').value)
+ #except:
+ # limit = settings.QUERY_LIMIT_MULTI_CHANGE
- try:
- limit = int(Configuration.objects.get(key='multi_change_limit').value)
- except:
- limit = settings.QUERY_LIMIT_MULTI_CHANGE
- query_list_of_ids = list(self.object_list.values_list('id'))[:limit]
+ query_list_of_ids = list(self.object_list.values_list('id'))[:settings.QUERY_LIMIT_MULTI_CHANGE]
filtered_tasks_as_list = []
for id in query_list_of_ids:
@@ -334,9 +334,15 @@ class ShowValidationPage(ListView):
context_object_name = 'my_tasks'
#@silk_profile(name='ShowValidationPage')
def get_queryset(self):
-
- #stored_tasks = Task.objects.filter(status__icontains=State.STORED.value)
- #tasks = get_filtered_tasks(self.request, stored_tasks, "sas_id")
+
+ # check filtered_tasks on the session
+ # if it is at its max, then it is not a query targeted at 1 SAS_ID.
+ # in that case clear the filter so that all the SAS_ID's show up on the validation page.
+ #if self.request.session['filtered_tasks_as_list']:
+ # filtered_tasks_on_session = len(self.request.session['filtered_tasks_as_list'])
+ # if filtered_tasks_on_session == settings.QUERY_LIMIT_MULTI_CHANGE:
+ # self.request.session['filtered_tasks_as_list'] = []
+
tasks = get_filtered_tasks(self.request, None, "sas_id").filter(status__icontains=State.STORED.value)
# exclude the failed tasks
@@ -504,8 +510,18 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
my_sort = '-creationTime'
# if there is already a 'filtered_tasks_as_list' on the session, then show that.
+ #try:
+ # filtered_tasks_as_list = request.session['filtered_tasks_as_list']
+ #except:
+ # pass
+
+ # check filtered_tasks on the session
+ # if it is at its max limit, then this is not a query targeted at 1 SAS_ID.
+ # in that case don't apply the filter, so that all SAS_ID's show up on the page.
try:
- filtered_tasks_as_list = request.session['filtered_tasks_as_list']
+ filtered_tasks_on_session = len(request.session['filtered_tasks_as_list'])
+ if filtered_tasks_on_session != settings.QUERY_LIMIT_MULTI_CHANGE:
+ filtered_tasks_as_list = request.session['filtered_tasks_as_list']
except:
pass