diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index ae95daa1e14932c2a91e13e6a1352acab105f1e5..31e142274c70e08f2307e10bfd435dfaf2742ed4 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -210,7 +210,7 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
AGGREGATES = ['failed','active','total']
-QUERY_LIMIT_MULTI_CHANGE = 5000
+QUERY_LIMIT_MULTI_CHANGE = 10000
MAX_MONITORING_HISTORY_HOURS = 7 * 24
SERVICES_LATE_WARNING_SECONDS = 1800
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html
index 482c707d93fb405cf0e21279dbe658852951fc89..ff52a498175e65560a1e5106358844d0fce21101 100644
--- a/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html
@@ -16,13 +16,14 @@
<tr>
<td>
{% include "taskdatabase/failures/clear_filter_button.html" %}
- <a href="{% url 'task-set-filter' 'processed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed</a>
- <a href="{% url 'task-set-filter' 'stored' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored</a>
- <a href="{% url 'task-set-filter' 'scrubbing' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing</i></a>
- <a href="{% url 'task-set-filter' 'scrubbed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed</a>
- <a href="{% url 'task-set-filter' 'archiving' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving</i></a>
- <a href="{% url 'task-set-filter' 'archived' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived</a>
- <a href="{% url 'task-set-filter' 'finished' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished</a>
+ <a href="{% url 'task-set-filter' 'staged_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">staged failed</a>
+ <a href="{% url 'task-set-filter' 'processed_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed failed</a>
+ <a href="{% url 'task-set-filter' 'stored_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored failed</a>
+ <a href="{% url 'task-set-filter' 'scrubbing_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing failed</i></a>
+ <a href="{% url 'task-set-filter' 'scrubbed_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed failed</a>
+ <a href="{% url 'task-set-filter' 'archiving_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving failed</i></a>
+ <a href="{% url 'task-set-filter' 'archived_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived failed</a>
+ <a href="{% url 'task-set-filter' 'finished_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished failed</a>
</td>
<td>
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
index 782d2ec1b8d3cc5d9d4a1f2017873af694b56ef6..118ccd3c2f664f1f55c376cb42a9ffe29de95fdd 100644
--- a/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
@@ -16,8 +16,8 @@
<tr>
<td>
{% include "taskdatabase/ingest/clear_filter_button.html" %}
- <a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
- <a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+ <a href="{% url 'task-set-ingest-filter' 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
+ <a href="{% url 'task-set-ingest-filter' 'archiving' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
</td>
<td>
diff --git a/atdb/taskdatabase/tests/test_filters.py b/atdb/taskdatabase/tests/test_filters.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2a5b361e2867e598ae34f8048180048b26eb5de
--- /dev/null
+++ b/atdb/taskdatabase/tests/test_filters.py
@@ -0,0 +1,36 @@
+from django.test import TestCase
+from django.test import RequestFactory
+from django.contrib.sessions.middleware import SessionMiddleware
+from taskdatabase.models import Task
+from taskdatabase.views import get_filtered_tasks
+
+class FiltersTest(TestCase):
+ def setUp(self):
+
+ self.task1 = Task.objects.create(sas_id=12345,status='defined')
+ self.task2 = Task.objects.create(sas_id=12345,status='scrubbed')
+ self.task3 = Task.objects.create(sas_id=12345,status='scrubbed')
+ self.task4 = Task.objects.create(sas_id=66666,status='scrubbed')
+ self.task5 = Task.objects.create(sas_id=66666,status='archived_failed')
+
+ def test_without_filter(self):
+ count = 0
+ for task in Task.objects.all():
+ count += 1
+
+ self.assertEqual(count,5)
+
+ def test_with_ingest_filter(self):
+ # create a request object
+ request = RequestFactory().get('/atdb/ingest')
+
+ # this simulates the 'Queue (scrubbed)' filter on the ingest page
+ request.session = {'ingest_filter': 'scrubbed'}
+
+ middleware = SessionMiddleware()
+ middleware.process_request(request)
+ request.session.save()
+
+ # after aggregating per sas_id, 2 objects with status 'scrubbed' remain
+ tasks = get_filtered_tasks(request, None, "sas_id")
+ self.assertEqual(tasks.count(),2)
\ No newline at end of file
diff --git a/atdb/taskdatabase/tests/test_path_to_lta.py b/atdb/taskdatabase/tests/test_path_to_lta.py
index 5f650159ca48f4946b1d5c96c1a8ef004ff4f7a2..38d884bddde566427ac47cd96696c9adfedbe173 100644
--- a/atdb/taskdatabase/tests/test_path_to_lta.py
+++ b/atdb/taskdatabase/tests/test_path_to_lta.py
@@ -1,7 +1,7 @@
from django.test import TestCase
from taskdatabase.models import Task
-class TaskModelTestCase(TestCase):
+class PathToLTATest(TestCase):
def setUp(self):
# Create tasks for testing
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index f292639079a899f970f26da8bc94f9dd50d373d5..b3da85f89777e789cbf2f2ae349fb07699b071f5 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -110,6 +110,7 @@ urlpatterns = [
path('tasks/sort-tasks/<sort>/<redirect_to_page>', views.SortTasks, name='sort-tasks'),
path('tasks/set_filter/<filter>/<redirect_to_page>', views.TaskSetFilter, name='task-set-filter'),
+ path('tasks/set_ingest_filter/<filter>', views.TaskSetIngestFilter, name='task-set-ingest-filter'),
path('tasks/set_active_filter/<redirect_to_page>', views.TaskSetActiveFilter, name='task-set-active-filter'),
path('tasks/task-set-onhold-filter/<onhold>/<redirect_to_page>', views.TaskSetOnHoldFilter, name='task-set-onhold-filter'),
path('tasks/clear_filter/<redirect_to_page>', views.TaskClearFilter, name='clear-filter'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 1e7a088abaaf74491b2ea1f217824f19b9d0c68f..85f3cf553cda1e97dda609e3e7127ce921177bfe 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -180,15 +180,10 @@ class QueryView(SingleTableMixin, FilterView):
def get_table_data(self):
# https://stackoverflow.com/questions/7763115/django-passing-data-between-views
- count = self.object_list.count()
-
- # nv4jan2024, removed the user setting for this.
- # It was never used and potentially overcomplicates things elsewhere.
- #try:
- # limit = int(Configuration.objects.get(key='multi_change_limit').value)
- #except:
- # limit = settings.QUERY_LIMIT_MULTI_CHANGE
+ #nv:16jan2024, this would be scary, but perhaps needed
+ #how large is the list that can go on the session?
+ #query_list_of_ids = list(self.object_list.values_list('id'))
query_list_of_ids = list(self.object_list.values_list('id'))[:settings.QUERY_LIMIT_MULTI_CHANGE]
@@ -545,34 +540,57 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
my_sort = '-creationTime'
# if there is already a 'filtered_tasks_as_list' on the session, then show that.
- #try:
- # filtered_tasks_as_list = request.session['filtered_tasks_as_list']
- #except:
- # pass
-
- # check filtered_tasks on the session
- # if it is at its max limit, then this is not a query targeted at 1 SAS_ID.
- # in that case don't apply the filter, so that all SAS_ID's show up on the page.
- # nv: 15jan2024, this disturbs how users now work with the filter, need to find a different solution.
+ # this is a way to propagate an earlier filter from the FILTER page to several pages
+
+ # nv:16jan24, this has a potential issue, because it limits to 5000 results
+ # users said that that is no problem, the advantage of this functionalty outweighs this potential issue.
try:
- filtered_tasks_on_session = len(request.session['filtered_tasks_as_list'])
- if filtered_tasks_on_session != settings.QUERY_LIMIT_MULTI_CHANGE:
- filtered_tasks_as_list = request.session['filtered_tasks_as_list']
+ filtered_tasks_as_list = request.session['filtered_tasks_as_list']
except:
pass
+
if filtered_tasks_as_list:
+ # there is an earlier list of filtered tasks on the session, use that as starting point
+
if pre_filtered_tasks:
- filtered_tasks = pre_filtered_tasks.filter(id__in=filtered_tasks_as_list)
+ # there is a list of pre-filtered tasks given as extra argument to this function (most pages do that)
+ filtered_tasks = pre_filtered_tasks.filter(id__in=filtered_tasks_as_list).defer('inputs','outputs')
+
+ # check if there is an ingest filter active
+ try:
+ ingest_filter = request.session['ingest_filter']
+ if ingest_filter != 'all':
+ if type(ingest_filter) is list:
+ filtered_tasks = filtered_tasks.filter(status__in=ingest_filter)
+ else:
+ filtered_tasks = filtered_tasks.filter(status__icontains=ingest_filter)
+ except:
+ pass
+
else:
- filtered_tasks = Task.objects.filter(id__in=filtered_tasks_as_list)
+ # there is no list of filtered tasks given as extra argument to this function
+ filtered_tasks = Task.objects.filter(id__in=filtered_tasks_as_list).defer('inputs','outputs')
else:
+
+ # there is no list of previously filtered tasks on the session, use all tasks
if pre_filtered_tasks:
filtered_tasks = pre_filtered_tasks
else:
#filtered_tasks = Task.objects.all()
filtered_tasks = Task.objects.defer('inputs','outputs')
+ # check if there is an ingest filter active
+ try:
+ ingest_filter = request.session['ingest_filter']
+ if ingest_filter != 'all':
+ if type(ingest_filter) is list:
+ filtered_tasks = filtered_tasks.filter(status__in=ingest_filter)
+ else:
+ filtered_tasks = filtered_tasks.filter(status__icontains=ingest_filter)
+ except:
+ pass
+
# check if there is a status filter active
try:
status_filter = request.session['task_filter']
@@ -1326,6 +1344,9 @@ def TaskSetFilter(request, filter, redirect_to_page):
return redirect_with_params('index', '?page=1')
+def TaskSetIngestFilter(request, filter):
+ request.session['ingest_filter'] = filter
+ return redirect_with_params('ingest', '?page=1')
# set the defined list of ACTIVE_STATUSSES on the session, used later by the 'get_searched_tasks' mechanism
def TaskSetActiveFilter(request, redirect_to_page):
@@ -1351,6 +1372,7 @@ def TaskClearFilter(request, redirect_to_page):
request.session['filtered_tasks_as_list'] = []
request.session['search_box'] = ''
request.session['filtered'] = False
+ request.session['ingest_filter'] = 'all'
try:
return redirect(redirect_to_page)