better ingestq filter and restore previous functionality

atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html

@@ -16,13 +16,14 @@
              <tr>
                <td>
                    {% include "taskdatabase/failures/clear_filter_button.html" %}
-                    <a href="{% url 'task-set-filter' 'processed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed</a>
-                    <a href="{% url 'task-set-filter' 'stored' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored</a>
-                    <a href="{% url 'task-set-filter' 'scrubbing' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing</i></a>
-                    <a href="{% url 'task-set-filter' 'scrubbed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed</a>
-                    <a href="{% url 'task-set-filter' 'archiving' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving</i></a>
-                    <a href="{% url 'task-set-filter' 'archived' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived</a>
-                    <a href="{% url 'task-set-filter' 'finished' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished</a>
+                    <a href="{% url 'task-set-filter' 'staged_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">staged failed</a>
+                    <a href="{% url 'task-set-filter' 'processed_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed failed</a>
+                    <a href="{% url 'task-set-filter' 'stored_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored failed</a>
+                    <a href="{% url 'task-set-filter' 'scrubbing_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing failed</i></a>
+                    <a href="{% url 'task-set-filter' 'scrubbed_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed failed</a>
+                    <a href="{% url 'task-set-filter' 'archiving_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving failed</i></a>
+                    <a href="{% url 'task-set-filter' 'archived_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived failed</a>
+                    <a href="{% url 'task-set-filter' 'finished_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished failed</a>
 
                </td>
                  <td>

atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html

@@ -16,8 +16,8 @@
              <tr>
                <td>
                    {% include "taskdatabase/ingest/clear_filter_button.html" %}
-                    <a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
-                    <a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+                    <a href="{% url 'task-set-ingest-filter' 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
+                    <a href="{% url 'task-set-ingest-filter' 'archiving' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
 
                </td>
                  <td>

atdb/taskdatabase/urls.py

@@ -110,6 +110,7 @@ urlpatterns = [
 
     path('tasks/sort-tasks/<sort>/<redirect_to_page>', views.SortTasks, name='sort-tasks'),
     path('tasks/set_filter/<filter>/<redirect_to_page>', views.TaskSetFilter, name='task-set-filter'),
+    path('tasks/set_ingest_filter/<filter>', views.TaskSetIngestFilter, name='task-set-ingest-filter'),
     path('tasks/set_active_filter/<redirect_to_page>', views.TaskSetActiveFilter, name='task-set-active-filter'),
     path('tasks/task-set-onhold-filter/<onhold>/<redirect_to_page>', views.TaskSetOnHoldFilter, name='task-set-onhold-filter'),
     path('tasks/clear_filter/<redirect_to_page>', views.TaskClearFilter, name='clear-filter'),

atdb/taskdatabase/views.py

@@ -545,34 +545,57 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
         my_sort = '-creationTime'
 
     # if there is already a 'filtered_tasks_as_list' on the session, then show that.
-    #try:
-    #    filtered_tasks_as_list = request.session['filtered_tasks_as_list']
-    #except:
-    #    pass
-
-    # check filtered_tasks on the session
-    # if it is at its max limit, then this is not a query targeted at 1 SAS_ID.
-    # in that case don't apply the filter, so that all SAS_ID's show up on the page.
-    # nv: 15jan2024, this disturbs how users now work with the filter, need to find a different solution.
+    # this is a way to propagate an earlier filter from the FILTER page to several pages
+
+    # nv:16jan24, this has a potential issue, because it limits to 5000 results
+    # users said that that is no problem, the advantage of this functionalty outweighs this potential issue.
     try:
-        filtered_tasks_on_session = len(request.session['filtered_tasks_as_list'])
-        if filtered_tasks_on_session != settings.QUERY_LIMIT_MULTI_CHANGE:
-            filtered_tasks_as_list = request.session['filtered_tasks_as_list']
+        filtered_tasks_as_list = request.session['filtered_tasks_as_list']
     except:
         pass
 
+
     if filtered_tasks_as_list:
+        # there is an earlier list of filtered tasks on the session, use that as starting point
+
         if pre_filtered_tasks:
-            filtered_tasks = pre_filtered_tasks.filter(id__in=filtered_tasks_as_list)
+            # there is a list of pre-filtered tasks given as extra argument to this function (most pages do that)
+            filtered_tasks = pre_filtered_tasks.filter(id__in=filtered_tasks_as_list).defer('inputs','outputs')
+
+            # check if there is an ingest filter active
+            try:
+                ingest_filter = request.session['ingest_filter']
+                if ingest_filter != 'all':
+                    if type(ingest_filter) is list:
+                        filtered_tasks = filtered_tasks.filter(status__in=ingest_filter)
+                    else:
+                        filtered_tasks = filtered_tasks.filter(status__icontains=ingest_filter)
+            except:
+                pass
+
         else:
-            filtered_tasks = Task.objects.filter(id__in=filtered_tasks_as_list)
+            # there is no list of filtered tasks given as extra argument to this function
+            filtered_tasks = Task.objects.filter(id__in=filtered_tasks_as_list).defer('inputs','outputs')
     else:
+
+        # there is no list of previously filtered tasks on the session, use all tasks
         if pre_filtered_tasks:
             filtered_tasks = pre_filtered_tasks
         else:
             #filtered_tasks = Task.objects.all()
             filtered_tasks = Task.objects.defer('inputs','outputs')
 
+        # check if there is an ingest filter active
+        try:
+            ingest_filter = request.session['ingest_filter']
+            if ingest_filter != 'all':
+                if type(ingest_filter) is list:
+                    filtered_tasks = filtered_tasks.filter(status__in=ingest_filter)
+                else:
+                    filtered_tasks = filtered_tasks.filter(status__icontains=ingest_filter)
+        except:
+            pass
+
         # check if there is a status filter active
         try:
             status_filter = request.session['task_filter']
@@ -1326,6 +1349,9 @@ def TaskSetFilter(request, filter, redirect_to_page):
 
     return redirect_with_params('index', '?page=1')
 
+def TaskSetIngestFilter(request, filter):
+    request.session['ingest_filter'] = filter
+    return redirect_with_params('ingest', '?page=1')
 
 # set the defined list of ACTIVE_STATUSSES on the session, used later by the 'get_searched_tasks' mechanism
 def TaskSetActiveFilter(request, redirect_to_page):
@@ -1351,6 +1377,7 @@ def TaskClearFilter(request, redirect_to_page):
     request.session['filtered_tasks_as_list'] = []
     request.session['search_box'] = ''
     request.session['filtered'] = False
+    request.session['ingest_filter'] = 'all'
 
     try:
         return redirect(redirect_to_page)