Skip to content
Snippets Groups Projects
Commit e17dad1f authored by Nico Vermaas's avatar Nico Vermaas
Browse files

Merge branch 'ingestq-filter' into 'master'

Ingestq filter

See merge request !336
parents 69f7c06a 688b1441
No related branches found
No related tags found
1 merge request!336Ingestq filter
Pipeline #69146 passed
......@@ -210,7 +210,7 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
AGGREGATES = ['failed','active','total']
QUERY_LIMIT_MULTI_CHANGE = 5000
QUERY_LIMIT_MULTI_CHANGE = 10000
MAX_MONITORING_HISTORY_HOURS = 7 * 24
SERVICES_LATE_WARNING_SECONDS = 1800
......
......@@ -16,13 +16,14 @@
<tr>
<td>
{% include "taskdatabase/failures/clear_filter_button.html" %}
<a href="{% url 'task-set-filter' 'processed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed</a>
<a href="{% url 'task-set-filter' 'stored' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored</a>
<a href="{% url 'task-set-filter' 'scrubbing' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing</i></a>
<a href="{% url 'task-set-filter' 'scrubbed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed</a>
<a href="{% url 'task-set-filter' 'archiving' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving</i></a>
<a href="{% url 'task-set-filter' 'archived' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived</a>
<a href="{% url 'task-set-filter' 'finished' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished</a>
<a href="{% url 'task-set-filter' 'staged_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">staged failed</a>
<a href="{% url 'task-set-filter' 'processed_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed failed</a>
<a href="{% url 'task-set-filter' 'stored_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored failed</a>
<a href="{% url 'task-set-filter' 'scrubbing_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing failed</i></a>
<a href="{% url 'task-set-filter' 'scrubbed_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed failed</a>
<a href="{% url 'task-set-filter' 'archiving_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving failed</i></a>
<a href="{% url 'task-set-filter' 'archived_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived failed</a>
<a href="{% url 'task-set-filter' 'finished_failed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished failed</a>
</td>
<td>
......
......@@ -16,8 +16,8 @@
<tr>
<td>
{% include "taskdatabase/ingest/clear_filter_button.html" %}
<a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
<a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
<a href="{% url 'task-set-ingest-filter' 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
<a href="{% url 'task-set-ingest-filter' 'archiving' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
</td>
<td>
......
from django.test import TestCase
from django.test import RequestFactory
from django.contrib.sessions.middleware import SessionMiddleware
from taskdatabase.models import Task
from taskdatabase.views import get_filtered_tasks
class FiltersTest(TestCase):
def setUp(self):
self.task1 = Task.objects.create(sas_id=12345,status='defined')
self.task2 = Task.objects.create(sas_id=12345,status='scrubbed')
self.task3 = Task.objects.create(sas_id=12345,status='scrubbed')
self.task4 = Task.objects.create(sas_id=66666,status='scrubbed')
self.task5 = Task.objects.create(sas_id=66666,status='archived_failed')
def test_without_filter(self):
count = 0
for task in Task.objects.all():
count += 1
self.assertEqual(count,5)
def test_with_ingest_filter(self):
# create a request object
request = RequestFactory().get('/atdb/ingest')
# this simulates the 'Queue (scrubbed)' filter on the ingest page
request.session = {'ingest_filter': 'scrubbed'}
middleware = SessionMiddleware()
middleware.process_request(request)
request.session.save()
# after aggregating per sas_id, 2 objects with status 'scrubbed' remain
tasks = get_filtered_tasks(request, None, "sas_id")
self.assertEqual(tasks.count(),2)
\ No newline at end of file
from django.test import TestCase
from taskdatabase.models import Task
class TaskModelTestCase(TestCase):
class PathToLTATest(TestCase):
def setUp(self):
# Create tasks for testing
......
......@@ -110,6 +110,7 @@ urlpatterns = [
path('tasks/sort-tasks/<sort>/<redirect_to_page>', views.SortTasks, name='sort-tasks'),
path('tasks/set_filter/<filter>/<redirect_to_page>', views.TaskSetFilter, name='task-set-filter'),
path('tasks/set_ingest_filter/<filter>', views.TaskSetIngestFilter, name='task-set-ingest-filter'),
path('tasks/set_active_filter/<redirect_to_page>', views.TaskSetActiveFilter, name='task-set-active-filter'),
path('tasks/task-set-onhold-filter/<onhold>/<redirect_to_page>', views.TaskSetOnHoldFilter, name='task-set-onhold-filter'),
path('tasks/clear_filter/<redirect_to_page>', views.TaskClearFilter, name='clear-filter'),
......
......@@ -180,15 +180,10 @@ class QueryView(SingleTableMixin, FilterView):
def get_table_data(self):
# https://stackoverflow.com/questions/7763115/django-passing-data-between-views
count = self.object_list.count()
# nv4jan2024, removed the user setting for this.
# It was never used and potentially overcomplicates things elsewhere.
#try:
# limit = int(Configuration.objects.get(key='multi_change_limit').value)
#except:
# limit = settings.QUERY_LIMIT_MULTI_CHANGE
#nv:16jan2024, this would be scary, but perhaps needed
#how large is the list that can go on the session?
#query_list_of_ids = list(self.object_list.values_list('id'))
query_list_of_ids = list(self.object_list.values_list('id'))[:settings.QUERY_LIMIT_MULTI_CHANGE]
......@@ -545,34 +540,57 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
my_sort = '-creationTime'
# if there is already a 'filtered_tasks_as_list' on the session, then show that.
#try:
# filtered_tasks_as_list = request.session['filtered_tasks_as_list']
#except:
# pass
# check filtered_tasks on the session
# if it is at its max limit, then this is not a query targeted at 1 SAS_ID.
# in that case don't apply the filter, so that all SAS_ID's show up on the page.
# nv: 15jan2024, this disturbs how users now work with the filter, need to find a different solution.
# this is a way to propagate an earlier filter from the FILTER page to several pages
# nv:16jan24, this has a potential issue, because it limits to 5000 results
# users said that that is no problem, the advantage of this functionalty outweighs this potential issue.
try:
filtered_tasks_on_session = len(request.session['filtered_tasks_as_list'])
if filtered_tasks_on_session != settings.QUERY_LIMIT_MULTI_CHANGE:
filtered_tasks_as_list = request.session['filtered_tasks_as_list']
except:
pass
if filtered_tasks_as_list:
# there is an earlier list of filtered tasks on the session, use that as starting point
if pre_filtered_tasks:
filtered_tasks = pre_filtered_tasks.filter(id__in=filtered_tasks_as_list)
# there is a list of pre-filtered tasks given as extra argument to this function (most pages do that)
filtered_tasks = pre_filtered_tasks.filter(id__in=filtered_tasks_as_list).defer('inputs','outputs')
# check if there is an ingest filter active
try:
ingest_filter = request.session['ingest_filter']
if ingest_filter != 'all':
if type(ingest_filter) is list:
filtered_tasks = filtered_tasks.filter(status__in=ingest_filter)
else:
filtered_tasks = filtered_tasks.filter(status__icontains=ingest_filter)
except:
pass
else:
filtered_tasks = Task.objects.filter(id__in=filtered_tasks_as_list)
# there is no list of filtered tasks given as extra argument to this function
filtered_tasks = Task.objects.filter(id__in=filtered_tasks_as_list).defer('inputs','outputs')
else:
# there is no list of previously filtered tasks on the session, use all tasks
if pre_filtered_tasks:
filtered_tasks = pre_filtered_tasks
else:
#filtered_tasks = Task.objects.all()
filtered_tasks = Task.objects.defer('inputs','outputs')
# check if there is an ingest filter active
try:
ingest_filter = request.session['ingest_filter']
if ingest_filter != 'all':
if type(ingest_filter) is list:
filtered_tasks = filtered_tasks.filter(status__in=ingest_filter)
else:
filtered_tasks = filtered_tasks.filter(status__icontains=ingest_filter)
except:
pass
# check if there is a status filter active
try:
status_filter = request.session['task_filter']
......@@ -1326,6 +1344,9 @@ def TaskSetFilter(request, filter, redirect_to_page):
return redirect_with_params('index', '?page=1')
def TaskSetIngestFilter(request, filter):
request.session['ingest_filter'] = filter
return redirect_with_params('ingest', '?page=1')
# set the defined list of ACTIVE_STATUSSES on the session, used later by the 'get_searched_tasks' mechanism
def TaskSetActiveFilter(request, redirect_to_page):
......@@ -1351,6 +1372,7 @@ def TaskClearFilter(request, redirect_to_page):
request.session['filtered_tasks_as_list'] = []
request.session['search_box'] = ''
request.session['filtered'] = False
request.session['ingest_filter'] = 'all'
try:
return redirect(redirect_to_page)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment