Merge branch 'add_unique_aggregation_key_endpoint' into 'master'

Add unique aggregation key endpoint

See merge request !174

atdb/requirements/base.txt

 Django==3.1.4
 djangorestframework==3.12.2
 django-filter==2.3.0
-psycopg2==2.8.6
+psycopg2-binary==2.9.3
 django-cors-headers==3.6.0
 django-extensions==3.1.0
 django-bootstrap-pagination==1.7.0

atdb/taskdatabase/services/algorithms.py

@@ -554,3 +554,8 @@ def construct_dashboard_html(request, selection):
         results_logs = construct_logs_per_workflow_html(request, log_records)
 
     return results_tasks, results_logs
+
+
+def unique_values_for_aggregation_key(queryset, aggregation_key):
+
+    return list(map(lambda x: x[aggregation_key], queryset.values(aggregation_key).distinct()))
\ No newline at end of file

atdb/taskdatabase/urls.py

@@ -61,6 +61,9 @@ urlpatterns = [
     path('tasks/get_size/', views.GetSizeView.as_view(), name='get-size-view'),
     # /atdb/get_min_start_and_max_end_time?sas_id=65005
     path('get_min_start_and_max_end_time/', views.GetMinMaxTimeView.as_view(), name='get-min-start-and-max-end-time-view'),
+    # /atdb/get_unique_values_for_key/<aggregation_key>
+    path('get_unique_values_for_key/<str:aggregation_key>', views.GetUniqueValuesForKey.as_view(),
+         name='get-unique-values-for-key-view'),
 
     # --- controller resources ---
     path('tasks/<int:pk>/setstatus/<new_status>/<page>', views.TaskSetStatus, name='task-setstatus-view'),

atdb/taskdatabase/views.py

-
 import logging
 import json
 
@@ -20,6 +19,7 @@ from django_tables2 import SingleTableView
 
 from django.shortcuts import render, redirect
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from rest_framework.request import Request
 
 from django.conf import settings
 from .models import Task, Status, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor
@@ -30,9 +30,9 @@ from .serializers import \
     TaskWriteSerializer, \
     TaskReadSerializer, \
     TaskReadSerializerFast, \
-    WorkflowSerializer,\
-    LogEntrySerializer,\
-    ConfigurationSerializer,\
+    WorkflowSerializer, \
+    LogEntrySerializer, \
+    ConfigurationSerializer, \
     JobSerializer, \
     PostProcessingRuleSerializer, \
     MonitorSerializer
@@ -41,9 +41,9 @@ from .services import algorithms
 
 logger = logging.getLogger(__name__)
 
+
 # ---------- filters (in the REST API) ---------
 class TaskFilter(filters.FilterSet):
-
     class Meta:
         model = Task
 
@@ -51,18 +51,19 @@ class TaskFilter(filters.FilterSet):
             'task_type': ['exact', 'icontains', 'in'],
             'creationTime': ['icontains'],
             'filter': ['exact', 'icontains'],
-            'workflow__id' : ['exact', 'icontains'],
+            'workflow__id': ['exact', 'icontains'],
             'project': ['exact', 'icontains'],
-            'sas_id': ['exact', 'icontains','in'],
+            'sas_id': ['exact', 'icontains', 'in'],
             'status': ['exact', 'icontains', 'in', 'startswith'],
             'purge_policy': ['exact'],
-            'priority': ['exact','lte','gte'],
+            'priority': ['exact', 'lte', 'gte'],
             'resume': ['exact'],
             # http://localhost:8000/atdb/tasks/?predecessor__isnull=True
             'predecessor': ['isnull'],
             'predecessor__status': ['exact', 'icontains', 'in', 'startswith'],
         }
 
+
 class TaskFilterQueryPage(filters.FilterSet):
     resume = django_filters.BooleanFilter(lookup_expr='exact', label='resuming')
 
@@ -71,19 +72,18 @@ class TaskFilterQueryPage(filters.FilterSet):
 
         fields = {
             'id': ['exact', 'gte', 'lte'],
-            #'task_type': ['exact','in'],
+            # 'task_type': ['exact','in'],
             'workflow__id': ['exact'],
             'filter': ['exact', 'icontains'],
             'priority': ['exact', 'gte', 'lte'],
             'status': ['icontains', 'in'],
-            'project': ['exact', 'icontains','in'],
+            'project': ['exact', 'icontains', 'in'],
             'sas_id': ['exact', 'icontains', 'in'],
-            #'resume': ['exact'],
+            # 'resume': ['exact'],
         }
 
 
 class WorkflowFilter(filters.FilterSet):
-
     class Meta:
         model = Workflow
 
@@ -95,18 +95,17 @@ class WorkflowFilter(filters.FilterSet):
 
 
 class LogEntryFilter(filters.FilterSet):
-
     class Meta:
         model = LogEntry
 
         fields = {
             'task__id': ['exact'],
             'step_name': ['exact', 'icontains', 'in', 'startswith'],
-            'status': ['exact','in'],
+            'status': ['exact', 'in'],
         }
 
-class ConfigurationFilter(filters.FilterSet):
 
+class ConfigurationFilter(filters.FilterSet):
     class Meta:
         model = Configuration
 
@@ -116,8 +115,8 @@ class ConfigurationFilter(filters.FilterSet):
             'value': ['exact', 'icontains'],
         }
 
-class JobFilter(filters.FilterSet):
 
+class JobFilter(filters.FilterSet):
     class Meta:
         model = Job
 
@@ -127,8 +126,8 @@ class JobFilter(filters.FilterSet):
             'job_id': ['exact'],
         }
 
-class PostProcessingFilter(filters.FilterSet):
 
+class PostProcessingFilter(filters.FilterSet):
     class Meta:
         model = PostProcessingRule
 
@@ -139,6 +138,7 @@ class PostProcessingFilter(filters.FilterSet):
             'workflow_to_apply__id': ['exact'],
         }
 
+
 # ---------- Tables2 Views (experimental) -----------
 # implementation with tables2: http://localhost:8000/atdb/tables2
 class QueryView(SingleTableMixin, FilterView):
@@ -192,7 +192,7 @@ class IndexView(ListView):
         # check if there is a 'task_filter' put on the session
         try:
             filter = self.request.session['task_filter']
-            if filter!='all':
+            if filter != 'all':
                 tasks = get_searched_tasks(filter, sort)
         except:
             pass
@@ -200,7 +200,7 @@ class IndexView(ListView):
         # check if there is a 'task_onhold_filter' put on the session
         try:
             onhold = self.request.session['task_onhold_filter']
-            if onhold!=None:
+            if onhold != None:
                 tasks = tasks.filter(resume=not onhold)
 
         except:
@@ -254,7 +254,6 @@ class TaskTables2View(SingleTableView):
 
 
 def TaskDetails(request, id=0, page=0):
-
     try:
         task = Task.objects.get(id=id)
 
@@ -277,9 +276,7 @@ def TaskDetails(request, id=0, page=0):
     log_entries = LogEntry.objects.filter(task=task).order_by('-timestamp')
     logentries_html = algorithms.convert_logentries_to_html(log_entries)
 
-    return render(request, "taskdatabase/tasks/task_details.html", {'task': task, 'logentries': logentries_html })
-
-
+    return render(request, "taskdatabase/tasks/task_details.html", {'task': task, 'logentries': logentries_html})
 
 
 def ShowInputs(request, id):
@@ -315,7 +312,7 @@ def ShowConfig(request):
 
 def ShowDashboard(request, selection):
     # gather the results
-    results_tasks,results_logs = algorithms.construct_dashboard_html(request, selection)
+    results_tasks, results_logs = algorithms.construct_dashboard_html(request, selection)
     return render(request, "dashboard/dashboard.html",
                   {'results_tasks': results_tasks,
                    'results_logs': results_logs,
@@ -336,6 +333,7 @@ class DiagramView(ListView):
     model = Task
     template_name = "taskdatabase/diagram.html"
 
+
 # ---------- REST API views -----------
 
 # example: /atdb/tasks/
@@ -346,8 +344,8 @@ class TaskListViewAPI(generics.ListCreateAPIView):
     A pagination list of tasks, unsorted.
     """
     model = Task
-    queryset = Task.objects.filter(task_type='regular').order_by('-priority','id')
-    #serializer_class = TaskSerializer
+    queryset = Task.objects.filter(task_type='regular').order_by('-priority', 'id')
+    # serializer_class = TaskSerializer
 
     # using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
     filter_backends = (filters.DjangoFilterBackend,)
@@ -359,13 +357,14 @@ class TaskListViewAPI(generics.ListCreateAPIView):
         else:
             return TaskWriteSerializer
 
+
 class PostProcessingTaskListViewAPI(generics.ListCreateAPIView):
     """
     A pagination list of tasks, unsorted.
     """
     model = Task
-    queryset = Task.objects.filter(task_type='postprocessing').order_by('-priority','id')
-    #serializer_class = TaskSerializer
+    queryset = Task.objects.filter(task_type='postprocessing').order_by('-priority', 'id')
+    # serializer_class = TaskSerializer
 
     # using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
     filter_backends = (filters.DjangoFilterBackend,)
@@ -377,14 +376,15 @@ class PostProcessingTaskListViewAPI(generics.ListCreateAPIView):
         else:
             return TaskWriteSerializer
 
+
 # all tasks
 class AllTaskListViewAPI(generics.ListCreateAPIView):
     """
     A pagination list of tasks, unsorted.
     """
     model = Task
-    queryset = Task.objects.all().order_by('-priority','id')
-    #serializer_class = TaskSerializer
+    queryset = Task.objects.all().order_by('-priority', 'id')
+    # serializer_class = TaskSerializer
 
     # using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
     filter_backends = (filters.DjangoFilterBackend,)
@@ -402,7 +402,7 @@ class TaskListViewAPIFast(generics.ListAPIView):
     A pagination list of tasks, unsorted.
     """
     model = Task
-    queryset = Task.objects.all().order_by('-priority','id')
+    queryset = Task.objects.all().order_by('-priority', 'id')
     serializer_class = TaskReadSerializerFast
 
     # using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
@@ -418,6 +418,7 @@ class TaskDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
     """
     model = Task
     queryset = Task.objects.all()
+
     # serializer_class = TaskSerializer
 
     def get_serializer_class(self):
@@ -445,6 +446,7 @@ class WorkflowListViewAPI(generics.ListCreateAPIView):
     filter_backends = (filters.DjangoFilterBackend,)
     filter_class = WorkflowFilter
 
+
 # example: /atdb/workflows/5/
 class WorkflowDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
     model = Workflow
@@ -547,7 +549,7 @@ class MonitorListViewAPI(generics.ListCreateAPIView):
     serializer_class = MonitorSerializer
 
     filter_backends = (filters.DjangoFilterBackend,)
-    #filter_class = PostProcessingFilter
+    # filter_class = PostProcessingFilter
 
 
 # example: /atdb/job/5/
@@ -555,26 +557,26 @@ class MonitorDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
     model = Monitor
     queryset = Monitor.objects.all()
     serializer_class = MonitorSerializer
-    
-    
+
+
 # --- controller resources, triggered by a button in the GUI or directoy with a URL ---
 # set task status to 'new_status' - called from the GUI
 
 @login_required
-def Hold(request,pk,hold_it,page=0):
+def Hold(request, pk, hold_it, page=0):
     model = Task
     task = Task.objects.get(pk=pk)
     task.resume = (hold_it == 'resume')
     task.save()
-    if page==0:
+    if page == 0:
         # redirect to details screen
         return redirect('/atdb/query')
     else:
         # redirect to tasks list
-        return redirect('/atdb/?page='+page)
+        return redirect('/atdb/?page=' + page)
 
 
-def HoldQuery(request,pk,hold_it,query_params):
+def HoldQuery(request, pk, hold_it, query_params):
     model = Task
     task = Task.objects.get(pk=pk)
     task.resume = (hold_it == 'resume')
@@ -585,21 +587,22 @@ def HoldQuery(request,pk,hold_it,query_params):
 
 
 @login_required
-def TaskSetStatus(request,pk,new_status,page=0):
+def TaskSetStatus(request, pk, new_status, page=0):
     model = Task
     task = Task.objects.get(pk=pk)
     task.new_status = new_status
     task.save()
 
-    if page==0:
+    if page == 0:
         # redirect to details screen
         return redirect('/atdb/task_details')
     else:
         # redirect to tasks list
-        return redirect('/atdb/?page='+page)
+        return redirect('/atdb/?page=' + page)
+
 
 # set a filter value in the session, used later by the 'get_searched_tasks' mechanism
-def TaskSetFilter(request,filter):
+def TaskSetFilter(request, filter):
     request.session['task_filter'] = filter
 
     # switch off the other filters
@@ -608,12 +611,14 @@ def TaskSetFilter(request,filter):
 
     return redirect('/atdb/?page=1')
 
+
 # set the defined list of ACTIVE_STATUSSES on the session, used later by the 'get_searched_tasks' mechanism
 def TaskSetActiveFilter(request):
     request.session['task_filter'] = settings.ACTIVE_STATUSSES
     request.session['task_onhold_filter'] = None
     return redirect('/atdb/?page=1')
 
+
 def TaskSetOnHoldFilter(request, onhold):
     request.session['task_onhold_filter'] = onhold
     return redirect('/atdb/?page=1')
@@ -625,21 +630,21 @@ def ChangePriority(request, pk, priority_change, page=0):
     task = Task.objects.get(pk=pk)
     priority = task.priority + int(priority_change)
 
-    if priority<0:
-        priority=0
+    if priority < 0:
+        priority = 0
 
     task.priority = priority
     task.save()
 
-    if page==0:
+    if page == 0:
         # redirect to details screen
         return redirect('/atdb/task_details')
     else:
         # redirect to tasks list
-        return redirect('/atdb/?page='+page)
+        return redirect('/atdb/?page=' + page)
 
 
-def SortTasks(request,sort):
+def SortTasks(request, sort):
     # store the sort field on the session
     request.session['sort'] = sort
     return redirect('/atdb')
@@ -652,8 +657,8 @@ def convert_query_params_to_url(query_params):
     # because the query_params come in as a QueryDict converted to a string
     # it needs some converting to a json string that can be loaded into a dict
 
-    s = query_params.replace('<QueryDict: ','')[:-1]
-    s = s.replace('[','')
+    s = query_params.replace('<QueryDict: ', '')[:-1]
+    s = s.replace('[', '')
     s = s.replace(']', '')
     s = s.replace('\'', '"')
 
@@ -670,18 +675,17 @@ def convert_query_params_to_url(query_params):
 
 
 @login_required
-def TaskSetStatusTables2(request,pk,new_status,query_params):
+def TaskSetStatusTables2(request, pk, new_status, query_params):
     model = Task
     task = Task.objects.get(pk=pk)
     task.new_status = new_status
     task.save()
 
     current_query_params = convert_query_params_to_url(query_params)
-    #current_query_params = "id=&id__gte=&id__lte=&workflow__id=&filter=%09test&filter__icontains=&priority=&priority__gte=&priority__lte=&status__icontains=&status__in=&project=&project__icontains=&sas_id=&sas_id__icontains=&resume=unknown"
+    # current_query_params = "id=&id__gte=&id__lte=&workflow__id=&filter=%09test&filter__icontains=&priority=&priority__gte=&priority__lte=&status__icontains=&status__in=&project=&project__icontains=&sas_id=&sas_id__icontains=&resume=unknown"
     return redirect('/atdb/query/?' + current_query_params)
 
 
-
 @login_required
 def TaskMultiStatus(request, new_status, query_params):
     # get the list of id's from the session
@@ -696,14 +700,14 @@ def TaskMultiStatus(request, new_status, query_params):
             task.save()
 
         current_query_params = request.session['current_query_params']
-        return redirect('/atdb/query?'+current_query_params)
+        return redirect('/atdb/query?' + current_query_params)
 
     # add the current query parameters to the session so that they survive
     # the request/response to the confirmation page (which has other query parameters)
     current_query_params = convert_query_params_to_url(query_params)
     request.session['current_query_params'] = current_query_params
 
-    return render(request, "query/confirm_multi_change.html",{'new_value': new_status, 'count' : count})
+    return render(request, "query/confirm_multi_change.html", {'new_value': new_status, 'count': count})
 
 
 @login_required
@@ -720,14 +724,14 @@ def TaskMultiHold(request, onhold, query_params):
             task.save()
 
         current_query_params = request.session['current_query_params']
-        return redirect('/atdb/query?'+current_query_params)
+        return redirect('/atdb/query?' + current_query_params)
 
     # add the current query parameters to the session so that they survive
     # the request/response to the confirmation page (which has other query parameters)
     current_query_params = convert_query_params_to_url(query_params)
     request.session['current_query_params'] = current_query_params
 
-    return render(request, "query/confirm_multi_change.html",{'new_value': onhold, 'count' : count})
+    return render(request, "query/confirm_multi_change.html", {'new_value': onhold, 'count': count})
 
 
 # /atdb/get_size?status__in=defined,staged
@@ -740,8 +744,8 @@ class GetSizeView(generics.ListAPIView):
         query_params = dict(self.request.query_params)
         try:
             status_in = query_params['status__in']
-            status_list=status_in[0].split(',')
-            if status_list==['']:
+            status_list = status_in[0].split(',')
+            if status_list == ['']:
                 status_list = settings.STATUSSES_WITH_DATA
         except:
             # if no 'status__in=' is given, then use the default list
@@ -780,4 +784,30 @@ class GetMinMaxTimeView(generics.ListAPIView):
         except Exception as error:
             return Response({
                 'error': str(error)
-            })
+            })
\ No newline at end of file
+
+from rest_framework.serializers import ListSerializer
+# /atdb/get_unique_values_for_key/{key}
+class GetUniqueValuesForKey(generics.ListAPIView):
+    queryset = Task.objects.all()
+    model = Task
+    filter_backends = (filters.DjangoFilterBackend,)
+    filter_class = TaskFilter
+    # override list and generate a custom response
+
+
+    def list(self, request: Request, *args, **kwargs):
+        try:
+
+            aggregation_key = kwargs['aggregation_key']
+            queryset = self.get_queryset()
+            queryset = self.filter_queryset(queryset)
+            return Response({'aggregation_key': aggregation_key,
+                             'result': algorithms.unique_values_for_aggregation_key(
+                                 queryset,
+                                 aggregation_key)
+                             })
+        except Exception as error:
+            return Response({
+                'error': str(error)
+            })