diff --git a/atdb/requirements/base.txt b/atdb/requirements/base.txt
index c52d7065c7e389f671bdbd370b3da06d3a492a40..ac71b471c94cb22ea3a47e7a4ae72571378190e4 100644
--- a/atdb/requirements/base.txt
+++ b/atdb/requirements/base.txt
@@ -1,7 +1,7 @@
Django==3.1.4
djangorestframework==3.12.2
django-filter==2.3.0
-psycopg2==2.8.6
+psycopg2-binary==2.9.3
django-cors-headers==3.6.0
django-extensions==3.1.0
django-bootstrap-pagination==1.7.0
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index d1f5dc250c95db46bef0c6424055c41f68f3868c..91b45e6fe0cd3d532065bdc091acc9d49c6dbc18 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -554,3 +554,8 @@ def construct_dashboard_html(request, selection):
results_logs = construct_logs_per_workflow_html(request, log_records)
return results_tasks, results_logs
+
+
+def unique_values_for_aggregation_key(queryset, aggregation_key):
+
+ return list(map(lambda x: x[aggregation_key], queryset.values(aggregation_key).distinct()))
\ No newline at end of file
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index df6c25f048cace4621a808efa6e70adec0b989d5..e669d7290bd8d656d27c5489250ea1f09df315b4 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -61,6 +61,9 @@ urlpatterns = [
path('tasks/get_size/', views.GetSizeView.as_view(), name='get-size-view'),
# /atdb/get_min_start_and_max_end_time?sas_id=65005
path('get_min_start_and_max_end_time/', views.GetMinMaxTimeView.as_view(), name='get-min-start-and-max-end-time-view'),
+ # /atdb/get_unique_values_for_key/<aggregation_key>
+ path('get_unique_values_for_key/<str:aggregation_key>', views.GetUniqueValuesForKey.as_view(),
+ name='get-unique-values-for-key-view'),
# --- controller resources ---
path('tasks/<int:pk>/setstatus/<new_status>/<page>', views.TaskSetStatus, name='task-setstatus-view'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index a3542fd6662d4a10495f919d8e2f5fa99c711be1..933e2c7e324aa4d910cc9be2b22f1290e5ca1fd1 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -1,4 +1,3 @@
-
import logging
import json
@@ -20,6 +19,7 @@ from django_tables2 import SingleTableView
from django.shortcuts import render, redirect
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+from rest_framework.request import Request
from django.conf import settings
from .models import Task, Status, Workflow, LogEntry, Configuration, Job, PostProcessingRule, Monitor
@@ -30,9 +30,9 @@ from .serializers import \
TaskWriteSerializer, \
TaskReadSerializer, \
TaskReadSerializerFast, \
- WorkflowSerializer,\
- LogEntrySerializer,\
- ConfigurationSerializer,\
+ WorkflowSerializer, \
+ LogEntrySerializer, \
+ ConfigurationSerializer, \
JobSerializer, \
PostProcessingRuleSerializer, \
MonitorSerializer
@@ -41,9 +41,9 @@ from .services import algorithms
logger = logging.getLogger(__name__)
+
# ---------- filters (in the REST API) ---------
class TaskFilter(filters.FilterSet):
-
class Meta:
model = Task
@@ -51,18 +51,19 @@ class TaskFilter(filters.FilterSet):
'task_type': ['exact', 'icontains', 'in'],
'creationTime': ['icontains'],
'filter': ['exact', 'icontains'],
- 'workflow__id' : ['exact', 'icontains'],
+ 'workflow__id': ['exact', 'icontains'],
'project': ['exact', 'icontains'],
- 'sas_id': ['exact', 'icontains','in'],
+ 'sas_id': ['exact', 'icontains', 'in'],
'status': ['exact', 'icontains', 'in', 'startswith'],
'purge_policy': ['exact'],
- 'priority': ['exact','lte','gte'],
+ 'priority': ['exact', 'lte', 'gte'],
'resume': ['exact'],
# http://localhost:8000/atdb/tasks/?predecessor__isnull=True
'predecessor': ['isnull'],
'predecessor__status': ['exact', 'icontains', 'in', 'startswith'],
}
+
class TaskFilterQueryPage(filters.FilterSet):
resume = django_filters.BooleanFilter(lookup_expr='exact', label='resuming')
@@ -71,19 +72,18 @@ class TaskFilterQueryPage(filters.FilterSet):
fields = {
'id': ['exact', 'gte', 'lte'],
- #'task_type': ['exact','in'],
+ # 'task_type': ['exact','in'],
'workflow__id': ['exact'],
'filter': ['exact', 'icontains'],
'priority': ['exact', 'gte', 'lte'],
'status': ['icontains', 'in'],
- 'project': ['exact', 'icontains','in'],
+ 'project': ['exact', 'icontains', 'in'],
'sas_id': ['exact', 'icontains', 'in'],
- #'resume': ['exact'],
+ # 'resume': ['exact'],
}
class WorkflowFilter(filters.FilterSet):
-
class Meta:
model = Workflow
@@ -95,18 +95,17 @@ class WorkflowFilter(filters.FilterSet):
class LogEntryFilter(filters.FilterSet):
-
class Meta:
model = LogEntry
fields = {
'task__id': ['exact'],
'step_name': ['exact', 'icontains', 'in', 'startswith'],
- 'status': ['exact','in'],
+ 'status': ['exact', 'in'],
}
-class ConfigurationFilter(filters.FilterSet):
+class ConfigurationFilter(filters.FilterSet):
class Meta:
model = Configuration
@@ -116,8 +115,8 @@ class ConfigurationFilter(filters.FilterSet):
'value': ['exact', 'icontains'],
}
-class JobFilter(filters.FilterSet):
+class JobFilter(filters.FilterSet):
class Meta:
model = Job
@@ -127,8 +126,8 @@ class JobFilter(filters.FilterSet):
'job_id': ['exact'],
}
-class PostProcessingFilter(filters.FilterSet):
+class PostProcessingFilter(filters.FilterSet):
class Meta:
model = PostProcessingRule
@@ -139,6 +138,7 @@ class PostProcessingFilter(filters.FilterSet):
'workflow_to_apply__id': ['exact'],
}
+
# ---------- Tables2 Views (experimental) -----------
# implementation with tables2: http://localhost:8000/atdb/tables2
class QueryView(SingleTableMixin, FilterView):
@@ -192,7 +192,7 @@ class IndexView(ListView):
# check if there is a 'task_filter' put on the session
try:
filter = self.request.session['task_filter']
- if filter!='all':
+ if filter != 'all':
tasks = get_searched_tasks(filter, sort)
except:
pass
@@ -200,7 +200,7 @@ class IndexView(ListView):
# check if there is a 'task_onhold_filter' put on the session
try:
onhold = self.request.session['task_onhold_filter']
- if onhold!=None:
+ if onhold != None:
tasks = tasks.filter(resume=not onhold)
except:
@@ -254,7 +254,6 @@ class TaskTables2View(SingleTableView):
def TaskDetails(request, id=0, page=0):
-
try:
task = Task.objects.get(id=id)
@@ -277,9 +276,7 @@ def TaskDetails(request, id=0, page=0):
log_entries = LogEntry.objects.filter(task=task).order_by('-timestamp')
logentries_html = algorithms.convert_logentries_to_html(log_entries)
- return render(request, "taskdatabase/tasks/task_details.html", {'task': task, 'logentries': logentries_html })
-
-
+ return render(request, "taskdatabase/tasks/task_details.html", {'task': task, 'logentries': logentries_html})
def ShowInputs(request, id):
@@ -315,7 +312,7 @@ def ShowConfig(request):
def ShowDashboard(request, selection):
# gather the results
- results_tasks,results_logs = algorithms.construct_dashboard_html(request, selection)
+ results_tasks, results_logs = algorithms.construct_dashboard_html(request, selection)
return render(request, "dashboard/dashboard.html",
{'results_tasks': results_tasks,
'results_logs': results_logs,
@@ -336,6 +333,7 @@ class DiagramView(ListView):
model = Task
template_name = "taskdatabase/diagram.html"
+
# ---------- REST API views -----------
# example: /atdb/tasks/
@@ -346,8 +344,8 @@ class TaskListViewAPI(generics.ListCreateAPIView):
A pagination list of tasks, unsorted.
"""
model = Task
- queryset = Task.objects.filter(task_type='regular').order_by('-priority','id')
- #serializer_class = TaskSerializer
+ queryset = Task.objects.filter(task_type='regular').order_by('-priority', 'id')
+ # serializer_class = TaskSerializer
# using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
filter_backends = (filters.DjangoFilterBackend,)
@@ -359,13 +357,14 @@ class TaskListViewAPI(generics.ListCreateAPIView):
else:
return TaskWriteSerializer
+
class PostProcessingTaskListViewAPI(generics.ListCreateAPIView):
"""
A pagination list of tasks, unsorted.
"""
model = Task
- queryset = Task.objects.filter(task_type='postprocessing').order_by('-priority','id')
- #serializer_class = TaskSerializer
+ queryset = Task.objects.filter(task_type='postprocessing').order_by('-priority', 'id')
+ # serializer_class = TaskSerializer
# using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
filter_backends = (filters.DjangoFilterBackend,)
@@ -377,14 +376,15 @@ class PostProcessingTaskListViewAPI(generics.ListCreateAPIView):
else:
return TaskWriteSerializer
+
# all tasks
class AllTaskListViewAPI(generics.ListCreateAPIView):
"""
A pagination list of tasks, unsorted.
"""
model = Task
- queryset = Task.objects.all().order_by('-priority','id')
- #serializer_class = TaskSerializer
+ queryset = Task.objects.all().order_by('-priority', 'id')
+ # serializer_class = TaskSerializer
# using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
filter_backends = (filters.DjangoFilterBackend,)
@@ -402,7 +402,7 @@ class TaskListViewAPIFast(generics.ListAPIView):
A pagination list of tasks, unsorted.
"""
model = Task
- queryset = Task.objects.all().order_by('-priority','id')
+ queryset = Task.objects.all().order_by('-priority', 'id')
serializer_class = TaskReadSerializerFast
# using the Django Filter Backend - https://django-filter.readthedocs.io/en/latest/index.html
@@ -418,6 +418,7 @@ class TaskDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
"""
model = Task
queryset = Task.objects.all()
+
# serializer_class = TaskSerializer
def get_serializer_class(self):
@@ -445,6 +446,7 @@ class WorkflowListViewAPI(generics.ListCreateAPIView):
filter_backends = (filters.DjangoFilterBackend,)
filter_class = WorkflowFilter
+
# example: /atdb/workflows/5/
class WorkflowDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
model = Workflow
@@ -547,7 +549,7 @@ class MonitorListViewAPI(generics.ListCreateAPIView):
serializer_class = MonitorSerializer
filter_backends = (filters.DjangoFilterBackend,)
- #filter_class = PostProcessingFilter
+ # filter_class = PostProcessingFilter
# example: /atdb/job/5/
@@ -555,26 +557,26 @@ class MonitorDetailsViewAPI(generics.RetrieveUpdateDestroyAPIView):
model = Monitor
queryset = Monitor.objects.all()
serializer_class = MonitorSerializer
-
-
+
+
# --- controller resources, triggered by a button in the GUI or directoy with a URL ---
# set task status to 'new_status' - called from the GUI
@login_required
-def Hold(request,pk,hold_it,page=0):
+def Hold(request, pk, hold_it, page=0):
model = Task
task = Task.objects.get(pk=pk)
task.resume = (hold_it == 'resume')
task.save()
- if page==0:
+ if page == 0:
# redirect to details screen
return redirect('/atdb/query')
else:
# redirect to tasks list
- return redirect('/atdb/?page='+page)
+ return redirect('/atdb/?page=' + page)
-def HoldQuery(request,pk,hold_it,query_params):
+def HoldQuery(request, pk, hold_it, query_params):
model = Task
task = Task.objects.get(pk=pk)
task.resume = (hold_it == 'resume')
@@ -585,21 +587,22 @@ def HoldQuery(request,pk,hold_it,query_params):
@login_required
-def TaskSetStatus(request,pk,new_status,page=0):
+def TaskSetStatus(request, pk, new_status, page=0):
model = Task
task = Task.objects.get(pk=pk)
task.new_status = new_status
task.save()
- if page==0:
+ if page == 0:
# redirect to details screen
return redirect('/atdb/task_details')
else:
# redirect to tasks list
- return redirect('/atdb/?page='+page)
+ return redirect('/atdb/?page=' + page)
+
# set a filter value in the session, used later by the 'get_searched_tasks' mechanism
-def TaskSetFilter(request,filter):
+def TaskSetFilter(request, filter):
request.session['task_filter'] = filter
# switch off the other filters
@@ -608,12 +611,14 @@ def TaskSetFilter(request,filter):
return redirect('/atdb/?page=1')
+
# set the defined list of ACTIVE_STATUSSES on the session, used later by the 'get_searched_tasks' mechanism
def TaskSetActiveFilter(request):
request.session['task_filter'] = settings.ACTIVE_STATUSSES
request.session['task_onhold_filter'] = None
return redirect('/atdb/?page=1')
+
def TaskSetOnHoldFilter(request, onhold):
request.session['task_onhold_filter'] = onhold
return redirect('/atdb/?page=1')
@@ -625,21 +630,21 @@ def ChangePriority(request, pk, priority_change, page=0):
task = Task.objects.get(pk=pk)
priority = task.priority + int(priority_change)
- if priority<0:
- priority=0
+ if priority < 0:
+ priority = 0
task.priority = priority
task.save()
- if page==0:
+ if page == 0:
# redirect to details screen
return redirect('/atdb/task_details')
else:
# redirect to tasks list
- return redirect('/atdb/?page='+page)
+ return redirect('/atdb/?page=' + page)
-def SortTasks(request,sort):
+def SortTasks(request, sort):
# store the sort field on the session
request.session['sort'] = sort
return redirect('/atdb')
@@ -652,8 +657,8 @@ def convert_query_params_to_url(query_params):
# because the query_params come in as a QueryDict converted to a string
# it needs some converting to a json string that can be loaded into a dict
- s = query_params.replace('<QueryDict: ','')[:-1]
- s = s.replace('[','')
+ s = query_params.replace('<QueryDict: ', '')[:-1]
+ s = s.replace('[', '')
s = s.replace(']', '')
s = s.replace('\'', '"')
@@ -670,18 +675,17 @@ def convert_query_params_to_url(query_params):
@login_required
-def TaskSetStatusTables2(request,pk,new_status,query_params):
+def TaskSetStatusTables2(request, pk, new_status, query_params):
model = Task
task = Task.objects.get(pk=pk)
task.new_status = new_status
task.save()
current_query_params = convert_query_params_to_url(query_params)
- #current_query_params = "id=&id__gte=&id__lte=&workflow__id=&filter=%09test&filter__icontains=&priority=&priority__gte=&priority__lte=&status__icontains=&status__in=&project=&project__icontains=&sas_id=&sas_id__icontains=&resume=unknown"
+ # current_query_params = "id=&id__gte=&id__lte=&workflow__id=&filter=%09test&filter__icontains=&priority=&priority__gte=&priority__lte=&status__icontains=&status__in=&project=&project__icontains=&sas_id=&sas_id__icontains=&resume=unknown"
return redirect('/atdb/query/?' + current_query_params)
-
@login_required
def TaskMultiStatus(request, new_status, query_params):
# get the list of id's from the session
@@ -696,14 +700,14 @@ def TaskMultiStatus(request, new_status, query_params):
task.save()
current_query_params = request.session['current_query_params']
- return redirect('/atdb/query?'+current_query_params)
+ return redirect('/atdb/query?' + current_query_params)
# add the current query parameters to the session so that they survive
# the request/response to the confirmation page (which has other query parameters)
current_query_params = convert_query_params_to_url(query_params)
request.session['current_query_params'] = current_query_params
- return render(request, "query/confirm_multi_change.html",{'new_value': new_status, 'count' : count})
+ return render(request, "query/confirm_multi_change.html", {'new_value': new_status, 'count': count})
@login_required
@@ -720,14 +724,14 @@ def TaskMultiHold(request, onhold, query_params):
task.save()
current_query_params = request.session['current_query_params']
- return redirect('/atdb/query?'+current_query_params)
+ return redirect('/atdb/query?' + current_query_params)
# add the current query parameters to the session so that they survive
# the request/response to the confirmation page (which has other query parameters)
current_query_params = convert_query_params_to_url(query_params)
request.session['current_query_params'] = current_query_params
- return render(request, "query/confirm_multi_change.html",{'new_value': onhold, 'count' : count})
+ return render(request, "query/confirm_multi_change.html", {'new_value': onhold, 'count': count})
# /atdb/get_size?status__in=defined,staged
@@ -740,8 +744,8 @@ class GetSizeView(generics.ListAPIView):
query_params = dict(self.request.query_params)
try:
status_in = query_params['status__in']
- status_list=status_in[0].split(',')
- if status_list==['']:
+ status_list = status_in[0].split(',')
+ if status_list == ['']:
status_list = settings.STATUSSES_WITH_DATA
except:
# if no 'status__in=' is given, then use the default list
@@ -780,4 +784,30 @@ class GetMinMaxTimeView(generics.ListAPIView):
except Exception as error:
return Response({
'error': str(error)
- })
\ No newline at end of file
+ })
+
+from rest_framework.serializers import ListSerializer
+# /atdb/get_unique_values_for_key/{key}
+class GetUniqueValuesForKey(generics.ListAPIView):
+ queryset = Task.objects.all()
+ model = Task
+ filter_backends = (filters.DjangoFilterBackend,)
+ filter_class = TaskFilter
+ # override list and generate a custom response
+
+
+ def list(self, request: Request, *args, **kwargs):
+ try:
+
+ aggregation_key = kwargs['aggregation_key']
+ queryset = self.get_queryset()
+ queryset = self.filter_queryset(queryset)
+ return Response({'aggregation_key': aggregation_key,
+ 'result': algorithms.unique_values_for_aggregation_key(
+ queryset,
+ aggregation_key)
+ })
+ except Exception as error:
+ return Response({
+ 'error': str(error)
+ })