Skip to content
Snippets Groups Projects
Commit 4eb2872c authored by Nico Vermaas's avatar Nico Vermaas
Browse files

dashboard & query screens

parent 87a62634
No related branches found
No related tags found
3 merge requests!74Acceptance,!73Master,!65Dev nico
"""atdb URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin from django.contrib import admin
from django.urls import include, path from django.urls import include, path
......
...@@ -104,55 +104,14 @@ def convert_config_to_html(querylist): ...@@ -104,55 +104,14 @@ def convert_config_to_html(querylist):
return results return results
# aggregate information from the tasks table per workflow per status
def aggregate_resources_logs():
records = []
# get all active tasks
active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
active_tasks_count = active_tasks.count()
# retrieve all unique workflows
active_workflows = active_tasks.values('workflow').distinct()
# iterate through the filters and accumulate logentries
for w in active_workflows:
workflow_result = {}
# extract the workflow object (cheap)
workflow = Workflow.objects.get(id = w['workflow'])
# aggregate logentries per step for all active statusses
for status in settings.ACTIVE_STATUSSES:
record = {}
record['name'] = str(workflow.id) +' - '+ workflow.workflow_uri
record['status'] = status
# aggregate logentries per step for all active statusses (expensive)
logs = LogEntry.objects.filter(status=status)\
.filter(task__status__in=settings.ACTIVE_STATUSSES)\
.filter(task__workflow=workflow)
sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
records.append(record)
return records
def aggregate_resources_tasks(): def aggregate_resources_tasks():
workflow_results = [] workflow_results = []
records = []
# get all active tasks # get all active tasks
active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES) active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
active_tasks_count = active_tasks.count() # active_tasks_count = active_tasks.count()
# retrieve all unique workflows # retrieve all unique workflows
active_workflows = active_tasks.values('workflow').distinct() active_workflows = active_tasks.values('workflow').distinct()
...@@ -170,6 +129,12 @@ def aggregate_resources_tasks(): ...@@ -170,6 +129,12 @@ def aggregate_resources_tasks():
tasks_per_workflow = Task.objects.filter(workflow=workflow) tasks_per_workflow = Task.objects.filter(workflow=workflow)
nr_of_tasks_per_workflow = tasks_per_workflow.count() nr_of_tasks_per_workflow = tasks_per_workflow.count()
sum_size_to_process = tasks_per_workflow.aggregate(Sum('size_to_process'))
workflow_result['size_to_process'] = sum_size_to_process['size_to_process__sum']
sum_size_processed = tasks_per_workflow.aggregate(Sum('size_processed'))
workflow_result['size_processed'] = sum_size_processed['size_processed__sum']
# all the active tasks # all the active tasks
active_tasks_per_workflow = tasks_per_workflow.filter(status__in=settings.ACTIVE_STATUSSES) active_tasks_per_workflow = tasks_per_workflow.filter(status__in=settings.ACTIVE_STATUSSES)
nr_of_active_tasks_per_workflow = active_tasks_per_workflow.count() nr_of_active_tasks_per_workflow = active_tasks_per_workflow.count()
...@@ -198,6 +163,47 @@ def aggregate_resources_tasks(): ...@@ -198,6 +163,47 @@ def aggregate_resources_tasks():
return workflow_results return workflow_results
# aggregate information from the logentries table per workflow per status
def aggregate_resources_logs():
records = []
# get all active tasks
active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
active_tasks_count = active_tasks.count()
# retrieve all unique workflows
active_workflows = active_tasks.values('workflow').distinct()
# iterate through the filters and accumulate logentries
for w in active_workflows:
workflow_result = {}
# extract the workflow object (cheap)
workflow = Workflow.objects.get(id = w['workflow'])
# aggregate logentries per step for all active statusses
for status in settings.ACTIVE_STATUSSES:
record = {}
record['name'] = str(workflow.id) +' - '+ workflow.workflow_uri
record['status'] = status
# aggregate logentries per step for all active statusses (expensive)
logs = LogEntry.objects.filter(status=status)\
.filter(task__status__in=settings.ACTIVE_STATUSSES)\
.filter(task__workflow=workflow)
sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
records.append(record)
return records
def construct_link(request, status, workflow_id, count): def construct_link(request, status, workflow_id, count):
link = str(count) link = str(count)
try: try:
...@@ -205,26 +211,43 @@ def construct_link(request, status, workflow_id, count): ...@@ -205,26 +211,43 @@ def construct_link(request, status, workflow_id, count):
if settings.DEV == True: if settings.DEV == True:
url = request.build_absolute_uri('/atdb/tasks') + query url = request.build_absolute_uri('/atdb/tasks') + query
else: else:
# Unclear why 'build_absolute_uri' doesn't return 'https' in production.
# Probably because the https is handled fully outside the container by Traefik
# and ATDB is not aware of that.
url = "https://" + request.get_host() + '/atdb/tasks' + query url = "https://" + request.get_host() + '/atdb/tasks' + query
link = '<a href="' + url + '" target="_blank">' + str(count) + "</a>" link = '<a href="' + url + '" target="_blank">' + str(count) + "</a>"
except: except:
pass pass
return link return link
def convert_aggregation_to_html(request):
def human_readable(size_in_bytes):
for count in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
if size_in_bytes > -1024.0 and size_in_bytes < 1024.0:
return "%3.1f %s" % (size_in_bytes, count)
size_in_bytes /= 1024.0
return "%3.1f %s" % (size_in_bytes, 'PB')
def construct_tasks_per_workflow_html(request, workflow_results):
# --- Progress of tasks per active workflow --- # --- Progress of tasks per active workflow ---
workflow_results = aggregate_resources_tasks() workflow_results = aggregate_resources_tasks()
results_tasks = "<p>Progress of tasks per (active) workflow</p>" results_tasks = "<p>Progress of tasks per (active) workflow</p>"
header = "<th>Workflow</th>" header = "<th>Workflow</th>"
for status in settings.ALL_STATUSSES: for status in settings.ALL_STATUSSES:
header += "<th>" + status + "</th>" header += "<th>" + status + "</th>"
results_tasks += header + '<th class="active">active</th><th>total</th>' results_tasks += header + '<th class="active">active</th><th>total</th><th>to process</th><th>processed</th>'
for workflow_result in workflow_results: for workflow_result in workflow_results:
d = workflow_result['nr_of_tasks_per_status'] d = workflow_result['nr_of_tasks_per_status']
values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>" #values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>"
values = "<td><b>" + str(workflow_result['id']) + "</b></td>"
for key in d: for key in d:
percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100) percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
...@@ -240,13 +263,21 @@ def convert_aggregation_to_html(request): ...@@ -240,13 +263,21 @@ def convert_aggregation_to_html(request):
#values += "<td class="+style+">" + str(percentage) + "% ("+str(d[key])+")</td>" #values += "<td class="+style+">" + str(percentage) + "% ("+str(d[key])+")</td>"
#values += "<td>" + str(d[key]) + "</td>" #values += "<td>" + str(d[key]) + "</td>"
# add sizes
values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
try:
percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
except:
percentage = 0
values += "<td>" + str(human_readable(workflow_result['size_processed'])) + " ("+ str(percentage) + "%) </td>"
results_tasks += "<tr>" + values + "</tr>" results_tasks += "<tr>" + values + "</tr>"
results_tasks = "<tbody>" + results_tasks + "</tbody>" results_tasks = "<tbody>" + results_tasks + "</tbody>"
return results_tasks
# --- logentries --- def construct_logs_per_workflow_html(log_records):
log_records = aggregate_resources_logs()
results_logs = "" results_logs = ""
for record in log_records: for record in log_records:
...@@ -261,5 +292,18 @@ def convert_aggregation_to_html(request): ...@@ -261,5 +292,18 @@ def convert_aggregation_to_html(request):
"</td><td>" + str(record['wall_clock_time']) + "</td><tr>" "</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
results_logs += line results_logs += line
return results_logs
def construct_dashboard_html(request):
# --- Progress of tasks per active workflow ---
workflow_results = aggregate_resources_tasks()
results_tasks = construct_tasks_per_workflow_html(request, workflow_results)
# --- logentries ---
log_records = aggregate_resources_logs()
results_logs = construct_logs_per_workflow_html(log_records)
return results_tasks,results_logs return results_tasks,results_logs
...@@ -21,11 +21,11 @@ class TaskTable(tables.Table): ...@@ -21,11 +21,11 @@ class TaskTable(tables.Table):
class Meta: class Meta:
model = Task model = Task
template_name = "django_tables2/bootstrap4.html" template_name = "django_tables2/bootstrap4.html"
fields = ("id", "workflow","priority", "status","filter","project","sas_id","creationtime", "size_to_process","buttons") fields = ("id", "workflow","priority", "status","project","sas_id","buttons")
# columns that need specific rendering # columns that need specific rendering
status = StatusColumn() status = StatusColumn()
creationtime = tables.Column(verbose_name='CreationTime') #creationtime = tables.Column(verbose_name='CreationTime')
buttons = tables.TemplateColumn(verbose_name='Set Status', buttons = tables.TemplateColumn(verbose_name='Set Status',
template_name='query/status_buttons_per_row.html', template_name='query/status_buttons_per_row.html',
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
<p>Click to change Status for all these tasks</p> <p>Click to change Status for all these tasks</p>
<table> <table>
<tr> <tr>
<a href="{% url 'task-setstatus' 1 'defining' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> defining</a>&nbsp;
<a href="{% url 'task-setstatus' 1 'staged' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> staged</a>&nbsp; <a href="{% url 'task-setstatus' 1 'staged' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> staged</a>&nbsp;
<a href="{% url 'task-setstatus' 1 'processed' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> processed</a>&nbsp; <a href="{% url 'task-setstatus' 1 'processed' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> processed</a>&nbsp;
<a href="{% url 'task-setstatus' 1 'validated' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> validated</a>&nbsp; <a href="{% url 'task-setstatus' 1 'validated' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> validated</a>&nbsp;
......
...@@ -64,8 +64,6 @@ ...@@ -64,8 +64,6 @@
{% endif %} {% endif %}
</div> </div>
</nav> </nav>
{% for message in messages %} {% for message in messages %}
......
...@@ -80,7 +80,7 @@ ...@@ -80,7 +80,7 @@
{% include 'taskdatabase/pagination.html' %} {% include 'taskdatabase/pagination.html' %}
</div> </div>
</div> </div>
<p class="footer"> Version 1.0.0 (12 mar 2021 - 21:00) <p class="footer"> Version 1.0.0 (15 mar 2021 - 13:00)
</div> </div>
......
...@@ -67,8 +67,6 @@ class TaskFilterQueryPage(filters.FilterSet): ...@@ -67,8 +67,6 @@ class TaskFilterQueryPage(filters.FilterSet):
'status': ['icontains', 'in'], 'status': ['icontains', 'in'],
'project': ['exact', 'icontains'], 'project': ['exact', 'icontains'],
'sas_id': ['exact', 'icontains'], 'sas_id': ['exact', 'icontains'],
'creationTime': ['icontains'],
'size_to_process' : ['lte', 'gte'],
} }
...@@ -137,6 +135,7 @@ class IndexView(ListView): ...@@ -137,6 +135,7 @@ class IndexView(ListView):
tasks = Task.objects.order_by(sort) tasks = Task.objects.order_by(sort)
# check if there is a 'task_filter' put on the session
try: try:
filter = self.request.session['task_filter'] filter = self.request.session['task_filter']
if filter!='all': if filter!='all':
...@@ -253,7 +252,7 @@ def ShowConfig(request): ...@@ -253,7 +252,7 @@ def ShowConfig(request):
def ShowDashboard(request): def ShowDashboard(request):
# gather the results # gather the results
results_tasks,results_logs = algorithms.convert_aggregation_to_html(request) results_tasks,results_logs = algorithms.construct_dashboard_html(request)
return render(request, "dashboard/dashboard.html", {'results_tasks': results_tasks, 'results_logs': results_logs}) return render(request, "dashboard/dashboard.html", {'results_tasks': results_tasks, 'results_logs': results_logs})
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment