From 5e87ffab5455b0cfc0e0047bb54d2e8cea9c1b6b Mon Sep 17 00:00:00 2001
From: Nico Vermaas <vermaas@astron.nl>
Date: Tue, 16 Mar 2021 16:53:03 +0100
Subject: [PATCH] dashboard now shows all workflows (not just active tasks)
---
atdb/taskdatabase/services/algorithms.py | 158 +++++++++++++-----
.../templates/taskdatabase/index.html | 2 +-
.../tasks/set_status_buttons.html | 1 +
3 files changed, 115 insertions(+), 46 deletions(-)
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 58fb0c82..c822d3ca 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -1,7 +1,6 @@
"""
File name: algorithms.py
Author: Nico Vermaas - Astron
- Date created: 2019-04-04
Description: Business logic for ATDB. These functions are called from the views (views.py).
"""
@@ -106,22 +105,25 @@ def convert_config_to_html(querylist):
# aggregate information from the tasks table per workflow per status
def aggregate_resources_tasks():
+ # NOTE: uncomment (or refactor) the lines with ### to only aggregate the data for active tasks
workflow_results = []
# get all active tasks
- active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
- # active_tasks_count = active_tasks.count()
+ ### active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
- # retrieve all unique workflows
- active_workflows = active_tasks.values('workflow').distinct()
+ # retrieve all unique workflows from the active tasks
+ ### active_workflows = active_tasks.values('workflow').distinct()
+
+ all_workflows = Workflow.objects.all()
# iterate through the filters and accumulate logentries
- for w in active_workflows:
+ ### for w in all_workflows:
+ for workflow in all_workflows:
workflow_result = {}
# extract the workflow object (cheap)
- workflow = Workflow.objects.get(id = w['workflow'])
+ ### workflow = Workflow.objects.get(id = w['workflow'])
# get the numbers for this workflow
@@ -143,9 +145,7 @@ def aggregate_resources_tasks():
nr_per_status = {}
for status in settings.ALL_STATUSSES:
- record = {}
nr_for_this_status = Task.objects.filter(workflow=workflow, status=status).count()
- record[status] = nr_for_this_status
nr_per_status[status] = nr_for_this_status
nr_per_status['active'] = nr_of_active_tasks_per_workflow
@@ -165,6 +165,54 @@ def aggregate_resources_tasks():
# aggregate information from the logentries table per workflow per status
def aggregate_resources_logs():
+ # NOTE: uncomment (or refactor) the lines with ### to only aggregate the data for active tasks
+
+ workflow_results = []
+
+ # get all active tasks
+ ### active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
+
+ # retrieve all unique workflows
+ ### active_workflows = active_tasks.values('workflow').distinct()
+
+ all_workflows = Workflow.objects.all()
+ # iterate through the filters and accumulate logentries
+
+ ### for w in active_workflows:
+ for workflow in all_workflows:
+ workflow_result = {}
+
+ # extract the workflow object (cheap)
+ ### workflow = Workflow.objects.get(id = w['workflow'])
+
+ # aggregate logentries per step for all active statusses
+ record_per_status = {}
+ for status in settings.ALL_STATUSSES:
+ record = {}
+
+ # aggregate logentries per step for all active statusses (expensive)
+ logs = LogEntry.objects.filter(status=status)\
+ .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+ .filter(task__workflow=workflow)
+
+ sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
+ record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
+
+ wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
+ record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
+ record_per_status[status] = record
+
+ workflow_result['id'] = workflow.id
+ workflow_result['name'] = workflow.workflow_uri
+ workflow_result['records_per_status'] = record_per_status
+
+ workflow_results.append(workflow_result)
+
+ return workflow_results
+
+
+# aggregate information from the logentries table per workflow per status
+def aggregate_resources_logs_version1():
records = []
@@ -243,19 +291,20 @@ def construct_link_to_workflow_api(request, workflow_result):
def human_readable(size_in_bytes):
- for count in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
- if size_in_bytes > -1024.0 and size_in_bytes < 1024.0:
- return "%3.1f %s" % (size_in_bytes, count)
- size_in_bytes /= 1024.0
- return "%3.1f %s" % (size_in_bytes, 'PB')
+ try:
+ for count in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
+ if size_in_bytes > -1024.0 and size_in_bytes < 1024.0:
+ return "%3.1f %s" % (size_in_bytes, count)
+ size_in_bytes /= 1024.0
+ return "%3.1f %s" % (size_in_bytes, 'PB')
+ except:
+ return "0"
def construct_tasks_per_workflow_html(request, workflow_results):
# --- Progress of tasks per active workflow ---
- workflow_results = aggregate_resources_tasks()
-
- results_tasks = "<p>Progress of tasks per (active) workflow</p>"
+ results_tasks = "<p>Progress of tasks per workflow</p>"
# construct the header
header = "<th>Workflow</th>"
@@ -267,22 +316,23 @@ def construct_tasks_per_workflow_html(request, workflow_results):
for workflow_result in workflow_results:
d = workflow_result['nr_of_tasks_per_status']
- #values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>"
- # values = "<td><b>" + str(workflow_result['id']) + "</b></td>"
link = construct_link_to_workflow_api(request, workflow_result)
values = "<td><b>" + link + "</b></td>"
for key in d:
- percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
+ try:
+ percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
+ except:
+ percentage = 0
- # distinguish active statusses
- style = ""
- if key in settings.ACTIVE_STATUSSES or key=='active':
+ # distinguish active statusses
+ style = ""
+ if key in settings.ACTIVE_STATUSSES or key=='active':
style = "active"
- # bonus: add a query link
- link = construct_link_to_tasks_api(request, key, workflow_result['id'], d[key])
- values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"
+ # bonus: add a query link
+ link = construct_link_to_tasks_api(request, key, workflow_result['id'], d[key])
+ values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"
# add sizes
values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
@@ -298,7 +348,7 @@ def construct_tasks_per_workflow_html(request, workflow_results):
return results_tasks
-def construct_logs_per_workflow_html(log_records):
+def construct_logs_per_workflow_html_version1(log_records):
results_logs = "<p>Resources used per step per active workflow</p>"
# construct the header
@@ -312,7 +362,7 @@ def construct_logs_per_workflow_html(log_records):
style = "active"
line = "<tr><td><b>" + record['name'] + "</b></td>" \
- '<td class="' + style + '" >' + record['status'] + \
+ '<td class="' + style + '" >' + record['status'] + \
"</td><td>" + str(record['cpu_cycles']) + \
"</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
@@ -322,32 +372,47 @@ def construct_logs_per_workflow_html(log_records):
return results_logs
-def construct_cpu_cycles_per_workflow_html(log_records):
- results_logs = "<p>Resources used per step per active workflow</p>"
+def construct_logs_per_workflow_html(request, workflow_results):
+ results = "<p>Resources used per step per workflow: <b>cpu_cycles/wall_clock_time</b></p>"
# construct the header
header = "<th>Workflow</th>"
for status in settings.ALL_STATUSSES:
header += "<th>" + status + "</th>"
+ results += header
- results_logs += header + '<th>CPU cycles</th><th>wall clock time</th><th>to process</th><th>processed</th>'
+ for workflow_result in workflow_results:
- for record in log_records:
- # distinguish active statusses
- style = ""
- if record['status'] in settings.ACTIVE_STATUSSES:
- style = "active"
+ records_per_status = workflow_result['records_per_status']
+ link = construct_link_to_workflow_api(request, workflow_result)
+ values = "<td><b>" + link + "</b></td>"
- line = "<tr><td><b>" + record['name'] + "</b></td>" \
- '<td class="' + style + '" >' + record['status'] + \
- "</td><td>" + str(record['cpu_cycles']) + \
- "</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
+ for status in records_per_status:
+
+ record = records_per_status[status]
+ # distinguish active statusses
+ style = ""
+ if status in settings.ACTIVE_STATUSSES or status=='active':
+ style = "active"
+ # show the values (done with a weird ternary operator)
+ if record['cpu_cycles']:
+ cpu_cycles = str(record['cpu_cycles'])
+ else:
+ cpu_cycles = '0'
- results_logs += line
+ if record['wall_clock_time']:
+ wall_clock_time = str(record['wall_clock_time'])
+ else:
+ wall_clock_time = '0'
- results_logs = "<tbody>" + results_logs + "</tbody>"
- return results_logs
+ value = cpu_cycles + '/' + wall_clock_time
+ values += "<td class=" + style + ">" + value + "</td>"
+
+ results += "<tr>" + values + "</tr>"
+
+ results = "<tbody>" + results + "</tbody>"
+ return results
def construct_dashboard_html(request):
@@ -356,9 +421,12 @@ def construct_dashboard_html(request):
workflow_results = aggregate_resources_tasks()
results_tasks = construct_tasks_per_workflow_html(request, workflow_results)
+ # --- logentries (first version with list instead of matrix---
+ # log_records = aggregate_resources_logs_version1()
+ #results_logs = construct_logs_per_workflow_html_version1(log_records)
+
# --- logentries ---
log_records = aggregate_resources_logs()
- results_logs = construct_logs_per_workflow_html(log_records)
-
+ results_logs = construct_logs_per_workflow_html(request, log_records)
return results_tasks,results_logs
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index 3ec33ed1..1ae8bb7b 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -80,7 +80,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 1.0.0 (16 mar 2021 - 13:00)
+ <p class="footer"> Version 1.0.0 (16 mar 2021 - 16:00)
</div>
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html b/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html
index 226f201e..fe918a0e 100644
--- a/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html
@@ -6,3 +6,4 @@
<a href="{% url 'task-details-setstatus' task.pk 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-sync-alt"></i> scrubbed</a>
<a href="{% url 'task-details-setstatus' task.pk 'archived' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-sync-alt"></i> archived</a>
<a href="{% url 'task-details-setstatus' task.pk 'finished' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-sync-alt"></i> finished</a>
+<a href="{% url 'task-details-setstatus' task.pk 'failed (manual)' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> failed (manual)</a>
--
GitLab