diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 054ad10b091c7bf3931a25874465a780968e5426..7baaa014d0599d4efdc2ca453207b966cfeaada7 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -4,7 +4,7 @@
Description: Business logic for ATDB. These functions are called from the views (views.py).
"""
-from django.db.models import Q,Sum
+from django.db.models import Q, Sum
import logging
from .common import timeit
from ..models import Task, LogEntry, Workflow
@@ -16,6 +16,7 @@ DJANGO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
logger = logging.getLogger(__name__)
+
@timeit
def get_size(status_list, type):
"""
@@ -24,7 +25,7 @@ def get_size(status_list, type):
:return: summed sizes
"""
- logger.info("get_size("+str(status_list)+")")
+ logger.info("get_size(" + str(status_list) + ")")
if type == 'processed':
field = 'size_processed'
@@ -49,7 +50,7 @@ def convert_logentries_to_html(log_entries):
for log in log_entries:
line = "<tr><td><b>" + str(log.service) + '</b></td>'
line += "<td><b>" + str(log.step_name) + '</b></td>'
- line +='<td class="' + log.status + '" >' + log.status + "</td>"
+ line += '<td class="' + log.status + '" >' + log.status + "</td>"
try:
line += "<td>" + str(log.timestamp.strftime("%m-%d-%Y, %H:%M:%S")) + "</td>"
except:
@@ -57,7 +58,7 @@ def convert_logentries_to_html(log_entries):
line += "<td>" + str(log.cpu_cycles) + "</td>"
line += "<td>" + str(log.wall_clock_time) + "</td>"
- if log.url_to_log_file!=None:
+ if log.url_to_log_file != None:
link = "<a href=" + '"' + str(log.url_to_log_file) + '" target="_blank">' + "logfile" + "</a>"
else:
link = "-"
@@ -66,8 +67,8 @@ def convert_logentries_to_html(log_entries):
results += "</tbody>"
except Exception as err:
- results = "<tr><td>"+str(err)+"</td></tr>"
- #results = "<tr><td>no data</td></tr>"
+ results = "<tr><td>" + str(err) + "</td></tr>"
+ # results = "<tr><td>no data</td></tr>"
return results
@@ -85,10 +86,10 @@ def convert_list_of_dicts_to_html(my_blob):
try:
for my_dict in my_list:
# iterate through the dict of key/values
- for key,value in my_dict.items():
+ for key, value in my_dict.items():
try:
if "://" in value:
- link = "<a href=" + '"' + value + '">' + key +"</a>"
+ link = "<a href=" + '"' + value + '">' + key + "</a>"
value = link
except:
pass
@@ -100,6 +101,48 @@ def convert_list_of_dicts_to_html(my_blob):
return results
+import xml.etree.ElementTree as ElementTree
+from typing import Union, List, Dict
+
+
+def _generate_html_from_json_tree(json_blob: Union[List, Dict], element: ElementTree.Element):
+ if isinstance(json_blob, list) or isinstance(json_blob, tuple):
+
+ if element.tag != 'tbody':
+ sub_table = ElementTree.SubElement(element, 'table')
+ else:
+ sub_table = element
+ for item in json_blob:
+ row = ElementTree.SubElement(sub_table, 'tr')
+ element = ElementTree.SubElement(row, 'td')
+ _generate_html_from_json_tree(item, element)
+
+ elif isinstance(json_blob, dict):
+ if element.tag != 'tbody':
+ sub_table = ElementTree.SubElement(element, 'table')
+ else:
+ sub_table = element
+ for key, value in json_blob.items():
+ row = ElementTree.SubElement(sub_table, 'tr')
+ key_element = ElementTree.SubElement(row, 'td')
+ bold_key = ElementTree.SubElement(key_element, 'b')
+ bold_key.text = key
+ value_element = ElementTree.SubElement(row, 'td')
+
+ _generate_html_from_json_tree(value, value_element)
+
+ else:
+ value = ElementTree.SubElement(element, 'td', attrib={"style": "max-width:25rem"})
+ value.text = str(json_blob)
+
+
+def convert_json_to_nested_table(json_blob):
+ root_element = ElementTree.Element('tbody')
+ _generate_html_from_json_tree(json_blob, root_element)
+
+ return ElementTree.tostring(root_element, method='xml').decode()
+
+
def convert_config_to_html(querylist):
results = ""
try:
@@ -111,12 +154,13 @@ def convert_config_to_html(querylist):
try:
if "://" in value:
- link = "<a href=" + '"' + value + '">' + key +"</a>"
+ link = "<a href=" + '"' + value + '">' + key + "</a>"
value = link
except:
pass
- line = "<tr><td><b>" + str(filter) + "</b></td> <td><b>" + str(key) + "</b></td><td>" + str(value) + "</td></tr>"
+ line = "<tr><td><b>" + str(filter) + "</b></td> <td><b>" + str(key) + "</b></td><td>" + str(
+ value) + "</td></tr>"
results = results + line
except:
results = "<tr><td>no data</td></tr>"
@@ -126,7 +170,6 @@ def convert_config_to_html(querylist):
# aggregate information from the tasks table per workflow per status
def aggregate_resources_tasks(selection):
-
workflow_results = []
my_workflows = []
@@ -140,7 +183,7 @@ def aggregate_resources_tasks(selection):
# construct the list of workflows (cheap)
for w in active_workflows:
try:
- workflow = Workflow.objects.get(id = w['workflow'])
+ workflow = Workflow.objects.get(id=w['workflow'])
my_workflows.append(workflow)
except:
pass
@@ -209,7 +252,7 @@ def aggregate_resources_logs(selection):
# construct the list of workflows (cheap)
for w in active_workflows:
try:
- workflow = Workflow.objects.get(id = w['workflow'])
+ workflow = Workflow.objects.get(id=w['workflow'])
my_workflows.append(workflow)
except:
pass
@@ -226,8 +269,8 @@ def aggregate_resources_logs(selection):
record = {}
# aggregate logentries per step for all active statusses (expensive)
- logs = LogEntry.objects.filter(status=status)\
- .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+ logs = LogEntry.objects.filter(status=status) \
+ .filter(task__status__in=settings.ACTIVE_STATUSSES) \
.filter(task__workflow=workflow)
sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
@@ -242,13 +285,12 @@ def aggregate_resources_logs(selection):
workflow_result['records_per_status'] = record_per_status
workflow_results.append(workflow_result)
-
+
return workflow_results
# aggregate information from the logentries table per workflow per status
def aggregate_resources_logs_version1():
-
records = []
# get all active tasks
@@ -263,18 +305,18 @@ def aggregate_resources_logs_version1():
workflow_result = {}
# extract the workflow object (cheap)
- workflow = Workflow.objects.get(id = w['workflow'])
+ workflow = Workflow.objects.get(id=w['workflow'])
# aggregate logentries per step for all active statusses
for status in settings.ACTIVE_STATUSSES:
record = {}
- record['name'] = str(workflow.id) +' - '+ workflow.workflow_uri
+ record['name'] = str(workflow.id) + ' - ' + workflow.workflow_uri
# record['name'] = str(workflow.id)
record['status'] = status
# aggregate logentries per step for all active statusses (expensive)
- logs = LogEntry.objects.filter(status=status)\
- .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+ logs = LogEntry.objects.filter(status=status) \
+ .filter(task__status__in=settings.ACTIVE_STATUSSES) \
.filter(task__workflow=workflow)
sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
@@ -344,7 +386,6 @@ def human_readable(size_in_bytes):
def highlight_value(values, value_to_highlight):
-
# find 'class' left of the value
pos_value = values.find(str(value_to_highlight))
@@ -362,8 +403,8 @@ def highlight_value(values, value_to_highlight):
return values
-def construct_tasks_per_workflow_html(request, workflow_results):
+def construct_tasks_per_workflow_html(request, workflow_results):
# --- Progress of tasks per active workflow ---
results_tasks = "<p>Progress of tasks per workflow</p>"
@@ -390,7 +431,8 @@ def construct_tasks_per_workflow_html(request, workflow_results):
percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
except:
percentage = 0
- values += "<td><b>size processed:</b> " + str(human_readable(workflow_result['size_processed'])) + " (<b>"+ str(percentage) + "%</b>) </td>"
+ values += "<td><b>size processed:</b> " + str(
+ human_readable(workflow_result['size_processed'])) + " (<b>" + str(percentage) + "%</b>) </td>"
values += "<td><b>processing time:</b> " + str(workflow_result['total_processing_time']) + "</td>"
values += "<td colspan='8'></td></tr><tr>"
@@ -409,7 +451,7 @@ def construct_tasks_per_workflow_html(request, workflow_results):
# distinguish active statusses
style = "inactive"
- if key in settings.ACTIVE_STATUSSES or key=='active':
+ if key in settings.ACTIVE_STATUSSES or key == 'active':
style = "active"
# bonus: add a query link
@@ -417,15 +459,15 @@ def construct_tasks_per_workflow_html(request, workflow_results):
values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"
# add sizes
-# values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
-# try:
-# percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
-# except:
-# percentage = 0
-# values += "<td>" + str(human_readable(workflow_result['size_processed'])) + " ("+ str(percentage) + "%) </td>"
-# values += "<td>" + str(workflow_result['total_processing_time']) + "</td>"
-
- if max>0:
+ # values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
+ # try:
+ # percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
+ # except:
+ # percentage = 0
+ # values += "<td>" + str(human_readable(workflow_result['size_processed'])) + " ("+ str(percentage) + "%) </td>"
+ # values += "<td>" + str(workflow_result['total_processing_time']) + "</td>"
+
+ if max > 0:
values = highlight_value(values, max)
results_tasks += "</tr><tr>" + values + "</tr>"
@@ -448,7 +490,7 @@ def construct_logs_per_workflow_html_version1(log_records):
style = "active"
line = "<tr><td><b>" + record['name'] + "</b></td>" \
- '<td class="' + style + '" >' + record['status'] + \
+ '<td class="' + style + '" >' + record['status'] + \
"</td><td>" + str(record['cpu_cycles']) + \
"</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
@@ -475,25 +517,25 @@ def construct_logs_per_workflow_html(request, workflow_results):
for status in records_per_status:
- record = records_per_status[status]
- # distinguish active statusses
- style = ""
- if status in settings.ACTIVE_STATUSSES or status=='active':
+ record = records_per_status[status]
+ # distinguish active statusses
+ style = ""
+ if status in settings.ACTIVE_STATUSSES or status == 'active':
style = "active"
- # show the values (done with a weird ternary operator)
- if record['cpu_cycles']:
- cpu_cycles = str(record['cpu_cycles'])
- else:
- cpu_cycles = '0'
+ # show the values (done with a weird ternary operator)
+ if record['cpu_cycles']:
+ cpu_cycles = str(record['cpu_cycles'])
+ else:
+ cpu_cycles = '0'
- if record['wall_clock_time']:
- wall_clock_time = str(record['wall_clock_time'])
- else:
- wall_clock_time = '0'
+ if record['wall_clock_time']:
+ wall_clock_time = str(record['wall_clock_time'])
+ else:
+ wall_clock_time = '0'
- value = cpu_cycles + '/' + wall_clock_time
- values += "<td class=" + style + ">" + value + "</td>"
+ value = cpu_cycles + '/' + wall_clock_time
+ values += "<td class=" + style + ">" + value + "</td>"
results += "<tr>" + values + "</tr>"
@@ -502,7 +544,6 @@ def construct_logs_per_workflow_html(request, workflow_results):
def construct_dashboard_html(request, selection):
-
# gather and construct the dashboard based on the requested selection
# --- Progress of tasks per active workflow ---
@@ -515,5 +556,4 @@ def construct_dashboard_html(request, selection):
log_records = aggregate_resources_logs(selection)
results_logs = construct_logs_per_workflow_html(request, log_records)
- return results_tasks,results_logs
-
+ return results_tasks, results_logs
diff --git a/atdb/taskdatabase/templates/taskdatabase/details/inputs.html b/atdb/taskdatabase/templates/taskdatabase/details/inputs.html
index fa84403790f770fcc41086570b9aa6d4134062bf..3d6bb9935f8209171ea00bf59adab92d3b9d8abe 100644
--- a/atdb/taskdatabase/templates/taskdatabase/details/inputs.html
+++ b/atdb/taskdatabase/templates/taskdatabase/details/inputs.html
@@ -7,37 +7,10 @@
<div class="card-body">
<h3>Inputs </h3>
<table class="table table-striped">
- <tbody id="inputs_table">
+ <tbody>
+ {{ results | safe }}
</tbody>
</table>
</div>
</div>
-<script>
- var inputs_values = {{ resultsjson | safe }}
-
- function generate_tree(tree, div) {
- if (Array.isArray(tree)) {
- var array = $("<tbody></tbody>")
- div.append(array)
- for (let row_index in tree) {
- var row = $('<tr></tr>')
- generate_tree(tree[row_index], row)
- array.append(row)
- }
-
- } else if (typeof (tree) === 'object' && tree !== null) {
- for (var att_name in tree) {
- var row = $(`<tr><td><b>${att_name}</b></td></tr>`)
- generate_tree(tree[att_name], row);
- div.append(
- row
- )
- }
- } else {
- div.append(`<td>${tree}</td>`)
- }
- }
-
- generate_tree(inputs_values, $('#inputs_table'))
-</script>
{% endblock %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/details/outputs.html b/atdb/taskdatabase/templates/taskdatabase/details/outputs.html
index 91851d607e85fe9c255c468ddd69abafb4fd448d..9b31ef46489dcda8e6b305f102ae2bd95db78a67 100644
--- a/atdb/taskdatabase/templates/taskdatabase/details/outputs.html
+++ b/atdb/taskdatabase/templates/taskdatabase/details/outputs.html
@@ -6,39 +6,9 @@
<div class="card-body">
<h3>Outputs </h3>
<table class="table table-striped">
- <tbody id="output_values">
-
- </tbody>
+ {{ results | safe }}
</table>
</div>
</div>
-
-<script>
- var output_values = {{ resultsjson | safe }}
-
- function generate_tree(tree, div) {
- if (Array.isArray(tree)) {
- var array = $("<tbody></tbody>")
- div.append(array)
- for (let row_index in tree) {
- var row = $('<tr></tr>')
- generate_tree(tree[row_index], row)
- array.append(row)
- }
- } else if (typeof (tree) === 'object' && tree !== null) {
- for (var att_name in tree) {
- var row = $(`<tr><td><b>${att_name}</b></td></tr>`)
- generate_tree(tree[att_name], row);
- div.append(
- row
- )
- }
- } else {
- div.append(`<td>${tree}</td>`)
- }
- }
-
- generate_tree(output_values, $('#output_values'))
-</script>
{% endblock %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 91714ac8f2d4bff1e73cd8b9eff67f3172505cce..57d6d022f4cbe4b3a975dd486729286c648ada04 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -266,16 +266,16 @@ def ShowInputs(request, id):
task = Task.objects.get(id=id)
# convert the json to a presentable piece of html for the output template
- results = algorithms.convert_list_of_dicts_to_html(task.inputs)
- return render(request, "taskdatabase/details/inputs.html", {'results': results, 'resultsjson': task.inputs})
+ results = algorithms.convert_json_to_nested_table(task.inputs)
+ return render(request, "taskdatabase/details/inputs.html", {'results': results})
def ShowOutputs(request, id):
task = Task.objects.get(id=id)
-
# convert the json to a presentable piece of html for the output template
- results = algorithms.convert_list_of_dicts_to_html(task.outputs)
- return render(request, "taskdatabase/details/outputs.html", {'results': results, 'resultsjson': task.outputs})
+ results = algorithms.convert_json_to_nested_table(task.outputs)
+
+ return render(request, "taskdatabase/details/outputs.html", {'results': results})
def ShowMetrics(request, id):