Add static rendering of input output objects

atdb/taskdatabase/services/algorithms.py

@@ -4,7 +4,7 @@
     Description:  Business logic for ATDB. These functions are called from the views (views.py).
 """
 
-from django.db.models import Q,Sum
+from django.db.models import Q, Sum
 import logging
 from .common import timeit
 from ..models import Task, LogEntry, Workflow
@@ -16,6 +16,7 @@ DJANGO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 
 logger = logging.getLogger(__name__)
 
+
 @timeit
 def get_size(status_list, type):
     """
@@ -24,7 +25,7 @@ def get_size(status_list, type):
     :return: summed sizes
     """
 
-    logger.info("get_size("+str(status_list)+")")
+    logger.info("get_size(" + str(status_list) + ")")
 
     if type == 'processed':
         field = 'size_processed'
@@ -49,7 +50,7 @@ def convert_logentries_to_html(log_entries):
         for log in log_entries:
             line = "<tr><td><b>" + str(log.service) + '</b></td>'
             line += "<td><b>" + str(log.step_name) + '</b></td>'
-            line +='<td class="' + log.status + '" >' + log.status + "</td>"
+            line += '<td class="' + log.status + '" >' + log.status + "</td>"
             try:
                 line += "<td>" + str(log.timestamp.strftime("%m-%d-%Y, %H:%M:%S")) + "</td>"
             except:
@@ -57,7 +58,7 @@ def convert_logentries_to_html(log_entries):
 
             line += "<td>" + str(log.cpu_cycles) + "</td>"
             line += "<td>" + str(log.wall_clock_time) + "</td>"
-            if log.url_to_log_file!=None:
+            if log.url_to_log_file != None:
                 link = "<a href=" + '"' + str(log.url_to_log_file) + '" target="_blank">' + "logfile" + "</a>"
             else:
                 link = "-"
@@ -66,8 +67,8 @@ def convert_logentries_to_html(log_entries):
 
         results += "</tbody>"
     except Exception as err:
-        results = "<tr><td>"+str(err)+"</td></tr>"
-        #results = "<tr><td>no data</td></tr>"
+        results = "<tr><td>" + str(err) + "</td></tr>"
+        # results = "<tr><td>no data</td></tr>"
 
     return results
 
@@ -85,10 +86,10 @@ def convert_list_of_dicts_to_html(my_blob):
     try:
         for my_dict in my_list:
             # iterate through the dict of key/values
-            for key,value in my_dict.items():
+            for key, value in my_dict.items():
                 try:
                     if "://" in value:
-                        link = "<a href=" + '"' + value + '">' + key +"</a>"
+                        link = "<a href=" + '"' + value + '">' + key + "</a>"
                         value = link
                 except:
                     pass
@@ -100,6 +101,48 @@ def convert_list_of_dicts_to_html(my_blob):
     return results
 
 
+import xml.etree.ElementTree as ElementTree
+from typing import Union, List, Dict
+
+
+def _generate_html_from_json_tree(json_blob: Union[List, Dict], element: ElementTree.Element):
+    if isinstance(json_blob, list) or isinstance(json_blob, tuple):
+
+        if element.tag != 'tbody':
+            sub_table = ElementTree.SubElement(element, 'table')
+        else:
+            sub_table = element
+        for item in json_blob:
+            row = ElementTree.SubElement(sub_table, 'tr')
+            element = ElementTree.SubElement(row, 'td')
+            _generate_html_from_json_tree(item, element)
+
+    elif isinstance(json_blob, dict):
+        if element.tag != 'tbody':
+            sub_table = ElementTree.SubElement(element, 'table')
+        else:
+            sub_table = element
+        for key, value in json_blob.items():
+            row = ElementTree.SubElement(sub_table, 'tr')
+            key_element = ElementTree.SubElement(row, 'td')
+            bold_key = ElementTree.SubElement(key_element, 'b')
+            bold_key.text = key
+            value_element = ElementTree.SubElement(row, 'td')
+
+            _generate_html_from_json_tree(value, value_element)
+
+    else:
+        value = ElementTree.SubElement(element, 'td', attrib={"style": "max-width:25rem"})
+        value.text = str(json_blob)
+
+
+def convert_json_to_nested_table(json_blob):
+    root_element = ElementTree.Element('tbody')
+    _generate_html_from_json_tree(json_blob, root_element)
+
+    return ElementTree.tostring(root_element, method='xml').decode()
+
+
 def convert_config_to_html(querylist):
     results = ""
     try:
@@ -111,12 +154,13 @@ def convert_config_to_html(querylist):
 
             try:
                 if "://" in value:
-                    link = "<a href=" + '"' + value + '">' + key +"</a>"
+                    link = "<a href=" + '"' + value + '">' + key + "</a>"
                     value = link
             except:
                 pass
 
-            line = "<tr><td><b>" + str(filter) + "</b></td> <td><b>" + str(key) + "</b></td><td>" + str(value) + "</td></tr>"
+            line = "<tr><td><b>" + str(filter) + "</b></td> <td><b>" + str(key) + "</b></td><td>" + str(
+                value) + "</td></tr>"
             results = results + line
     except:
         results = "<tr><td>no data</td></tr>"
@@ -126,7 +170,6 @@ def convert_config_to_html(querylist):
 
 # aggregate information from the tasks table per workflow per status
 def aggregate_resources_tasks(selection):
-
     workflow_results = []
     my_workflows = []
 
@@ -140,7 +183,7 @@ def aggregate_resources_tasks(selection):
         # construct the list of workflows (cheap)
         for w in active_workflows:
             try:
-                workflow = Workflow.objects.get(id = w['workflow'])
+                workflow = Workflow.objects.get(id=w['workflow'])
                 my_workflows.append(workflow)
             except:
                 pass
@@ -209,7 +252,7 @@ def aggregate_resources_logs(selection):
         # construct the list of workflows (cheap)
         for w in active_workflows:
             try:
-                workflow = Workflow.objects.get(id = w['workflow'])
+                workflow = Workflow.objects.get(id=w['workflow'])
                 my_workflows.append(workflow)
             except:
                 pass
@@ -226,8 +269,8 @@ def aggregate_resources_logs(selection):
             record = {}
 
             # aggregate logentries per step for all active statusses (expensive)
-            logs = LogEntry.objects.filter(status=status)\
-                .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+            logs = LogEntry.objects.filter(status=status) \
+                .filter(task__status__in=settings.ACTIVE_STATUSSES) \
                 .filter(task__workflow=workflow)
 
             sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
@@ -242,13 +285,12 @@ def aggregate_resources_logs(selection):
         workflow_result['records_per_status'] = record_per_status
 
         workflow_results.append(workflow_result)
-        
+
     return workflow_results
 
 
 # aggregate information from the logentries table per workflow per status
 def aggregate_resources_logs_version1():
-
     records = []
 
     # get all active tasks
@@ -263,18 +305,18 @@ def aggregate_resources_logs_version1():
         workflow_result = {}
 
         # extract the workflow object (cheap)
-        workflow = Workflow.objects.get(id = w['workflow'])
+        workflow = Workflow.objects.get(id=w['workflow'])
 
         # aggregate logentries per step for all active statusses
         for status in settings.ACTIVE_STATUSSES:
             record = {}
-            record['name'] = str(workflow.id) +' - '+ workflow.workflow_uri
+            record['name'] = str(workflow.id) + ' - ' + workflow.workflow_uri
             # record['name'] = str(workflow.id)
             record['status'] = status
 
             # aggregate logentries per step for all active statusses (expensive)
-            logs = LogEntry.objects.filter(status=status)\
-                .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+            logs = LogEntry.objects.filter(status=status) \
+                .filter(task__status__in=settings.ACTIVE_STATUSSES) \
                 .filter(task__workflow=workflow)
 
             sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
@@ -344,7 +386,6 @@ def human_readable(size_in_bytes):
 
 
 def highlight_value(values, value_to_highlight):
-
     # find 'class' left of the value
     pos_value = values.find(str(value_to_highlight))
 
@@ -362,8 +403,8 @@ def highlight_value(values, value_to_highlight):
 
     return values
 
-def construct_tasks_per_workflow_html(request, workflow_results):
 
+def construct_tasks_per_workflow_html(request, workflow_results):
     # --- Progress of tasks per active workflow ---
     results_tasks = "<p>Progress of tasks per workflow</p>"
 
@@ -390,7 +431,8 @@ def construct_tasks_per_workflow_html(request, workflow_results):
             percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
         except:
             percentage = 0
-        values += "<td><b>size processed:</b> " + str(human_readable(workflow_result['size_processed'])) + " (<b>"+ str(percentage) + "%</b>) </td>"
+        values += "<td><b>size processed:</b> " + str(
+            human_readable(workflow_result['size_processed'])) + " (<b>" + str(percentage) + "%</b>) </td>"
         values += "<td><b>processing time:</b> " + str(workflow_result['total_processing_time']) + "</td>"
 
         values += "<td colspan='8'></td></tr><tr>"
@@ -409,7 +451,7 @@ def construct_tasks_per_workflow_html(request, workflow_results):
 
             # distinguish active statusses
             style = "inactive"
-            if key in settings.ACTIVE_STATUSSES or key=='active':
+            if key in settings.ACTIVE_STATUSSES or key == 'active':
                 style = "active"
 
             # bonus: add a query link
@@ -417,15 +459,15 @@ def construct_tasks_per_workflow_html(request, workflow_results):
             values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"
 
         # add sizes
-#        values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
-#        try:
-#            percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
-#        except:
-#            percentage = 0
-#        values += "<td>" + str(human_readable(workflow_result['size_processed'])) + " ("+ str(percentage) + "%) </td>"
-#        values += "<td>" + str(workflow_result['total_processing_time']) + "</td>"
-
-        if max>0:
+        #        values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
+        #        try:
+        #            percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
+        #        except:
+        #            percentage = 0
+        #        values += "<td>" + str(human_readable(workflow_result['size_processed'])) + " ("+ str(percentage) + "%) </td>"
+        #        values += "<td>" + str(workflow_result['total_processing_time']) + "</td>"
+
+        if max > 0:
             values = highlight_value(values, max)
 
         results_tasks += "</tr><tr>" + values + "</tr>"
@@ -448,7 +490,7 @@ def construct_logs_per_workflow_html_version1(log_records):
             style = "active"
 
         line = "<tr><td><b>" + record['name'] + "</b></td>" \
-                '<td class="' + style + '" >' + record['status'] + \
+                                                '<td class="' + style + '" >' + record['status'] + \
                "</td><td>" + str(record['cpu_cycles']) + \
                "</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
 
@@ -475,25 +517,25 @@ def construct_logs_per_workflow_html(request, workflow_results):
 
         for status in records_per_status:
 
-             record = records_per_status[status]
-             # distinguish active statusses
-             style = ""
-             if status in settings.ACTIVE_STATUSSES or status=='active':
+            record = records_per_status[status]
+            # distinguish active statusses
+            style = ""
+            if status in settings.ACTIVE_STATUSSES or status == 'active':
                 style = "active"
 
-             # show the values (done with a weird ternary operator)
-             if record['cpu_cycles']:
-                 cpu_cycles = str(record['cpu_cycles'])
-             else:
-                 cpu_cycles = '0'
+            # show the values (done with a weird ternary operator)
+            if record['cpu_cycles']:
+                cpu_cycles = str(record['cpu_cycles'])
+            else:
+                cpu_cycles = '0'
 
-             if record['wall_clock_time']:
-                 wall_clock_time = str(record['wall_clock_time'])
-             else:
-                 wall_clock_time = '0'
+            if record['wall_clock_time']:
+                wall_clock_time = str(record['wall_clock_time'])
+            else:
+                wall_clock_time = '0'
 
-             value = cpu_cycles + '/' + wall_clock_time
-             values += "<td class=" + style + ">" + value + "</td>"
+            value = cpu_cycles + '/' + wall_clock_time
+            values += "<td class=" + style + ">" + value + "</td>"
 
         results += "<tr>" + values + "</tr>"
 
@@ -502,7 +544,6 @@ def construct_logs_per_workflow_html(request, workflow_results):
 
 
 def construct_dashboard_html(request, selection):
-
     # gather and construct the dashboard based on the requested selection
 
     # --- Progress of tasks per active workflow ---
@@ -515,5 +556,4 @@ def construct_dashboard_html(request, selection):
         log_records = aggregate_resources_logs(selection)
         results_logs = construct_logs_per_workflow_html(request, log_records)
 
-    return results_tasks,results_logs
-
+    return results_tasks, results_logs

atdb/taskdatabase/templates/taskdatabase/details/inputs.html

@@ -7,37 +7,10 @@
  <div class="card-body">
      <h3>Inputs </h3>
        <table class="table table-striped">
-           <tbody  id="inputs_table">
+           <tbody>
+                {{ results | safe }}
            </tbody>
        </table>
  </div>
 </div>
-<script>
-    var inputs_values = {{ resultsjson | safe }}
-
-    function generate_tree(tree, div) {
-        if (Array.isArray(tree)) {
-            var array = $("<tbody></tbody>")
-            div.append(array)
-            for (let row_index in tree) {
-                var row = $('<tr></tr>')
-                generate_tree(tree[row_index], row)
-                array.append(row)
-            }
-
-        } else if (typeof (tree) === 'object' && tree !== null) {
-            for (var att_name in tree) {
-                var row = $(`<tr><td><b>${att_name}</b></td></tr>`)
-                generate_tree(tree[att_name], row);
-                div.append(
-                    row
-                )
-            }
-        } else {
-            div.append(`<td>${tree}</td>`)
-        }
-    }
-
-    generate_tree(inputs_values, $('#inputs_table'))
-</script>
 {% endblock %}
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/details/outputs.html

@@ -6,39 +6,9 @@
     <div class="card-body">
         <h3>Outputs </h3>
         <table class="table table-striped">
-            <tbody id="output_values">
-
-            </tbody>
+            {{ results | safe }}
         </table>
     </div>
 </div>
 
-
-<script>
-    var output_values = {{ resultsjson | safe }}
-
-    function generate_tree(tree, div) {
-        if (Array.isArray(tree)) {
-            var array = $("<tbody></tbody>")
-            div.append(array)
-            for (let row_index in tree) {
-                var row = $('<tr></tr>')
-                generate_tree(tree[row_index], row)
-                array.append(row)
-            }
-        } else if (typeof (tree) === 'object' && tree !== null) {
-            for (var att_name in tree) {
-                var row = $(`<tr><td><b>${att_name}</b></td></tr>`)
-                generate_tree(tree[att_name], row);
-                div.append(
-                    row
-                )
-            }
-        } else {
-            div.append(`<td>${tree}</td>`)
-        }
-    }
-
-    generate_tree(output_values, $('#output_values'))
-</script>
 {% endblock %}
\ No newline at end of file

atdb/taskdatabase/views.py

@@ -266,16 +266,16 @@ def ShowInputs(request, id):
     task = Task.objects.get(id=id)
 
     # convert the json to a presentable piece of html for the output template
-    results = algorithms.convert_list_of_dicts_to_html(task.inputs)
-    return render(request, "taskdatabase/details/inputs.html", {'results': results, 'resultsjson': task.inputs})
+    results = algorithms.convert_json_to_nested_table(task.inputs)
+    return render(request, "taskdatabase/details/inputs.html", {'results': results})
 
 
 def ShowOutputs(request, id):
     task = Task.objects.get(id=id)
-
     # convert the json to a presentable piece of html for the output template
-    results = algorithms.convert_list_of_dicts_to_html(task.outputs)
-    return render(request, "taskdatabase/details/outputs.html", {'results': results, 'resultsjson': task.outputs})
+    results = algorithms.convert_json_to_nested_table(task.outputs)
+
+    return render(request, "taskdatabase/details/outputs.html", {'results': results})
 
 
 def ShowMetrics(request, id):