dashboard now shows all workflows (not just active tasks)

atdb/taskdatabase/services/algorithms.py

 """
     File name: algorithms.py
     Author: Nico Vermaas - Astron
-    Date created: 2019-04-04
     Description:  Business logic for ATDB. These functions are called from the views (views.py).
 """
 
@@ -106,22 +105,25 @@ def convert_config_to_html(querylist):
 
 # aggregate information from the tasks table per workflow per status
 def aggregate_resources_tasks():
+    # NOTE: uncomment (or refactor) the lines with ### to only aggregate the data for active tasks
 
     workflow_results = []
 
     # get all active tasks
-    active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
-    # active_tasks_count = active_tasks.count()
+    ### active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
 
-    # retrieve all unique workflows
-    active_workflows = active_tasks.values('workflow').distinct()
+    # retrieve all unique workflows from the active tasks
+    ### active_workflows = active_tasks.values('workflow').distinct()
+
+    all_workflows = Workflow.objects.all()
 
     # iterate through the filters and accumulate logentries
-    for w in active_workflows:
+    ### for w in all_workflows:
+    for workflow in all_workflows:
         workflow_result = {}
 
         # extract the workflow object (cheap)
-        workflow = Workflow.objects.get(id = w['workflow'])
+        ### workflow = Workflow.objects.get(id = w['workflow'])
 
         # get the numbers for this workflow
 
@@ -143,9 +145,7 @@ def aggregate_resources_tasks():
         nr_per_status = {}
 
         for status in settings.ALL_STATUSSES:
-            record = {}
             nr_for_this_status = Task.objects.filter(workflow=workflow, status=status).count()
-            record[status] = nr_for_this_status
             nr_per_status[status] = nr_for_this_status
 
         nr_per_status['active'] = nr_of_active_tasks_per_workflow
@@ -165,6 +165,54 @@ def aggregate_resources_tasks():
 
 # aggregate information from the logentries table per workflow per status
 def aggregate_resources_logs():
+    # NOTE: uncomment (or refactor) the lines with ### to only aggregate the data for active tasks
+
+    workflow_results = []
+
+    # get all active tasks
+    ### active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
+
+    # retrieve all unique workflows
+    ### active_workflows = active_tasks.values('workflow').distinct()
+
+    all_workflows = Workflow.objects.all()
+    # iterate through the filters and accumulate logentries
+
+    ### for w in active_workflows:
+    for workflow in all_workflows:
+        workflow_result = {}
+
+        # extract the workflow object (cheap)
+        ### workflow = Workflow.objects.get(id = w['workflow'])
+
+        # aggregate logentries per step for all active statusses
+        record_per_status = {}
+        for status in settings.ALL_STATUSSES:
+            record = {}
+
+            # aggregate logentries per step for all active statusses (expensive)
+            logs = LogEntry.objects.filter(status=status)\
+                .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+                .filter(task__workflow=workflow)
+
+            sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
+            record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
+
+            wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
+            record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
+            record_per_status[status] = record
+
+        workflow_result['id'] = workflow.id
+        workflow_result['name'] = workflow.workflow_uri
+        workflow_result['records_per_status'] = record_per_status
+
+        workflow_results.append(workflow_result)
+        
+    return workflow_results
+
+
+# aggregate information from the logentries table per workflow per status
+def aggregate_resources_logs_version1():
 
     records = []
 
@@ -243,19 +291,20 @@ def construct_link_to_workflow_api(request, workflow_result):
 
 
 def human_readable(size_in_bytes):
-    for count in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
-        if size_in_bytes > -1024.0 and size_in_bytes < 1024.0:
-            return "%3.1f %s" % (size_in_bytes, count)
-        size_in_bytes /= 1024.0
-    return "%3.1f %s" % (size_in_bytes, 'PB')
+    try:
+        for count in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
+            if size_in_bytes > -1024.0 and size_in_bytes < 1024.0:
+                return "%3.1f %s" % (size_in_bytes, count)
+            size_in_bytes /= 1024.0
+        return "%3.1f %s" % (size_in_bytes, 'PB')
+    except:
+        return "0"
 
 
 def construct_tasks_per_workflow_html(request, workflow_results):
 
     # --- Progress of tasks per active workflow ---
-    workflow_results = aggregate_resources_tasks()
-
-    results_tasks = "<p>Progress of tasks per (active) workflow</p>"
+    results_tasks = "<p>Progress of tasks per workflow</p>"
 
     # construct the header
     header = "<th>Workflow</th>"
@@ -267,22 +316,23 @@ def construct_tasks_per_workflow_html(request, workflow_results):
     for workflow_result in workflow_results:
 
         d = workflow_result['nr_of_tasks_per_status']
-        #values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>"
-        # values = "<td><b>" + str(workflow_result['id']) + "</b></td>"
         link = construct_link_to_workflow_api(request, workflow_result)
         values = "<td><b>" + link + "</b></td>"
 
         for key in d:
-             percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
+            try:
+                percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
+            except:
+                percentage = 0
 
-             # distinguish active statusses
-             style = ""
-             if key in settings.ACTIVE_STATUSSES or key=='active':
+            # distinguish active statusses
+            style = ""
+            if key in settings.ACTIVE_STATUSSES or key=='active':
                 style = "active"
 
-             # bonus: add a query link
-             link = construct_link_to_tasks_api(request, key, workflow_result['id'], d[key])
-             values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"
+            # bonus: add a query link
+            link = construct_link_to_tasks_api(request, key, workflow_result['id'], d[key])
+            values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"
 
         # add sizes
         values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
@@ -298,7 +348,7 @@ def construct_tasks_per_workflow_html(request, workflow_results):
     return results_tasks
 
 
-def construct_logs_per_workflow_html(log_records):
+def construct_logs_per_workflow_html_version1(log_records):
     results_logs = "<p>Resources used per step per active workflow</p>"
 
     # construct the header
@@ -312,7 +362,7 @@ def construct_logs_per_workflow_html(log_records):
             style = "active"
 
         line = "<tr><td><b>" + record['name'] + "</b></td>" \
-                                                '<td class="' + style + '" >' + record['status'] + \
+                '<td class="' + style + '" >' + record['status'] + \
                "</td><td>" + str(record['cpu_cycles']) + \
                "</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
 
@@ -322,32 +372,47 @@ def construct_logs_per_workflow_html(log_records):
     return results_logs
 
 
-def construct_cpu_cycles_per_workflow_html(log_records):
-    results_logs = "<p>Resources used per step per active workflow</p>"
+def construct_logs_per_workflow_html(request, workflow_results):
+    results = "<p>Resources used per step per workflow: <b>cpu_cycles/wall_clock_time</b></p>"
 
     # construct the header
     header = "<th>Workflow</th>"
     for status in settings.ALL_STATUSSES:
         header += "<th>" + status + "</th>"
+    results += header
 
-    results_logs += header + '<th>CPU cycles</th><th>wall clock time</th><th>to process</th><th>processed</th>'
+    for workflow_result in workflow_results:
 
-    for record in log_records:
-        # distinguish active statusses
-        style = ""
-        if record['status'] in settings.ACTIVE_STATUSSES:
-            style = "active"
+        records_per_status = workflow_result['records_per_status']
+        link = construct_link_to_workflow_api(request, workflow_result)
+        values = "<td><b>" + link + "</b></td>"
 
-        line = "<tr><td><b>" + record['name'] + "</b></td>" \
-                                                '<td class="' + style + '" >' + record['status'] + \
-               "</td><td>" + str(record['cpu_cycles']) + \
-               "</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
+        for status in records_per_status:
+
+             record = records_per_status[status]
+             # distinguish active statusses
+             style = ""
+             if status in settings.ACTIVE_STATUSSES or status=='active':
+                style = "active"
 
+             # show the values (done with a weird ternary operator)
+             if record['cpu_cycles']:
+                 cpu_cycles = str(record['cpu_cycles'])
+             else:
+                 cpu_cycles = '0'
 
-        results_logs += line
+             if record['wall_clock_time']:
+                 wall_clock_time = str(record['wall_clock_time'])
+             else:
+                 wall_clock_time = '0'
 
-    results_logs = "<tbody>" + results_logs + "</tbody>"
-    return results_logs
+             value = cpu_cycles + '/' + wall_clock_time
+             values += "<td class=" + style + ">" + value + "</td>"
+
+        results += "<tr>" + values + "</tr>"
+
+    results = "<tbody>" + results + "</tbody>"
+    return results
 
 
 def construct_dashboard_html(request):
@@ -356,9 +421,12 @@ def construct_dashboard_html(request):
     workflow_results = aggregate_resources_tasks()
     results_tasks = construct_tasks_per_workflow_html(request, workflow_results)
 
+    # --- logentries (first version with list instead of matrix---
+    # log_records = aggregate_resources_logs_version1()
+    #results_logs = construct_logs_per_workflow_html_version1(log_records)
+
     # --- logentries ---
     log_records = aggregate_resources_logs()
-    results_logs = construct_logs_per_workflow_html(log_records)
-
+    results_logs = construct_logs_per_workflow_html(request, log_records)
     return results_tasks,results_logs
 

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -80,7 +80,7 @@
         {% include 'taskdatabase/pagination.html' %}
        </div>
     </div>
-    <p class="footer"> Version 1.0.0 (16 mar 2021 - 13:00)
+    <p class="footer"> Version 1.0.0 (16 mar 2021 - 16:00)
 
 </div>
 

atdb/taskdatabase/templates/taskdatabase/tasks/set_status_buttons.html

@@ -6,3 +6,4 @@
 <a href="{% url 'task-details-setstatus' task.pk 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-sync-alt"></i> scrubbed</a>&nbsp;
 <a href="{% url 'task-details-setstatus' task.pk 'archived' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-sync-alt"></i> archived</a>&nbsp;
 <a href="{% url 'task-details-setstatus' task.pk 'finished' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-sync-alt"></i> finished</a>&nbsp;
+<a href="{% url 'task-details-setstatus' task.pk 'failed (manual)' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> failed (manual)</a>&nbsp;