diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index af3b756e18245c8fa77ed344591972dbb67dda6e..d536e05e95822970d092061d29410e820c769476 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -188,5 +188,7 @@ LOGOUT_REDIRECT_URL = '/atdb'
STATIC_URL = '/atdb/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
-ACTIVE_STATUSSES = ['staged','processing','processed','validated', 'scrubbing','scrubbed','archiving','archived']
+ALL_STATUSSES = ['defining','defined','staging','staged','processing','processed','validated', 'scrubbing','scrubbed','archiving','archived','finished']
+ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated', 'scrubbing','scrubbed','archiving','archived']
+STATUSSES_WITH_DATA = ['staged','processing','processed','validated', 'scrubbing','scrubbed','archiving','archived']
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index c28e8c707f1f8b9af866e2ba15778d81ec831a84..109ccd8173b2cd4593a1c8df066b0a0205e8a31d 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -8,7 +8,8 @@
from django.db.models import Q,Sum
import logging
from .common import timeit
-from ..models import Task
+from ..models import Task, LogEntry, Workflow
+from django.conf import settings
DATE_FORMAT = "%Y-%m-%d"
TIME_FORMAT = "%Y-%m-%d %H:%M:%SZ"
@@ -36,5 +37,203 @@ def get_size(status_list):
return sum_value
+def convert_logentries_to_html(log_entries):
+ results = ""
+ try:
+ results += "<th>service</th><th>status</th><th>timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>logfile</th>"
+ results += "<tbody>"
+ for log in log_entries:
+ line = "<tr><td><b>" + log.step_name + '</b></td>'
+ line +='<td class="' + log.status + '" >' + log.status + "</td>"
+ line += "<td>" + str(log.timestamp.strftime("%m-%d-%Y, %H:%M:%S")) + "</td>"
+ line += "<td>" + str(log.cpu_cycles) + "</td>"
+ line += "<td>" + str(log.wall_clock_time) + "</td>"
+ if log.url_to_log_file!=None:
+ link = "<a href=" + '"' + str(log.url_to_log_file) + '" target="_blank">' + "logfile" + "</a>"
+ else:
+ link = "-"
+ line += "<td>" + link + "</td>"
+ results += line
+
+ results += "</tbody>"
+ except:
+ results = "<tr><td>no data</td></tr>"
+
+ return results
+
+
+def convert_list_of_dicts_to_html(my_list):
+ results = ""
+ try:
+ for my_dict in my_list:
+ # iterate through the dict of key/values
+ for key,value in my_dict.items():
+ try:
+ if "://" in value:
+ link = "<a href=" + '"' + value + '">' + key +"</a>"
+ value = link
+ except:
+ pass
+ line = "<tr><td><b>" + str(key) + "</b></td><td>" + str(value) + "</td></tr>"
+ results = results + line
+ except:
+ results = "<tr><td>no data</td></tr>"
+
+ return results
+
+
+def convert_config_to_html(querylist):
+ results = ""
+ try:
+ for record in querylist:
+ # iterate through the dict of key/values
+ key = record.key
+ value = record.value
+ try:
+ if "://" in value:
+ link = "<a href=" + '"' + value + '">' + key +"</a>"
+ value = link
+ except:
+ pass
+ line = "<tr><td><b>" + str(key) + "</b></td><td>" + str(value) + "</td></tr>"
+ results = results + line
+ except:
+ results = "<tr><td>no data</td></tr>"
+
+ return results
+
+
+
+def aggregate_resources_logs():
+
+ records = []
+
+ # get all active tasks
+ active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
+ active_tasks_count = active_tasks.count()
+
+ # retrieve all unique workflows
+ active_workflows = active_tasks.values('workflow').distinct()
+
+ # iterate through the filters and accumulate logentries
+ for w in active_workflows:
+ workflow_result = {}
+
+ # extract the workflow object (cheap)
+ workflow = Workflow.objects.get(id = w['workflow'])
+
+ # aggregate logentries per step for all active statusses
+ for status in settings.ACTIVE_STATUSSES:
+ record = {}
+ record['name'] = workflow.workflow_uri
+ record['status'] = status
+
+ # aggregate logentries per step for all active statusses (expensive)
+ logs = LogEntry.objects.filter(status=status)\
+ .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+ .filter(task__workflow=workflow)
+
+ sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
+ record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
+
+ wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
+ record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
+
+ records.append(record)
+
+ return records
+
+
+def aggregate_resources_tasks():
+
+ workflow_results = []
+ records = []
+
+ # get all active tasks
+ active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
+ active_tasks_count = active_tasks.count()
+
+ # retrieve all unique workflows
+ active_workflows = active_tasks.values('workflow').distinct()
+
+ # iterate through the filters and accumulate logentries
+ for w in active_workflows:
+ workflow_result = {}
+
+ # extract the workflow object (cheap)
+ workflow = Workflow.objects.get(id = w['workflow'])
+
+ # get the numbers for this workflow
+
+ # all tasks for this workflow for the 'grand total'
+ tasks_per_workflow = Task.objects.filter(workflow=workflow)
+ nr_of_tasks_per_workflow = tasks_per_workflow.count()
+
+ # all the active tasks
+ active_tasks_per_workflow = tasks_per_workflow.filter(status__in=settings.ACTIVE_STATUSSES)
+ nr_of_active_tasks_per_workflow = active_tasks_per_workflow.count()
+
+ # split per status, to see the progress
+ nr_per_status = {}
+
+ for status in settings.ALL_STATUSSES:
+ record = {}
+ nr_for_this_status = Task.objects.filter(workflow=workflow, status=status).count()
+ record[status] = nr_for_this_status
+ nr_per_status[status] = nr_for_this_status
+
+ nr_per_status['active'] = nr_of_active_tasks_per_workflow
+ nr_per_status['total'] = nr_of_tasks_per_workflow
+
+ # store the results in a dict
+ workflow_result['id'] = workflow.id
+ workflow_result['name'] = workflow.workflow_uri
+ workflow_result['nr_of_tasks'] = nr_of_tasks_per_workflow
+ workflow_result['nr_of_active_tasks'] = nr_of_active_tasks_per_workflow
+ workflow_result['nr_of_tasks_per_status'] = nr_per_status
+
+ workflow_results.append(workflow_result)
+
+ return workflow_results
+
+
+def convert_aggregation_to_html():
+
+ # gather the data
+ workflow_results = aggregate_resources_tasks()
+ records = aggregate_resources_logs()
+
+ # layout the data
+
+ # --- Progress of tasks per active workflow ---
+ results_tasks = "<p>Progress of tasks per (active) workflow</p>"
+ header = "<th>Workflow</th>"
+ for status in settings.ALL_STATUSSES:
+ header += "<th>" + status + "</th>"
+ results_tasks += header + "<th>active</th><th>total</th>"
+
+ for workflow_result in workflow_results:
+
+ d = workflow_result['nr_of_tasks_per_status']
+ values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>"
+ for key in d:
+ percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
+ values += "<td>" + str(percentage) + "% ("+str(d[key])+")</td>"
+ #values += "<td>" + str(d[key]) + "</td>"
+
+ results_tasks += "<tr>" + values + "</tr>"
+
+ results_tasks = "<tbody>" + results_tasks + "</tbody>"
+
+ # --- logentries ---
+ results_logs = ""
+ for record in records:
+ line = "<tr><td><b>" + record['name'] + "</b></td>"\
+ '<td class="' + record['status'] + '" >' + record['status'] + \
+ "</td><td>" + str(record['cpu_cycles']) + \
+ "</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
+ results_logs += line
+
+ return results_tasks,results_logs
diff --git a/atdb/taskdatabase/tables.py b/atdb/taskdatabase/tables.py
index 1fd018322254fe77a43333b85e19dfb82dc0064d..619b5ce30f4e03ae202c0017b53ea874a92bf337 100644
--- a/atdb/taskdatabase/tables.py
+++ b/atdb/taskdatabase/tables.py
@@ -21,7 +21,7 @@ class TaskTable(tables.Table):
class Meta:
model = Task
template_name = "django_tables2/bootstrap4.html"
- fields = ("id", "workflow","filter","priority", "status","project","sas_id","creationtime", "size_to_process","buttons")
+ fields = ("id", "workflow","priority", "status","filter","project","sas_id","creationtime", "size_to_process","buttons")
# columns that need specific rendering
status = StatusColumn()
diff --git a/atdb/taskdatabase/templates/dashboard/dashboard.html b/atdb/taskdatabase/templates/dashboard/dashboard.html
new file mode 100644
index 0000000000000000000000000000000000000000..2d36ed2a9acc4bb01a16874d779f1f97e2906ce3
--- /dev/null
+++ b/atdb/taskdatabase/templates/dashboard/dashboard.html
@@ -0,0 +1,30 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+
+ <div class="card">
+ <div class="card-body">
+ <h4>Dashboard</h4>
+ <table class="table table-striped">
+ {{ results_tasks | safe }}
+ </table>
+
+ <table class="table table-striped">
+ <p>Resources used per step per active workflow</p>
+ <th>Workflow</th><th>Status</th><th>CPU cycles</th><th>wall clock time</th>
+ <tbody>
+ {{ results_logs | safe }}
+ </tbody>
+ </table>
+ </div>
+ </div>
+
+</div>
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal.html" %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/dashboard/details_card.html b/atdb/taskdatabase/templates/dashboard/details_card.html
deleted file mode 100644
index 392611ec2feca31849685692f1b266843d1082f7..0000000000000000000000000000000000000000
--- a/atdb/taskdatabase/templates/dashboard/details_card.html
+++ /dev/null
@@ -1,12 +0,0 @@
-
-<div class="card">
- <div class="card-body">
- <h4>Dashboard</h4>
- <table class="table table-striped">
- <th>Filter</th><th>Fraction Done</th><th>Resources Used</th>
- <tbody>
-
- </tbody>
- </table>
- </div>
-</div>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/dashboard/index.html b/atdb/taskdatabase/templates/dashboard/index.html
deleted file mode 100644
index 3af6eafa83cf99db3d0a61b49353a75d7fc62a97..0000000000000000000000000000000000000000
--- a/atdb/taskdatabase/templates/dashboard/index.html
+++ /dev/null
@@ -1,13 +0,0 @@
-{% extends 'taskdatabase/base.html' %}
-{% load static %}
-{% block myBlock %}
-
-<div class="container-fluid details-container">
- {% include "dashboard/details_card.html" %}
-
-</div>
-
-{% include "taskdatabase/modal/modal_script.html" %}
-{% include "taskdatabase/modal/modal.html" %}
-
-{% endblock %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index e2bfd220010237b1a9a4ef67316717ecee4c01e1..f14ca702105e2022f01b2ce9a9f2090fa3f0b025 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,11 +31,6 @@
WF
<a href="{% url 'sort-tasks' 'workflow' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
</th>
- <th>
- <a href="{% url 'sort-tasks' '-filter' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
- Filter
- <a href="{% url 'sort-tasks' 'filter' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
- </th>
<th>
<a href="{% url 'sort-tasks' '-priority' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
Priority
@@ -85,7 +80,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 1.0.0 (11 mar 2021 - 9:00)
+ <p class="footer"> Version 1.0.0 (12 mar 2021 - 15:00)
</div>
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks.html b/atdb/taskdatabase/templates/taskdatabase/tasks.html
index e5f55f515a68c204b61bc66b27f105bcb4ad5ed3..cb9f66c73aad5b6effc72a2ca9912bbdae857b5e 100644
--- a/atdb/taskdatabase/templates/taskdatabase/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks.html
@@ -14,7 +14,6 @@
target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow }}
</a></td>
</td>
- <td>{{ task.filter }}</td>
<td>
{% if user.is_authenticated %}
<a href="{% url 'task-change-priority' task.pk '-10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">-10</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html b/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
index 90a32040573ee19a6d7e0b34b17b181afa2d0d9c..45c0e2b4b61dd3007ea134bfda5b73c0267a5503 100644
--- a/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
@@ -40,7 +40,7 @@
</a>
</td>
</tr>
-
+ <tr><td><b>filter</b></td><td>{{ task.filter }}</td></tr>
<tr><td><b>project</b></td><td>{{ task.project }}</td></tr>
<tr><td><b>sas_id</b></td><td>{{ task.sas_id }}</td></tr>
<tr><td><b>creationTime</b></td><td>{{ task.creationTime|date:"Y-m-d H:i:s" }}</td></tr>
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index a5d6821698b68c10ea0434b22d0fb930b844068c..cc58b8d3689d6a9598ca611b3ab4b4931dbac499 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -20,10 +20,11 @@ urlpatterns = [
path('show-inputs/<int:id>/', views.ShowInputs, name='show-inputs'),
path('show-outputs/<int:id>/', views.ShowOutputs, name='show-outputs'),
path('show-metrics/<int:id>/', views.ShowMetrics, name='show-metrics'),
+ path('dashboard/', views.ShowDashboard, name='dashboard'),
path('workflow_details/<id>/', views.WorkflowDetails, name='workflow-details'),
path('query/', views.QueryView.as_view(), name='query'),
- path('dashboard/', views.DashboardView.as_view(), name='dashboard'),
+ # path('dashboard/', views.DashboardView.as_view(), name='dashboard'),
path('config/', views.ShowConfig, name='config'),
path('diagram/', views.DiagramView.as_view(), name='diagram'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 7860da87eaa53e1e2cc51cf0f73471a337e76fc6..187daffec835219610e15993cdf93bfab69118e7 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -178,62 +178,19 @@ def get_searched_tasks(search, sort):
Q(id__contains=search) |
Q(sas_id__contains=search) |
Q(creationTime__icontains=search) |
+ Q(filter__icontains=search) |
Q(status__icontains=search) |
Q(status__in=search) |
Q(project__icontains=search)).order_by(sort)
return tasks
-class DashboardView(ListView):
- template_name = 'dashboard/index.html'
- model = Task
- queryset = Task.objects.all()
- # by default this returns the list in an object called object_list, so use 'object_list' in the html page.
- # but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
- context_object_name = 'my_tasks'
-
-
-class ChartsView(ListView):
- template_name = 'dashboard/charts.html'
- model = Task
- queryset = Task.objects.all()
- # by default this returns the list in an object called object_list, so use 'object_list' in the html page.
- # but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
- context_object_name = 'my_tasks'
-
-
class TaskTables2View(SingleTableView):
model = Task
table_class = TaskTable
template_name = 'query/tables2.html'
-def convert_logentries_to_html(log_entries):
- results = ""
-
- try:
- results += "<th>step</th><th>status</th><th>timestamp</th><th>cpu_cycles</th><th>wall_clock_time</th><th>logfile</th>"
- results += "<tbody>"
- for log in log_entries:
- line = "<tr><td><b>" + log.step_name + '</b></td>'
- line +='<td class="' + log.status + '" >' + log.status + "</td>"
- line += "<td>" + str(log.timestamp.strftime("%m-%d-%Y, %H:%M:%S")) + "</td>"
- line += "<td>" + str(log.cpu_cycles) + "</td>"
- line += "<td>" + str(log.wall_clock_time) + "</td>"
- if log.url_to_log_file!=None:
- link = "<a href=" + '"' + str(log.url_to_log_file) + '" target="_blank">' + "logfile" + "</a>"
- else:
- link = "-"
- line += "<td>" + link + "</td>"
- results += line
-
- results += "</tbody>"
- except:
- results = "<tr><td>no data</td></tr>"
-
- return results
-
-
def TaskDetails(request, id=0, page=0):
try:
@@ -256,36 +213,18 @@ def TaskDetails(request, id=0, page=0):
request.session['page'] = page
log_entries = LogEntry.objects.filter(task=task).order_by('-timestamp')
- logentries_html = convert_logentries_to_html(log_entries)
+ logentries_html = algorithms.convert_logentries_to_html(log_entries)
return render(request, "taskdatabase/tasks/task_details.html", {'task': task, 'logentries': logentries_html })
-def convert_list_of_dicts_to_html(my_list):
- results = ""
- try:
- for my_dict in my_list:
- # iterate through the dict of key/values
- for key,value in my_dict.items():
- try:
- if "://" in value:
- link = "<a href=" + '"' + value + '">' + key +"</a>"
- value = link
- except:
- pass
- line = "<tr><td><b>" + str(key) + "</b></td><td>" + str(value) + "</td></tr>"
- results = results + line
- except:
- results = "<tr><td>no data</td></tr>"
-
- return results
def ShowInputs(request, id):
task = Task.objects.get(id=id)
# convert the json to a presentable piece of html for the output template
- results = convert_list_of_dicts_to_html(task.inputs)
+ results = algorithms.convert_list_of_dicts_to_html(task.inputs)
return render(request, "taskdatabase/details/inputs.html", {'results': results})
@@ -293,7 +232,7 @@ def ShowOutputs(request, id):
task = Task.objects.get(id=id)
# convert the json to a presentable piece of html for the output template
- results = convert_list_of_dicts_to_html(task.outputs)
+ results = algorithms.convert_list_of_dicts_to_html(task.outputs)
return render(request, "taskdatabase/details/outputs.html", {'results': results})
@@ -301,38 +240,23 @@ def ShowMetrics(request, id):
task = Task.objects.get(id=id)
# convert the json to a presentable piece of html for the output template
- results = convert_list_of_dicts_to_html(task.metrics)
+ results = algorithms.convert_list_of_dicts_to_html(task.metrics)
return render(request, "taskdatabase/details/metrics.html", {'results': results})
-def convert_config_to_html(querylist):
- results = ""
- try:
- for record in querylist:
- # iterate through the dict of key/values
- key = record.key
- value = record.value
- try:
- if "://" in value:
- link = "<a href=" + '"' + value + '">' + key +"</a>"
- value = link
- except:
- pass
- line = "<tr><td><b>" + str(key) + "</b></td><td>" + str(value) + "</td></tr>"
- results = results + line
- except:
- results = "<tr><td>no data</td></tr>"
-
- return results
-
-
def ShowConfig(request):
configuration = Configuration.objects.all()
- results = convert_config_to_html(configuration)
+ results = algorithms.convert_config_to_html(configuration)
return render(request, "taskdatabase/config.html", {'results': results})
+def ShowDashboard(request):
+ # gather the results
+ results_tasks,results_logs = algorithms.convert_aggregation_to_html()
+ return render(request, "dashboard/dashboard.html", {'results_tasks': results_tasks, 'results_logs': results_logs})
+
+
def WorkflowDetails(request, id):
workflow = Workflow.objects.get(id=id)
return render(request, "taskdatabase/details/workflow_details.html", {'workflow': workflow})
@@ -533,7 +457,7 @@ class GetSizeView(generics.ListAPIView):
print(status_list)
except:
# if no 'status__in=' is given, then use the default list
- status_list = settings.ACTIVE_STATUSSES
+ status_list = settings.STATUSSES_WITH_DATA
size = algorithms.get_size(status_list)