diff --git a/atdb/atdb/urls.py b/atdb/atdb/urls.py
index 03a153f3ff5cc2eab177f4a513d23d425f0d741d..b5e26030204398c859780fb37edeeff0d684c834 100644
--- a/atdb/atdb/urls.py
+++ b/atdb/atdb/urls.py
@@ -1,18 +1,4 @@
-"""atdb URL Configuration
-The `urlpatterns` list routes URLs to views. For more information please see:
- https://docs.djangoproject.com/en/2.0/topics/http/urls/
-Examples:
-Function views
- 1. Add an import: from my_app import views
- 2. Add a URL to urlpatterns: path('', views.home, name='home')
-Class-based views
- 1. Add an import: from other_app.views import Home
- 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
-Including another URLconf
- 1. Import the include() function: from django.urls import include, path
- 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
-"""
from django.contrib import admin
from django.urls import include, path
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index c689ae3d01d4fd7b2b8e091883da427e22b7b768..262b0735d38d1e13b020fc3664132bdc4c1efb36 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -104,55 +104,14 @@ def convert_config_to_html(querylist):
return results
-
-def aggregate_resources_logs():
-
- records = []
-
- # get all active tasks
- active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
- active_tasks_count = active_tasks.count()
-
- # retrieve all unique workflows
- active_workflows = active_tasks.values('workflow').distinct()
-
- # iterate through the filters and accumulate logentries
- for w in active_workflows:
- workflow_result = {}
-
- # extract the workflow object (cheap)
- workflow = Workflow.objects.get(id = w['workflow'])
-
- # aggregate logentries per step for all active statusses
- for status in settings.ACTIVE_STATUSSES:
- record = {}
- record['name'] = str(workflow.id) +' - '+ workflow.workflow_uri
- record['status'] = status
-
- # aggregate logentries per step for all active statusses (expensive)
- logs = LogEntry.objects.filter(status=status)\
- .filter(task__status__in=settings.ACTIVE_STATUSSES)\
- .filter(task__workflow=workflow)
-
- sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
- record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
-
- wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
- record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
-
- records.append(record)
-
- return records
-
-
+# aggregate information from the tasks table per workflow per status
def aggregate_resources_tasks():
workflow_results = []
- records = []
# get all active tasks
active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
- active_tasks_count = active_tasks.count()
+ # active_tasks_count = active_tasks.count()
# retrieve all unique workflows
active_workflows = active_tasks.values('workflow').distinct()
@@ -170,6 +129,12 @@ def aggregate_resources_tasks():
tasks_per_workflow = Task.objects.filter(workflow=workflow)
nr_of_tasks_per_workflow = tasks_per_workflow.count()
+ sum_size_to_process = tasks_per_workflow.aggregate(Sum('size_to_process'))
+ workflow_result['size_to_process'] = sum_size_to_process['size_to_process__sum']
+
+ sum_size_processed = tasks_per_workflow.aggregate(Sum('size_processed'))
+ workflow_result['size_processed'] = sum_size_processed['size_processed__sum']
+
# all the active tasks
active_tasks_per_workflow = tasks_per_workflow.filter(status__in=settings.ACTIVE_STATUSSES)
nr_of_active_tasks_per_workflow = active_tasks_per_workflow.count()
@@ -198,6 +163,47 @@ def aggregate_resources_tasks():
return workflow_results
+# aggregate information from the logentries table per workflow per status
+def aggregate_resources_logs():
+
+ records = []
+
+ # get all active tasks
+ active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES)
+ active_tasks_count = active_tasks.count()
+
+ # retrieve all unique workflows
+ active_workflows = active_tasks.values('workflow').distinct()
+
+ # iterate through the filters and accumulate logentries
+ for w in active_workflows:
+ workflow_result = {}
+
+ # extract the workflow object (cheap)
+ workflow = Workflow.objects.get(id = w['workflow'])
+
+ # aggregate logentries per step for all active statusses
+ for status in settings.ACTIVE_STATUSSES:
+ record = {}
+ record['name'] = str(workflow.id) +' - '+ workflow.workflow_uri
+ record['status'] = status
+
+ # aggregate logentries per step for all active statusses (expensive)
+ logs = LogEntry.objects.filter(status=status)\
+ .filter(task__status__in=settings.ACTIVE_STATUSSES)\
+ .filter(task__workflow=workflow)
+
+ sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
+ record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
+
+ wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
+ record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
+
+ records.append(record)
+
+ return records
+
+
def construct_link(request, status, workflow_id, count):
link = str(count)
try:
@@ -205,26 +211,43 @@ def construct_link(request, status, workflow_id, count):
if settings.DEV == True:
url = request.build_absolute_uri('/atdb/tasks') + query
else:
+ # Unclear why 'build_absolute_uri' doesn't return 'https' in production.
+ # Probably because the https is handled fully outside the container by Traefik
+ # and ATDB is not aware of that.
+
url = "https://" + request.get_host() + '/atdb/tasks' + query
link = '<a href="' + url + '" target="_blank">' + str(count) + "</a>"
except:
pass
return link
-def convert_aggregation_to_html(request):
+
+def human_readable(size_in_bytes):
+ for count in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
+ if size_in_bytes > -1024.0 and size_in_bytes < 1024.0:
+ return "%3.1f %s" % (size_in_bytes, count)
+ size_in_bytes /= 1024.0
+ return "%3.1f %s" % (size_in_bytes, 'PB')
+
+
+def construct_tasks_per_workflow_html(request, workflow_results):
# --- Progress of tasks per active workflow ---
workflow_results = aggregate_resources_tasks()
+
results_tasks = "<p>Progress of tasks per (active) workflow</p>"
header = "<th>Workflow</th>"
+
for status in settings.ALL_STATUSSES:
header += "<th>" + status + "</th>"
- results_tasks += header + '<th class="active">active</th><th>total</th>'
+ results_tasks += header + '<th class="active">active</th><th>total</th><th>to process</th><th>processed</th>'
for workflow_result in workflow_results:
d = workflow_result['nr_of_tasks_per_status']
- values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>"
+ #values = "<td><b>" + str(workflow_result['id'])+" - "+workflow_result['name'] + "</b></td>"
+ values = "<td><b>" + str(workflow_result['id']) + "</b></td>"
+
for key in d:
percentage = round(int(d[key]) / int(workflow_result['nr_of_tasks']) * 100)
@@ -240,13 +263,21 @@ def convert_aggregation_to_html(request):
#values += "<td class="+style+">" + str(percentage) + "% ("+str(d[key])+")</td>"
#values += "<td>" + str(d[key]) + "</td>"
+ # add sizes
+ values += "<td>" + str(human_readable(workflow_result['size_to_process'])) + "</td>"
+ try:
+ percentage = round(int(workflow_result['size_processed']) / int(workflow_result['size_to_process']) * 100)
+ except:
+ percentage = 0
+ values += "<td>" + str(human_readable(workflow_result['size_processed'])) + " ("+ str(percentage) + "%) </td>"
+
results_tasks += "<tr>" + values + "</tr>"
results_tasks = "<tbody>" + results_tasks + "</tbody>"
+ return results_tasks
- # --- logentries ---
- log_records = aggregate_resources_logs()
+def construct_logs_per_workflow_html(log_records):
results_logs = ""
for record in log_records:
@@ -255,11 +286,24 @@ def convert_aggregation_to_html(request):
if record['status'] in settings.ACTIVE_STATUSSES:
style = "active"
- line = "<tr><td><b>" + record['name'] + "</b></td>"\
- '<td class="' + style + '" >' + record['status'] + \
+ line = "<tr><td><b>" + record['name'] + "</b></td>" \
+ '<td class="' + style + '" >' + record['status'] + \
"</td><td>" + str(record['cpu_cycles']) + \
"</td><td>" + str(record['wall_clock_time']) + "</td><tr>"
results_logs += line
+ return results_logs
+
+
+def construct_dashboard_html(request):
+
+ # --- Progress of tasks per active workflow ---
+ workflow_results = aggregate_resources_tasks()
+ results_tasks = construct_tasks_per_workflow_html(request, workflow_results)
+
+ # --- logentries ---
+ log_records = aggregate_resources_logs()
+ results_logs = construct_logs_per_workflow_html(log_records)
+
return results_tasks,results_logs
diff --git a/atdb/taskdatabase/tables.py b/atdb/taskdatabase/tables.py
index 107a4e7d9a6a4cb5ec62b6e63688d2a47f6a3713..44e62816601647fe0a8f0ca6ae58fca5d650f640 100644
--- a/atdb/taskdatabase/tables.py
+++ b/atdb/taskdatabase/tables.py
@@ -21,11 +21,11 @@ class TaskTable(tables.Table):
class Meta:
model = Task
template_name = "django_tables2/bootstrap4.html"
- fields = ("id", "workflow","priority", "status","filter","project","sas_id","creationtime", "size_to_process","buttons")
+ fields = ("id", "workflow","priority", "status","project","sas_id","buttons")
# columns that need specific rendering
status = StatusColumn()
- creationtime = tables.Column(verbose_name='CreationTime')
+ #creationtime = tables.Column(verbose_name='CreationTime')
buttons = tables.TemplateColumn(verbose_name='Set Status',
template_name='query/status_buttons_per_row.html',
diff --git a/atdb/taskdatabase/templates/query/set_multi_status_panel.html b/atdb/taskdatabase/templates/query/set_multi_status_panel.html
index 78c5cad9b0304607f52b777838eba92fd8a6b369..008b331d9250704db4ad79ad53ea3cdde7ad0f58 100644
--- a/atdb/taskdatabase/templates/query/set_multi_status_panel.html
+++ b/atdb/taskdatabase/templates/query/set_multi_status_panel.html
@@ -8,6 +8,7 @@
<p>Click to change Status for all these tasks</p>
<table>
<tr>
+ <a href="{% url 'task-setstatus' 1 'defining' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> defining</a>
<a href="{% url 'task-setstatus' 1 'staged' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> staged</a>
<a href="{% url 'task-setstatus' 1 'processed' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> processed</a>
<a href="{% url 'task-setstatus' 1 'validated' '1' %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> validated</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/base.html b/atdb/taskdatabase/templates/taskdatabase/base.html
index fa4f3a27953927503e617ff5b3e529217fbe6154..a81c0c385aeebc857eef181887f8c35b2be3666e 100644
--- a/atdb/taskdatabase/templates/taskdatabase/base.html
+++ b/atdb/taskdatabase/templates/taskdatabase/base.html
@@ -64,8 +64,6 @@
{% endif %}
</div>
-
-
</nav>
{% for message in messages %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index f17bad36383adef65b35f8106db37e6d2f02baf1..9fc89e2f992bbc1f30b17d418e875719fc0232ff 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -80,7 +80,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 1.0.0 (12 mar 2021 - 21:00)
+ <p class="footer"> Version 1.0.0 (15 mar 2021 - 13:00)
</div>
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 568e2364ca801d75a13ace0bbbd3955c356ea665..f2143ac3c0eb5b5395bb97d6b737128d129fbad1 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -67,8 +67,6 @@ class TaskFilterQueryPage(filters.FilterSet):
'status': ['icontains', 'in'],
'project': ['exact', 'icontains'],
'sas_id': ['exact', 'icontains'],
- 'creationTime': ['icontains'],
- 'size_to_process' : ['lte', 'gte'],
}
@@ -137,6 +135,7 @@ class IndexView(ListView):
tasks = Task.objects.order_by(sort)
+ # check if there is a 'task_filter' put on the session
try:
filter = self.request.session['task_filter']
if filter!='all':
@@ -253,7 +252,7 @@ def ShowConfig(request):
def ShowDashboard(request):
# gather the results
- results_tasks,results_logs = algorithms.convert_aggregation_to_html(request)
+ results_tasks,results_logs = algorithms.construct_dashboard_html(request)
return render(request, "dashboard/dashboard.html", {'results_tasks': results_tasks, 'results_logs': results_logs})