From be75a96078b7e740cbf2ff40e2d78b1d0d9c2d8b Mon Sep 17 00:00:00 2001
From: Nico Vermaas <vermaas@astron.nl>
Date: Thu, 3 Feb 2022 13:20:12 +0100
Subject: [PATCH] initial quality screen
---
atdb/taskdatabase/models.py | 9 ++
atdb/taskdatabase/services/algorithms.py | 31 ++++++
.../templates/taskdatabase/base.html | 9 +-
.../templates/taskdatabase/index.html | 2 +-
.../taskdatabase/postprocessing.html | 91 ++++++++++++++++++
.../templates/taskdatabase/tasks.html | 6 +-
.../taskdatabase/tasks/details_card.html | 2 +-
.../taskdatabase/tasks/quality_card.html | 8 ++
.../taskdatabase/tasks/task_quality.html | 19 ++++
atdb/taskdatabase/urls.py | 5 +-
atdb/taskdatabase/views.py | 94 +++++++++++++++++++
11 files changed, 268 insertions(+), 8 deletions(-)
create mode 100644 atdb/taskdatabase/templates/taskdatabase/postprocessing.html
create mode 100644 atdb/taskdatabase/templates/taskdatabase/tasks/quality_card.html
create mode 100644 atdb/taskdatabase/templates/taskdatabase/tasks/task_quality.html
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index f5e21fc6..3ce0ad35 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -69,6 +69,15 @@ class Task(models.Model):
except:
return "no_predecessor"
+ @property
+ def has_quality(self):
+ # todo: check if there is a 'quality' structure in the 'task.outputs'
+ try:
+ return True
+ except:
+ return False
+
+
class LogEntry(models.Model):
cpu_cycles = models.IntegerField(null=True,blank=True)
wall_clock_time = models.IntegerField(null=True,blank=True)
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 4b9c9d48..44a41a76 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -106,6 +106,37 @@ def convert_logentries_to_html(log_entries):
return results
+def convert_quality_to_html(task):
+ results = ""
+
+ try:
+ outputs = task.outputs
+ print(outputs[0])
+ quality = task.outputs[0]['quality']
+ print(quality)
+
+ results = ""
+
+ results += "<tr><td><b>SAS_ID</b></td><td>" + str(task.sas_id) + "</td></tr>"
+ results += "<tr><td><b>Project</b></td><td>" + str(task.project) + "</td></tr>"
+ results += "<tr><td><b>ATDB Filter</b></td><td>" + str(task.filter) + "</td></tr>"
+ results += "<tr><td><b>Input Type</b></td><td>" + "???" + "</td></tr>"
+ results += "<tr><td><b>NrJobs (MSs)</b></td><td>" + "???" + "</td></tr>"
+ results += "<tr><td><b>QA uv-coverage</b></td><td>" + str(quality['uv-coverage']) + "</td></tr>"
+ results += "<tr><td><b>QA sensitivity</b></td><td>" + str(quality['sensitivity']) + "</td></tr>"
+ results += "<tr><td><b>QA observing-conditions</b></td><td>" + str(quality['observing-conditions']) + "</td></tr>"
+ results += "<tr><td><b>QA diagnostic plots</b></td><td>" + "(link)" + "</td></tr>"
+ results += "<tr><td><b>Workflow summary parset</b></td><td>" + "(link)" + "</td></tr>"
+ results += "<tr><td><b>Summary logs</b></td><td>" + "(link)" + "</td></tr>"
+ results += "<tr><td><b>QA summary.hf5</b></td><td>" + "(link)" + "</td></tr>"
+
+ except Exception as err:
+ results = "<tr><td>" + str(err) + "</td></tr>"
+ # results = "<tr><td>no data</td></tr>"
+
+ return results
+
+
def convert_list_of_dicts_to_html(my_blob):
results = ""
my_list = []
diff --git a/atdb/taskdatabase/templates/taskdatabase/base.html b/atdb/taskdatabase/templates/taskdatabase/base.html
index fbeea95e..2bc5bbf8 100644
--- a/atdb/taskdatabase/templates/taskdatabase/base.html
+++ b/atdb/taskdatabase/templates/taskdatabase/base.html
@@ -37,19 +37,20 @@
<div class="container-fluid">
<ul class="nav navbar-nav">
<!-- Header -->
- <li><a class="navbar-brand" href="{% url 'homepage' %}">
+ <li><a class="navbar-brand" href="{% url 'index' %}">
<img src="{% static 'taskdatabase/new_ldv_logo.png' %}" height="30" alt="">
ATDB</a>
</li>
- <li><a class="nav-link" href="{% url 'homepage' %}">Tasks</a></li>
+ <li><a class="nav-link" href="{% url 'index' %}">Regular Tasks</a></li>
+ <li><a class="nav-link" href="{% url 'postprocessing-tasks' %}">PostProcessing</a></li>
{% if user.is_authenticated %}
- <li><a class="nav-link" href="{% url 'task-details'%}">Details</a></li>
+ <li><a class="nav-link" href="{% url 'task-details'%}">Task Details</a></li>
{% endif %}
<li><a class="nav-link" href="{% url 'dashboard' 'active_nores' %}">Dashboard</a></li>
- <li><a class="nav-link" href="{% url 'query' %}">Query</a></li>
+ <li><a class="nav-link" href="{% url 'query' %}">Filter</a></li>
<li><a class="nav-link" href="{% url 'monitoring' %}">Monitoring</a></li>
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index cb955d37..276050d3 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -80,7 +80,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 1.0.0 (1 feb 2021 - 14:00)
+ <p class="footer"> Version 1.0.0 (3 feb 2021 - 8:00)
</div>
diff --git a/atdb/taskdatabase/templates/taskdatabase/postprocessing.html b/atdb/taskdatabase/templates/taskdatabase/postprocessing.html
new file mode 100644
index 00000000..276050d3
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/postprocessing.html
@@ -0,0 +1,91 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+ {% include 'taskdatabase/filter/filter_buttons.html' %}
+
+ <div class="row">
+ <div class="col-sm-12 col-md-12 col-lg-12">
+ {% include 'taskdatabase/pagination.html' %}
+
+ {% if my_tasks %}
+ <div class="panel panel-success">
+
+ <div class="panel-body">
+
+ <table class="table table-striped table-bordered table-sm">
+
+ <thead>
+ <tr>
+ <th>
+ <a href="{% url 'sort-tasks' '-pk' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ ID
+ <a href="{% url 'sort-tasks' 'id' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>Details</th>
+ <th>
+ <a href="{% url 'sort-tasks' '-workflow' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Workflow
+ <a href="{% url 'sort-tasks' 'workflow' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-priority' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Priority
+ <a href="{% url 'sort-tasks' 'priority' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+
+ <th>
+ <a href="{% url 'sort-tasks' '-status' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Status
+ <a href="{% url 'sort-tasks' 'status' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-project' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Project
+ <a href="{% url 'sort-tasks' 'project' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-sas_id' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ SAS_ID
+ <a href="{% url 'sort-tasks' 'sas_id' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-creationTime' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ CreationTime
+ <a href="{% url 'sort-tasks' 'creationTime' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+
+ <th>
+ <a href="{% url 'sort-tasks' '-size_to_process' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Size
+ <a href="{% url 'sort-tasks' 'size_to_process' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+
+ <th>Actions</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% include 'taskdatabase/tasks.html' %}
+ </tbody>
+ </table>
+ {% else %}
+ <p>No recent Tasks.</p>
+ {% endif %}
+
+ </div>
+ </div>
+ {% include 'taskdatabase/pagination.html' %}
+ </div>
+ </div>
+ <p class="footer"> Version 1.0.0 (3 feb 2021 - 8:00)
+
+</div>
+
+{% include 'taskdatabase/localstorage_scripts.html' %}
+
+
+{% endblock %}
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks.html b/atdb/taskdatabase/templates/taskdatabase/tasks.html
index a4bcecb7..b6675ada 100644
--- a/atdb/taskdatabase/templates/taskdatabase/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks.html
@@ -14,7 +14,11 @@
data-popup-url="{% url 'task-details' task.id my_tasks.number %}"
><i class="fas fa-list"></i> Details
</a>
-
+ <a class="btn btn-primary btn-sm"
+ href="{% url 'task-quality' task.id my_tasks.number %}"
+ data-popup-url="{% url 'task-quality' task.id my_tasks.number %}"
+ ><i class="fas fa-balance-scale-right"></i> Quality
+ </a>
{% endif %}
</td>
<td>
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html b/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
index 78907480..27297632 100644
--- a/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
@@ -1,7 +1,7 @@
<div class="card">
<div class="card-body">
- <h4>Task: {{ task.id }}</h4>
+ <h4>Task: {{ task.id }} ({{ task.task_type }})</h4>
<table class="table table-striped">
<tbody>
<tr><td><b>id</b></td><td><a href="{% url 'task-detail-view-api' task.pk %}" target="_blank">{{ task.id }} </a></td></tr>
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/quality_card.html b/atdb/taskdatabase/templates/taskdatabase/tasks/quality_card.html
new file mode 100644
index 00000000..4b332179
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks/quality_card.html
@@ -0,0 +1,8 @@
+<div class="card">
+ <div class="card-body">
+ <h4>Quality</h4>
+ <table class="table table-striped">
+ {{ quality | safe }}
+ </table>
+ </div>
+</div>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/task_quality.html b/atdb/taskdatabase/templates/taskdatabase/tasks/task_quality.html
new file mode 100644
index 00000000..b54445d7
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/tasks/task_quality.html
@@ -0,0 +1,19 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+ <div class="row">
+
+ <div class="col-md">
+ {% include "taskdatabase/tasks/quality_card.html" %}
+ </div>
+ </div>
+</div>
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal.html" %}
+
+{% endblock %}
+
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 3fa60009..583c8387 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -12,10 +12,13 @@ urlpatterns = [
path('login/', auth_views.LoginView.as_view(template_name='registration/login.html')),
# --- GUI ---
- path('', views.IndexView.as_view(), name='homepage'),
+ path('', views.IndexView.as_view(), name='index'),
+ path('postprocessing-tasks', views.PostProcessingTasksView.as_view(), name='postprocessing-tasks'),
path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
path('task_details/', views.TaskDetails, name='task-details'),
+ path('task_quality/<int:id>/<page>', views.TaskQuality, name='task-quality'),
+ path('task_quality/', views.TaskQuality, name='task-quality'),
path('show-inputs/<int:id>/', views.ShowInputs, name='show-inputs'),
path('show-outputs/<int:id>/', views.ShowOutputs, name='show-outputs'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index bdc133ad..6e6cba77 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -246,6 +246,74 @@ class IndexView(ListView):
return tasks
+class PostProcessingTasksView(ListView):
+ """
+ This is the main view of ATDB. It shows a pagination list of tasks, sorted by creationTime.
+ """
+ template_name = 'taskdatabase/postprocessing.html'
+
+ # by default this returns the list in an object called object_list, so use 'object_list' in the html page.
+ # but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
+ context_object_name = 'my_tasks'
+
+ def get_queryset(self):
+ status = self.request.GET.get('status')
+ search_box = self.request.GET.get('search_box', None)
+
+ # get the sort variable from the session or use default
+ try:
+ sort = self.request.session['sort']
+ except:
+ sort = '-creationTime'
+
+ tasks = Task.objects.order_by(sort)
+
+ # check if there is a 'task_filter' put on the session
+ try:
+ filter = self.request.session['task_filter']
+ if filter != 'all':
+ tasks = get_searched_tasks(filter, sort)
+ except:
+ pass
+
+ # check if there is a 'task_onhold_filter' put on the session
+ try:
+ onhold = self.request.session['task_onhold_filter']
+ if onhold != None:
+ tasks = tasks.filter(resume=not onhold)
+
+ except:
+ pass
+
+ if (search_box is not None):
+ tasks = get_searched_tasks(search_box, sort)
+
+ # only return the 'regular' tasks, and not the 'postprocessing' tasks in the GUI
+ tasks = tasks.filter(task_type='postprocessing')
+
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ page = self.request.GET.get('page')
+
+ try:
+ # check if there was a page on the session, if so, use it.
+ if page == None:
+ page = self.request.session['page']
+ self.request.session['page'] = None
+ except:
+ pass
+
+ try:
+ tasks = paginator.page(page)
+ except PageNotAnInteger:
+ # If page is not an integer, deliver first page.
+ tasks = paginator.page(1)
+ except EmptyPage:
+ # If page is out of range (e.g. 9999), deliver last page of results.
+ tasks = paginator.page(paginator.num_pages)
+
+ return tasks
+
+
def get_searched_tasks(search, sort):
tasks = Task.objects.filter(
Q(id__contains=search) |
@@ -290,6 +358,32 @@ def TaskDetails(request, id=0, page=0):
return render(request, "taskdatabase/tasks/task_details.html", {'task': task, 'logentries': logentries_html})
+def TaskQuality(request, id=0, page=0):
+ try:
+ task = Task.objects.get(id=id)
+
+ # store the requested task_id on the session
+ request.session['task_id'] = task.id
+
+ except:
+ # when an invalid id is given, like '/atdb/task_details/0/',
+ # then look if there is a task stored on the session
+ try:
+ task_on_session = request.session['task_id']
+ task = Task.objects.get(id=task_on_session)
+ except:
+ messages.add_message(request, messages.WARNING, 'no task selected.')
+ return redirect('homepage')
+
+ # store the current page on the session
+ request.session['page'] = page
+
+ quality_html = algorithms.convert_quality_to_html(task)
+
+ return render(request, "taskdatabase/tasks/task_quality.html", {'task': task, 'quality': quality_html})
+
+
+
def ShowInputs(request, id):
task = Task.objects.get(id=id)
--
GitLab