diff --git a/atdb/atdb/static/taskdatabase/ATDB-LDV Workflow Diagram.png b/atdb/atdb/static/taskdatabase/ATDB-LDV Workflow Diagram.png
index 9640689bf19cd81dd6a93813f02dbad2f433ce35..32eb9a99e93f9fc7f7310005c55f7f21e14c4540 100644
Binary files a/atdb/atdb/static/taskdatabase/ATDB-LDV Workflow Diagram.png and b/atdb/atdb/static/taskdatabase/ATDB-LDV Workflow Diagram.png differ
diff --git a/atdb/docs/ATDB Context Diagram.png b/atdb/docs/ATDB Context Diagram.png
deleted file mode 100644
index 624425c3f79d6c9b909c728e61b6e687d11a1615..0000000000000000000000000000000000000000
Binary files a/atdb/docs/ATDB Context Diagram.png and /dev/null differ
diff --git a/atdb/docs/ATDB-LDV - Deployment Diagram.png b/atdb/docs/ATDB-LDV - Deployment Diagram.png
index 88d60db1f98c2ccbc2b5be45ef5b2d4609e0c6d2..94370d9e9bffe85dee25c32c1f46632f28af68ac 100644
Binary files a/atdb/docs/ATDB-LDV - Deployment Diagram.png and b/atdb/docs/ATDB-LDV - Deployment Diagram.png differ
diff --git a/atdb/docs/ATDB-LDV Workflow Diagram.png b/atdb/docs/ATDB-LDV Workflow Diagram.png
index fc44c244a6f9910638a23d0ef3a5fb25551ad835..2ef5b467bffbb3b10197899e0bcbbf8ffa66c6dc 100644
Binary files a/atdb/docs/ATDB-LDV Workflow Diagram.png and b/atdb/docs/ATDB-LDV Workflow Diagram.png differ
diff --git a/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png b/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png
index 9640689bf19cd81dd6a93813f02dbad2f433ce35..32eb9a99e93f9fc7f7310005c55f7f21e14c4540 100644
Binary files a/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png and b/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png differ
diff --git a/atdb/taskdatabase/templates/taskdatabase/base.html b/atdb/taskdatabase/templates/taskdatabase/base.html
index cfe0a241ee5ba90e96da266f7857fa4bd96f15c8..3dc3cc38cb478b906fe58570446e0cfe7f817076 100644
--- a/atdb/taskdatabase/templates/taskdatabase/base.html
+++ b/atdb/taskdatabase/templates/taskdatabase/base.html
@@ -49,7 +49,6 @@
{% if user.is_authenticated %}
<li><a class="nav-link" href="{% url 'task-details'%}">Task Details</a></li>
- <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
{% endif %}
<li><a class="nav-link" href="{% url 'dashboard' 'active_nores_nofilter' %}">Dashboard</a></li>
@@ -58,6 +57,11 @@
{% comment %}
<li><a class="nav-link" href="{% url 'query2' request|current_query_params %}">Filter2</a></li>
{% endcomment %}
+
+ <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
+ <li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
+ <li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
+
<li><a class="nav-link" href="{% url 'monitoring' %}">Monitoring</a></li>
<li><a class="nav-link" href="{% url 'diagram' %}">Diagram</a></li>
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/failures_page.html b/atdb/taskdatabase/templates/taskdatabase/failures/failures_page.html
new file mode 100644
index 0000000000000000000000000000000000000000..6f67507a5f61f7580aa9088f0217dec20ea09d5b
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/failures_page.html
@@ -0,0 +1,52 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+ <div class="card">
+ <div class="card-body">
+ <div class="row">
+ <div class="col-8">
+ <h3>Failures</h3>
+ These are all the tasks that failed.
+ Click 'Retry' to restart a specific step in the workflow.
+ <a class="nav-link" href="{% url 'diagram' %}">See workflow diagram</a>
+ <hr>
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-sm-12 col-md-12 col-lg-12">
+ {% include 'taskdatabase/pagination.html' %}
+
+ {% if my_tasks %}
+ <div class="panel panel-success">
+
+ <div class="panel-body">
+
+ <table class="table table-striped table-bordered table-sm">
+ <thead>
+ {% include 'taskdatabase/failures/tasks_failures_headers.html' %}
+ </thead>
+ <tbody>
+ {% include 'taskdatabase/failures/tasks_failures.html' %}
+ </tbody>
+ </table>
+ {% else %}
+ <p>No recent Tasks.</p>
+ {% endif %}
+
+ </div>
+ </div>
+ {% include 'taskdatabase/pagination.html' %}
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
+
+{% include 'taskdatabase/localstorage_scripts.html' %}
+
+
+{% endblock %}
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks_quality.html b/atdb/taskdatabase/templates/taskdatabase/failures/tasks_failures.html
similarity index 100%
rename from atdb/taskdatabase/templates/taskdatabase/tasks_quality.html
rename to atdb/taskdatabase/templates/taskdatabase/failures/tasks_failures.html
diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks_quality_headers.html b/atdb/taskdatabase/templates/taskdatabase/failures/tasks_failures_headers.html
similarity index 100%
rename from atdb/taskdatabase/templates/taskdatabase/tasks_quality_headers.html
rename to atdb/taskdatabase/templates/taskdatabase/failures/tasks_failures_headers.html
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index 096baa9294dcdeca33762e90bf41de489a39495a..3870b8fc9b36dba2ed67e42a01b4c42e9c6dbfd1 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -34,7 +34,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 1.0.0 (8 mar 2021 - 17:00)
+ <p class="footer"> Version 1.0.0 (1 apr 2021 - 08:00)
</div>
diff --git a/atdb/taskdatabase/templates/taskdatabase/quality_page.html b/atdb/taskdatabase/templates/taskdatabase/quality/quality_page.html
similarity index 80%
rename from atdb/taskdatabase/templates/taskdatabase/quality_page.html
rename to atdb/taskdatabase/templates/taskdatabase/quality/quality_page.html
index f837b06804ea32d38242d10b5d7b9f7c887b2743..e8511c0ba6645f16b8a57d6bc6799dd31b06ba8d 100644
--- a/atdb/taskdatabase/templates/taskdatabase/quality_page.html
+++ b/atdb/taskdatabase/templates/taskdatabase/quality/quality_page.html
@@ -9,6 +9,8 @@
<div class="row">
<div class="col-8">
<h3>Quality Diagnostics</h3>
+ This overview shows the Quality Diagnostics of all the task that have this information'.
+ <hr>
</div>
</div>
<div class="row">
@@ -22,10 +24,10 @@
<table class="table table-striped table-bordered table-sm">
<thead>
- {% include 'taskdatabase/tasks_quality_headers.html' %}
+ {% include 'taskdatabase/quality/tasks_quality_headers.html' %}
</thead>
<tbody>
- {% include 'taskdatabase/tasks_quality.html' %}
+ {% include 'taskdatabase/quality/tasks_quality.html' %}
</tbody>
</table>
{% else %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/quality/tasks_quality.html b/atdb/taskdatabase/templates/taskdatabase/quality/tasks_quality.html
new file mode 100644
index 0000000000000000000000000000000000000000..75c461e2296df90e3092b03e3231195942ff4ff4
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/quality/tasks_quality.html
@@ -0,0 +1,49 @@
+{% load static %}
+{% for task in my_tasks %}
+
+ {% if task.status != "removed_invisible" %}
+ <div class="row">
+ <tr class="{{ task.status }}">
+
+ <td>
+ <a href="{{ task.get_absolute_url }}" target="_blank">{{ task.id }} </a>
+ </td>
+ <td>
+ {% if user.is_authenticated %}
+ <a class="btn btn-primary btn-sm"
+ href="{% url 'task-details' task.id my_tasks.number %}"
+ data-popup-url="{% url 'task-details' task.id my_tasks.number %}"
+ ><i class="fas fa-list"></i> Details
+ </a>
+ {% endif %}
+ </td>
+
+ {% if task.resume %}
+ <td class="bigger">{{ task.status }}</td>
+ {% endif %}
+ {% if not task.resume %}
+ <td class="bigger">{{ task.status }} (holding)</td>
+ {% endif %}
+
+ <td>{{ task.project }}</td>
+ <td>{{ task.sas_id }}</td>
+
+ <td>{{ task.filter }} </td>
+
+ {% if task.has_quality %}
+ {% for value in task.quality_as_list %}
+ <td>{{value}}</td>
+ {% endfor %}
+ {% endif %}
+
+
+ <td>
+
+ </tr>
+ </div>
+ {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal.html" %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/quality/tasks_quality_headers.html b/atdb/taskdatabase/templates/taskdatabase/quality/tasks_quality_headers.html
new file mode 100644
index 0000000000000000000000000000000000000000..cd699a598401ff3848aa79ade6099795e5574dc3
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/quality/tasks_quality_headers.html
@@ -0,0 +1,16 @@
+<tr>
+ <th>ID</th>
+ <th>Details</th>
+
+ <th>Status</th>
+ <th>Project</th>
+ <th>SAS_ID</th>
+ <th>filter</th>
+ <th>uv-coverage</th>
+ <th>Sensitivity</th>
+ <th>Conditions</th>
+ <th>Diagnostic plots</th>
+ <th>Summary parset</th>
+ <th>Summary logs</th>
+ <th>Summary hf5</th>
+</tr>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation.html
new file mode 100644
index 0000000000000000000000000000000000000000..75c461e2296df90e3092b03e3231195942ff4ff4
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation.html
@@ -0,0 +1,49 @@
+{% load static %}
+{% for task in my_tasks %}
+
+ {% if task.status != "removed_invisible" %}
+ <div class="row">
+ <tr class="{{ task.status }}">
+
+ <td>
+ <a href="{{ task.get_absolute_url }}" target="_blank">{{ task.id }} </a>
+ </td>
+ <td>
+ {% if user.is_authenticated %}
+ <a class="btn btn-primary btn-sm"
+ href="{% url 'task-details' task.id my_tasks.number %}"
+ data-popup-url="{% url 'task-details' task.id my_tasks.number %}"
+ ><i class="fas fa-list"></i> Details
+ </a>
+ {% endif %}
+ </td>
+
+ {% if task.resume %}
+ <td class="bigger">{{ task.status }}</td>
+ {% endif %}
+ {% if not task.resume %}
+ <td class="bigger">{{ task.status }} (holding)</td>
+ {% endif %}
+
+ <td>{{ task.project }}</td>
+ <td>{{ task.sas_id }}</td>
+
+ <td>{{ task.filter }} </td>
+
+ {% if task.has_quality %}
+ {% for value in task.quality_as_list %}
+ <td>{{value}}</td>
+ {% endfor %}
+ {% endif %}
+
+
+ <td>
+
+ </tr>
+ </div>
+ {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal.html" %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation_headers.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation_headers.html
new file mode 100644
index 0000000000000000000000000000000000000000..cd699a598401ff3848aa79ade6099795e5574dc3
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation_headers.html
@@ -0,0 +1,16 @@
+<tr>
+ <th>ID</th>
+ <th>Details</th>
+
+ <th>Status</th>
+ <th>Project</th>
+ <th>SAS_ID</th>
+ <th>filter</th>
+ <th>uv-coverage</th>
+ <th>Sensitivity</th>
+ <th>Conditions</th>
+ <th>Diagnostic plots</th>
+ <th>Summary parset</th>
+ <th>Summary logs</th>
+ <th>Summary hf5</th>
+</tr>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/validation_page.html b/atdb/taskdatabase/templates/taskdatabase/validation/validation_page.html
new file mode 100644
index 0000000000000000000000000000000000000000..7382e4c58f9c29af36438efde0853c9e78758475
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/validation_page.html
@@ -0,0 +1,50 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+ <div class="card">
+ <div class="card-body">
+ <div class="row">
+ <div class="col-8">
+ <h3>Validation</h3>
+ These are the tasks in status 'stored' that can be validated. Click one of the quality buttons to validate.
+ <hr>
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-sm-12 col-md-12 col-lg-12">
+ {% include 'taskdatabase/pagination.html' %}
+
+ {% if my_tasks %}
+ <div class="panel panel-success">
+
+ <div class="panel-body">
+
+ <table class="table table-striped table-bordered table-sm">
+ <thead>
+ {% include 'taskdatabase/validation/tasks_validation_headers.html' %}
+ </thead>
+ <tbody>
+ {% include 'taskdatabase/validation/tasks_validation.html' %}
+ </tbody>
+ </table>
+ {% else %}
+ <p>No recent Tasks.</p>
+ {% endif %}
+
+ </div>
+ </div>
+ {% include 'taskdatabase/pagination.html' %}
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
+
+{% include 'taskdatabase/localstorage_scripts.html' %}
+
+
+{% endblock %}
+
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 622bf459fedf36af6799beaa441c93d340fcdebd..d69f7d87f110aa0a586ba61c09c4ace6016568c9 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -16,6 +16,8 @@ urlpatterns = [
path('', views.IndexView.as_view(), name='index'),
path('postprocessing-tasks', views.PostProcessingTasksView.as_view(), name='postprocessing-tasks'),
path('quality', views.ShowQualityPage.as_view(), name='quality'),
+ path('validation', views.ShowValidationPage.as_view(), name='validation'),
+ path('failures', views.ShowFailuresPage.as_view(), name='failures'),
path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
path('task_details/', views.TaskDetails, name='task-details'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 37456f44e39dfc305fb18cfb8cf17d593cca9d53..2597e4ee233dc793ce9efb63bf568033d6cbac12 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -327,9 +327,72 @@ class PostProcessingTasksView(ListView):
class ShowQualityPage(ListView):
"""
- This is the main view of ATDB. It shows a pagination list of tasks, sorted by creationTime.
+ This shows the tasks that have a quality statistics in its outputs[0]['quality'].
+ Not that the global filter is also applied
+ """
+ template_name = 'taskdatabase/quality/quality_page.html'
+
+ # by default this returns the list in an object called object_list, so use 'object_list' in the html page.
+ # but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
+ context_object_name = 'my_tasks'
+
+ def get_queryset(self):
+ status = self.request.GET.get('status')
+ search_box = self.request.GET.get('search_box', None)
+
+ # get the sort variable from the session or use default
+ try:
+ sort = self.request.session['sort']
+ except:
+ sort = '-creationTime'
+
+ tasks = Task.objects.order_by(sort)
+
+ # check if there is a 'task_filter' put on the session
+ try:
+ filter = self.request.session['task_filter']
+ if filter != 'all':
+ tasks = get_searched_tasks(filter, sort)
+ except:
+ pass
+
+ if (search_box is not None):
+ tasks = get_searched_tasks(search_box, sort)
+
+ # exclude the tasks without quality information
+ tasks = tasks.exclude(outputs__isnull=True)
+ tasks = tasks.exclude(outputs__0__quality__isnull=True)
+
+
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ page = self.request.GET.get('page')
+
+ try:
+ # check if there was a page on the session, if so, use it.
+ if page == None:
+ page = self.request.session['page']
+ self.request.session['page'] = None
+ except:
+ pass
+
+ try:
+ tasks = paginator.page(page)
+ except PageNotAnInteger:
+ # If page is not an integer, deliver first page.
+ tasks = paginator.page(1)
+ except EmptyPage:
+ # If page is out of range (e.g. 9999), deliver last page of results.
+ tasks = paginator.page(paginator.num_pages)
+
+ return tasks
+
+
+class ShowValidationPage(ListView):
+ """
+ This shows the tasks that are ready for validation
+ Not that the global filter is also applied
"""
- template_name = 'taskdatabase/quality_page.html'
+ template_name = 'taskdatabase/validation/validation_page.html'
# by default this returns the list in an object called object_list, so use 'object_list' in the html page.
# but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
@@ -386,6 +449,69 @@ class ShowQualityPage(ListView):
return tasks
+class ShowFailuresPage(ListView):
+ """
+ This shows the tasks that are ready for validation
+ Not that the global filter is also applied
+ """
+ template_name = 'taskdatabase/failures/failures_page.html'
+
+ # by default this returns the list in an object called object_list, so use 'object_list' in the html page.
+ # but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
+ context_object_name = 'my_tasks'
+
+ def get_queryset(self):
+ status = self.request.GET.get('status')
+ search_box = self.request.GET.get('search_box', None)
+
+ # get the sort variable from the session or use default
+ try:
+ sort = self.request.session['sort']
+ except:
+ sort = '-creationTime'
+
+ tasks = Task.objects.order_by(sort)
+
+ # check if there is a 'task_filter' put on the session
+ try:
+ filter = self.request.session['task_filter']
+ if filter != 'all':
+ tasks = get_searched_tasks(filter, sort)
+ except:
+ pass
+
+ if (search_box is not None):
+ tasks = get_searched_tasks(search_box, sort)
+
+ # exclude the tasks without quality information
+ tasks = tasks.exclude(outputs__isnull=True)
+ tasks = tasks.exclude(outputs__0__quality__isnull=True)
+
+
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ page = self.request.GET.get('page')
+
+ try:
+ # check if there was a page on the session, if so, use it.
+ if page == None:
+ page = self.request.session['page']
+ self.request.session['page'] = None
+ except:
+ pass
+
+ try:
+ tasks = paginator.page(page)
+ except PageNotAnInteger:
+ # If page is not an integer, deliver first page.
+ tasks = paginator.page(1)
+ except EmptyPage:
+ # If page is out of range (e.g. 9999), deliver last page of results.
+ tasks = paginator.page(paginator.num_pages)
+
+ return tasks
+
+
+
def get_searched_tasks(search, sort):
tasks = Task.objects.filter(
Q(id__contains=search) |