Merge branch 'SDC-436-validation' into 'master'

Sdc 436 validation

See merge request !222

atdb/taskdatabase/models.py

@@ -43,7 +43,6 @@ def convert_quality_to_shortlist_for_template(task):
         list.append(str(task.quality_json['uv-coverage']))
         list.append(str(task.quality_json['sensitivity']))
         list.append(str(task.quality_json['observing-conditions']))
-
     except Exception as err:
         pass
 

atdb/taskdatabase/services/algorithms.py

@@ -137,8 +137,6 @@ def convert_quality_to_html(task):
         except:
             pass
 
-
-
         results += "<tr><td><b>QA diagnostic plots</b></td><td>" + "-" + "</td></tr>"
         results += "<tr><td><b>Workflow summary parset</b></td><td>" + "-" + "</td></tr>"
         results += "<tr><td><b>Summary logs</b></td><td>" + "-" + "</td></tr>"

atdb/taskdatabase/tables.py

@@ -25,7 +25,7 @@ class TaskTable(tables.Table):
     class Meta:
         model = Task
         template_name = "django_tables2/bootstrap4.html"
-        fields = ("id", "workflow","filter","priority","status","project","sas_id","resume","actions","buttons")
+        fields = ("id", "workflow","filter","priority","status","quality","project","sas_id","resume","actions","buttons")
 
     # columns that need specific rendering
     status = StatusColumn()

atdb/taskdatabase/templates/taskdatabase/failures/failures_page.html

@@ -10,8 +10,8 @@
             <div class="col-8">
                 <h3>Failures</h3>
                 These are all the tasks that failed.
-                Click 'Retry' to restart a specific step in the workflow.
-                <a class="nav-link" href="{% url 'diagram' %}">See workflow diagram</a>
+                Click 'Retry' to restart the this step in the workflow (see 'Diagram' in top menu).
+                <td>{% include 'taskdatabase/filter/search.html' %}</td>
                 <hr>
             </div>
         </div>

atdb/taskdatabase/templates/taskdatabase/failures/tasks_failures.html

@@ -3,7 +3,7 @@
 
         {% if task.status != "removed_invisible" %}
         <div class="row">
-            <tr class="{{ task.status }}">
+            <tr>
 
                 <td>
                     <a href="{{ task.get_absolute_url }}" target="_blank">{{ task.id }} </a>
@@ -16,6 +16,13 @@
                            ><i class="fas fa-list"></i> Details
                         </a>&nbsp;
                     {% endif %}
+                    {% if task.has_quality %}
+                            <a class="open-modal btn btn-primary btn-sm"
+                               href="{% url 'task-quality' task.id my_tasks.number %}"
+                               data-popup-url="{% url 'task-quality' task.id my_tasks.number %}"
+                               ><i class="fas fa-balance-scale-right"></i> Quality
+                            </a>&nbsp;
+                    {% endif %}
                 </td>
 
                 {% if task.resume  %}
@@ -29,9 +36,35 @@
                 <td>{{ task.sas_id }}</td>
 
                 <td>{{ task.filter }} </td>
-                {% if task.status == "processed_failed" %}
+
+                {% if task.status == "staged_failed" or task.status == "staging_failed" %}
+                    <td>
+                        <a href="{% url 'task-retry-view' task.pk 'defined' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+                    </td>
+                {% endif %}
+                {% if task.status == "fetched_failed" or task.status == "fetching_failed" %}
+                    <td>
+                        <a href="{% url 'task-retry-view' task.pk 'staged' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+                    </td>
+                 {% endif %}
+                {% if task.status == "processed_failed" or task.status == "processing_failed" %}
+                    <td>
+                        <a href="{% url 'task-retry-view' task.pk 'fetched' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+                    </td>
+                {% endif %}
+                {% if task.status == "stored_failed" or task.status == "storing_failed" %}
+                    <td>
+                        <a href="{% url 'task-retry-view' task.pk 'processed' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+                    </td>
+                {% endif %}
+                {% if task.status == "scrubbed_failed" or task.status == "scrubbing_failed" %}
+                    <td>
+                        <a href="{% url 'task-retry-view' task.pk 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+                    </td>
+                {% endif %}
+                {% if task.status == "archived_failed" or task.status == "archiving_failed" %}
                     <td>
-                        <a href="{% url 'task-setstatus-view' task.pk 'fetched' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+                        <a href="{% url 'task-retry-view' task.pk 'scrubbed' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
                     </td>
                 {% endif %}
                 <td>

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -34,7 +34,7 @@
         {% include 'taskdatabase/pagination.html' %}
        </div>
     </div>
-    <p class="footer"> Version 1.0.0 (1 apr 2021 - 15:00)
+    <p class="footer"> Version 1.0.0 (2 apr 2021 - 6:00)
 
 </div>
 

atdb/taskdatabase/templates/taskdatabase/tasks_headers.html

 <tr>
     <th>
-        <a href="{% url 'sort-tasks' '-pk' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-pk' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         ID
-        <a href="{% url 'sort-tasks' 'id' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'id' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
     <th>Details</th>
     <th>
-        <a href="{% url 'sort-tasks' '-workflow' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-workflow' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         Workflow
-        <a href="{% url 'sort-tasks' 'workflow' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'workflow' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
     <th>
-        <a href="{% url 'sort-tasks' '-priority' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-priority' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         Priority
-        <a href="{% url 'sort-tasks' 'priority' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'priority' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
 
     <th>
-        <a href="{% url 'sort-tasks' '-status' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-status' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         Status
-        <a href="{% url 'sort-tasks' 'status' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'status' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
     <th>
-        <a href="{% url 'sort-tasks' '-project' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-project' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         Project
-        <a href="{% url 'sort-tasks' 'project' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'project' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
     <th>
-        <a href="{% url 'sort-tasks' '-sas_id' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-sas_id' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         SAS_ID
-        <a href="{% url 'sort-tasks' 'sas_id' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'sas_id' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
     <th>
-        <a href="{% url 'sort-tasks' '-creationTime' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-creationTime' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         CreationTime
-        <a href="{% url 'sort-tasks' 'creationTime' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'creationTime' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
 
     <th>
-        <a href="{% url 'sort-tasks' '-size_to_process' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        <a href="{% url 'sort-tasks' '-size_to_process' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
         Size
-        <a href="{% url 'sort-tasks' 'size_to_process' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+        <a href="{% url 'sort-tasks' 'size_to_process' 'atdb' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
     </th>
 
     <th>Actions</th>

atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation.html

@@ -34,15 +34,20 @@
                     {% for value in task.quality_as_shortlist %}
                         <td>{{value}}</td>
                      {% endfor %}
+                    <td>
+                      <a href="https://ganglia.grid.surfsara.nl/dcache.php?vo=Lofar&r=day"
+                       class="btn btn-info btn-sm" role="button" target="_blank"><img src="{% static 'taskdatabase/surfsara.jpg' %}"  height="20" alt=""></a>&nbsp;
+                    </td>
                 {% else %}
-                        <td>-</td><td>-</td><td>-</td>
+                        <td>-</td><td>-</td><td>-</td><td>-</td>
                 {% endif %}
 
+
                 <td>{{ task.quality|default_if_none:"-" }}</td>
                 <td>
-                    <a href="{% url 'task-validate-view' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> Poor</a>
-                    <a href="{% url 'task-validate-view' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> Moderate</a>
-                    <a href="{% url 'task-validate-view' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Good</a>
+                    <a href="{% url 'task-validate-view' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
+                    <a href="{% url 'task-validate-view' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> M</a>
+                    <a href="{% url 'task-validate-view' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> G</a>
 
                 </td>
 

atdb/taskdatabase/templates/taskdatabase/validation/tasks_validation_headers.html

 <tr>
-    <th>ID</th>
+    <th>
+        <a href="{% url 'sort-tasks' '-pk' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        ID
+        <a href="{% url 'sort-tasks' 'id' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
     <th>Details</th>
 
-    <th>Project</th>
-    <th>SAS_ID</th>
-    <th>filter</th>
+    <th>
+        <a href="{% url 'sort-tasks' '-project' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Project
+        <a href="{% url 'sort-tasks' 'project' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-sas_id' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        SAS_ID
+        <a href="{% url 'sort-tasks' 'sas_id' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-filter' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Filter
+        <a href="{% url 'sort-tasks' 'filter' 'validation' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
     <th>uv-coverage</th>
     <th>Sensitivity</th>
     <th>Conditions</th>
-    <th>Current Quality</th>
-    <th>Choose a Quality</th>
+    <th>Plots</th>
+    <th>Current Q</th>
+    <th>Choose</th>
 </tr>
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/validation/validation_page.html

@@ -10,9 +10,7 @@
             <div class="col-8">
                 <h3>Validation</h3>
                 These are the tasks in status 'stored' that can be validated. Click one of the quality buttons to validate.
-                <td>
-                     {% include 'taskdatabase/filter/search.html' %}
-                 </td>
+                <td>{% include 'taskdatabase/filter/search.html' %}</td>
                 <hr>
             </div>
         </div>

atdb/taskdatabase/urls.py

@@ -82,10 +82,11 @@ urlpatterns = [
     path('tasks/<int:pk>/setstatus/<new_status>', views.TaskSetStatus, name='task-details-setstatus'),
 
     path('tasks/<int:pk>/validate/<quality>/<new_status>/<page>', views.TaskValidate, name='task-validate-view'),
+    path('tasks/<int:pk>/retry//<new_status>/<page>', views.TaskRetry, name='task-retry-view'),
 
     path('tasks/<int:pk>/change_priority/<priority_change>/<page>', views.ChangePriority, name='task-change-priority'),
     path('tasks/<int:pk>/change_priority/<priority_change>', views.ChangePriority, name='task-change-priority'),
-    path('tasks/sort-tasks/<sort>', views.SortTasks, name='sort-tasks'),
+    path('tasks/sort-tasks/<sort>/<redirect_to_page>', views.SortTasks, name='sort-tasks'),
 
 
     path('tasks/set_filter/<filter>', views.TaskSetFilter, name='task-set-filter'),

atdb/taskdatabase/views.py

@@ -59,6 +59,7 @@ class TaskFilter(filters.FilterSet):
             'project': ['exact', 'icontains'],
             'sas_id': ['exact', 'icontains', 'in'],
             'status': ['exact', 'icontains', 'in', 'startswith'],
+            'quality': ['exact', 'icontains', 'in', 'startswith'],
             'purge_policy': ['exact'],
             'priority': ['exact', 'lte', 'gte'],
             'resume': ['exact'],
@@ -81,6 +82,7 @@ class TaskFilterQueryPage(filters.FilterSet):
             'filter': ['exact', 'icontains'],
             'priority': ['exact', 'gte', 'lte'],
             'status': ['icontains', 'in'],
+            'quality': ['icontains', 'in'],
             'project': ['exact', 'icontains', 'in'],
             'sas_id': ['exact', 'icontains', 'in'],
             # 'resume': ['exact'],
@@ -215,7 +217,7 @@ class IndexView(ListView):
         try:
             filter = self.request.session['task_filter']
             if filter != 'all':
-                tasks = get_searched_tasks(filter, sort)
+                tasks = get_filtered_tasks(filter, sort)
         except:
             pass
 
@@ -229,7 +231,7 @@ class IndexView(ListView):
             pass
 
         if (search_box is not None):
-            tasks = get_searched_tasks(search_box, sort)
+            tasks = get_filtered_tasks(search_box, sort)
 
         # only return the 'regular' tasks, and not the 'postprocessing' tasks in the GUI
         # tasks = tasks.filter(task_type='regular')
@@ -283,7 +285,7 @@ class PostProcessingTasksView(ListView):
         try:
             filter = self.request.session['task_filter']
             if filter != 'all':
-                tasks = get_searched_tasks(filter, sort)
+                tasks = get_filtered_tasks(filter, sort)
         except:
             pass
 
@@ -297,7 +299,7 @@ class PostProcessingTasksView(ListView):
             pass
 
         if (search_box is not None):
-            tasks = get_searched_tasks(search_box, sort)
+            tasks = get_filtered_tasks(search_box, sort)
 
         # only return the 'regular' tasks, and not the 'postprocessing' tasks in the GUI
         tasks = tasks.filter(task_type='postprocessing')
@@ -351,12 +353,12 @@ class ShowQualityPage(ListView):
         try:
             filter = self.request.session['task_filter']
             if filter != 'all':
-                tasks = get_searched_tasks(filter, sort)
+                tasks = get_filtered_tasks(filter, sort)
         except:
             pass
 
         if (search_box is not None):
-            tasks = get_searched_tasks(search_box, sort)
+            tasks = get_filtered_tasks(search_box, sort)
 
         # exclude the tasks without quality information
         #tasks = tasks.exclude(outputs__isnull=True)
@@ -406,18 +408,18 @@ class ShowValidationPage(ListView):
         except:
             sort = '-creationTime'
 
-        tasks = Task.objects.filter(status__icontains='stored').order_by(sort)
-
+        stored_tasks = Task.objects.filter(status__icontains='stored').order_by(sort)
+        tasks = stored_tasks
         # check if there is a 'task_filter' put on the session
         try:
             filter = self.request.session['task_filter']
             if filter != 'all':
-                tasks = get_searched_tasks(filter, sort)
+                tasks = get_filtered_tasks(filter, sort, stored_tasks)
         except:
             pass
 
         if (search_box is not None):
-            tasks = get_searched_tasks(search_box, sort)
+            tasks = get_filtered_tasks(search_box, sort, stored_tasks)
 
         # exclude the tasks without quality information
         tasks = tasks.exclude(status__icontains="failed")
@@ -457,7 +459,6 @@ class ShowFailuresPage(ListView):
     context_object_name = 'my_tasks'
 
     def get_queryset(self):
-        status = self.request.GET.get('status')
         search_box = self.request.GET.get('search_box', None)
 
         # get the sort variable from the session or use default
@@ -466,18 +467,19 @@ class ShowFailuresPage(ListView):
         except:
             sort = '-creationTime'
 
-        tasks = Task.objects.filter(status__icontains='failed').order_by(sort)
+        failed_tasks = Task.objects.filter(status__icontains='failed').order_by(sort)
+        tasks = failed_tasks
 
         # check if there is a 'task_filter' put on the session
         try:
             filter = self.request.session['task_filter']
             if filter != 'all':
-                tasks = get_searched_tasks(filter, sort)
+                tasks = get_filtered_tasks(filter, sort, failed_tasks)
         except:
             pass
 
         if (search_box is not None):
-            tasks = get_searched_tasks(search_box, sort)
+            tasks = get_filtered_tasks(search_box, sort, failed_tasks)
 
         paginator = Paginator(tasks, config.TASKS_PER_PAGE)  # Show 50 tasks per page
         page = self.request.GET.get('page')
@@ -502,9 +504,14 @@ class ShowFailuresPage(ListView):
         return tasks
 
 
+# this provides a broad range of filters for the search_box in the GUI
+def get_filtered_tasks(search, sort, tasks=None):
+    if tasks:
+        tasks_to_filter = tasks
+    else:
+        tasks_to_filter = Task.objects.all()
 
-def get_searched_tasks(search, sort):
-    tasks = Task.objects.filter(
+    filtered_tasks = tasks_to_filter.filter(
         Q(id__contains=search) |
         Q(sas_id__contains=search) |
         Q(creationTime__icontains=search) |
@@ -512,7 +519,7 @@ def get_searched_tasks(search, sort):
         Q(status__icontains=search) |
         Q(status__in=search) |
         Q(project__icontains=search)).order_by(sort)
-    return tasks
+    return filtered_tasks
 
 
 class TaskTables2View(SingleTableView):
@@ -924,6 +931,21 @@ def TaskValidate(request, pk, quality, new_status, page=0):
         return redirect('/atdb/validation?page=' + page)
 
 
+@login_required
+def TaskRetry(request, pk, new_status, page=0):
+    model = Task
+    task = Task.objects.get(pk=pk)
+    task.new_status = new_status
+    task.save()
+
+    if page == 0:
+        # redirect to details screen
+        return redirect('/atdb/task_details')
+    else:
+        # redirect to tasks list
+        return redirect('/atdb/failures?page=' + page)
+
+
 # set a filter value in the session, used later by the 'get_searched_tasks' mechanism
 def TaskSetFilter(request, filter):
     request.session['task_filter'] = filter
@@ -973,10 +995,14 @@ def ChangePriority(request, pk, priority_change, page=0):
         return redirect('/atdb/?page=' + page)
 
 
-def SortTasks(request, sort):
+def SortTasks(request, sort, redirect_to_page):
     # store the sort field on the session
     request.session['sort'] = sort
+    if redirect_to_page=='atdb':
         return redirect('/atdb')
+    else:
+        return redirect(redirect_to_page)
+
 
 
 def convert_query_params_to_url(query_params):