From 06fb00fe7df3960b2411fc5cbd329f8829f61351 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Thu, 8 Feb 2024 14:46:50 +0100
Subject: [PATCH] move more functionality to update_activities
---
atdb/taskdatabase/models.py | 45 +---------------
atdb/taskdatabase/services/activities.py | 21 ++++++--
atdb/taskdatabase/services/common.py | 4 ++
.../templates/astronauth/navbar.html | 1 -
.../taskdatabase/ingest2/activities.html | 45 ----------------
.../ingest2/clear_filter_button.html | 1 -
.../taskdatabase/ingest2/filter_buttons.html | 32 -----------
.../taskdatabase/ingest2/headers.html | 34 ------------
.../templates/taskdatabase/ingest2/page.html | 54 -------------------
.../taskdatabase/ingest2/pagination.html | 43 ---------------
.../validation/validation_buttons.html | 8 +--
atdb/taskdatabase/urls.py | 2 +-
atdb/taskdatabase/views.py | 49 ++++-------------
13 files changed, 38 insertions(+), 301 deletions(-)
delete mode 100644 atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html
delete mode 100644 atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html
delete mode 100644 atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html
delete mode 100644 atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html
delete mode 100644 atdb/taskdatabase/templates/taskdatabase/ingest2/page.html
delete mode 100644 atdb/taskdatabase/templates/taskdatabase/ingest2/pagination.html
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 06ec943e..d53a6a11 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -7,14 +7,14 @@ import json
import logging
from .services import calculated_qualities as qualities
-from .services.common import State
+from .services.common import State, verified_statusses
logger = logging.getLogger(__name__)
# constants
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
-verified_statusses = ['stored','validated','scrubbed','archived','finished','suspended','discarded']
+
class Workflow(models.Model):
description = models.CharField(max_length=500, blank=True, null=True)
@@ -311,31 +311,6 @@ class Task(models.Model):
except:
return None
- @property
- def sas_id_archived(self):
- """
- check if this task already has an output SAS_ID at the LTA
- """
- try:
- return self.archive['sas_id_archived']
- except:
- return None
-
- @property
- def sas_id_has_archived(self):
- """
- check if any task belonging to this sas_id already has an output SAS_ID at the LTA
- """
- try:
- for task in Task.objects.filter(sas_id=self.sas_id):
- try:
- if task.archive['sas_id_archived']:
- return task.archive['sas_id_archived']
- except:
- pass
- except:
- return None
-
@property
def path_to_lta(self):
"""
@@ -346,22 +321,6 @@ class Task(models.Model):
except:
return None
- @property
- def sasid_path_to_lta(self):
- """
- check if any task belonging to this sas_id already has a 'path_to_lta' setting
- """
- try:
- for task in Task.objects.filter(sas_id=self.sas_id):
- try:
- if task.archive['path_to_lta']:
- return task.archive['path_to_lta']
- except:
- # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
- pass
- except:
- return None
-
@property
def sasid_is_verified(self):
diff --git a/atdb/taskdatabase/services/activities.py b/atdb/taskdatabase/services/activities.py
index e46653fe..e5a91340 100644
--- a/atdb/taskdatabase/services/activities.py
+++ b/atdb/taskdatabase/services/activities.py
@@ -1,5 +1,5 @@
import logging;
-from .common import State
+from .common import State, verified_statusses
from taskdatabase.models import Task, Activity
logger = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ def calculate_ingested_fraction(this_task):
return result
-def associate_activity(task, save_task=True):
+def associate_task_with_activity(task, save_task=True):
if not task.activity:
@@ -81,7 +81,7 @@ def update_activity(task):
# do not save the task,
# because this function is called from signals where the task.save is delayed on purpose to avoid recursion
- activity = associate_activity(task, save_task=False)
+ activity = associate_task_with_activity(task, save_task=False)
# depending on the status transition, perform calculations
if task.status == State.STORED.value:
@@ -113,4 +113,17 @@ def update_activity(task):
except:
pass
- activity.save()
\ No newline at end of file
+ activity.save()
+
+ # check if all tasks of this SAS_ID have a status that is considered 'verified'
+ # this is used for the Validation Page
+ current_is_verified = activity.is_verified
+ activity.is_verified = True
+ for t in Task.objects.filter(sas_id=task.sas_id):
+ if t.status not in verified_statusses:
+ activity.is_verified = False
+ break
+
+ # only save when changed
+ if activity.is_verified != current_is_verified:
+ activity.save()
diff --git a/atdb/taskdatabase/services/common.py b/atdb/taskdatabase/services/common.py
index f0ada76f..3cf84aa8 100644
--- a/atdb/taskdatabase/services/common.py
+++ b/atdb/taskdatabase/services/common.py
@@ -23,6 +23,10 @@ class State(Enum):
DISCARDED = "discarded"
FAILED = "failed"
+verified_statusses = [State.STORED.value, State.VALIDATED.value, State.SCRUBBED.value, State.ARCHIVED.value,
+ State.FINISHED.value, State.SUSPENDED.value, State.DISCARDED.value]
+
+
class SummaryFlavour(Enum):
DEFAULT = "default"
LINC_CALIBRATOR = "linc_calibrator"
diff --git a/atdb/taskdatabase/templates/astronauth/navbar.html b/atdb/taskdatabase/templates/astronauth/navbar.html
index 28b693bc..944ae733 100644
--- a/atdb/taskdatabase/templates/astronauth/navbar.html
+++ b/atdb/taskdatabase/templates/astronauth/navbar.html
@@ -29,7 +29,6 @@
<li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
<li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
<li><a class="nav-link" href="{% url 'ingest' %}">IngestQ</a></li>
- <li><a class="nav-link" href="{% url 'ingest2' %}">IngestQ2</a></li>
<li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
<li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
<li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html
deleted file mode 100644
index acc7ff8c..00000000
--- a/atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html
+++ /dev/null
@@ -1,45 +0,0 @@
-{% load static %}
-{% for activity in my_activities %}
- {% if activity.status != "removed_invisible" %}
- <div class="row">
- <tr>
- <td>{{ activity.sas_id }}</td>
- <td>{{ activity.project }}</td>
- <td>
- <a class="open-modal btn btn-primary btn-sm"
- href="{% url 'workflow-details' activity.workflow.id %}"
- data-popup-url="{% url 'workflow-details' activity.workflow.id %}"
- target="_blank"><i class="fas fa-project-diagram"></i> {{ activity.workflow.id }}
- </a></td>
- </td>
- <td>{{ activity.filter }} </td>
- <td>
- {% if user.is_authenticated %}
- <a href="{% url 'task-change-priority-sasid' activity.sas_id '-10' my_activities.number %}" class="btn btn-warning btn-sm" role="button">-10</a>
- {% endif %}
- {{ task.priority }}
- {% if user.is_authenticated %}
- <a href="{% url 'task-change-priority-sasid' activity.sas_id '10' my_activities.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
- {% endif %}
- </td>
- <td>{{ activity.ingestq_status }}</td>
-
- <td>{{ activity.ingested_fraction }}%</td>
- <td>
- {% if activity.has_archived != None %}
- <a href={{ activity.archive.path_to_lta }} target="_blank">
- <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
- {{ activity.has_archived }}
- </a>
- {% else %}
- -
- {% endif %}
- </td>
- </tr>
- </div>
- {% endif %}
-
-{% endfor %}
-
-{% include "taskdatabase/modal/modal_script.html" %}
-{% include "taskdatabase/modal/modal_no_close.html" %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html
deleted file mode 100644
index 06d07ae1..00000000
--- a/atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html
+++ /dev/null
@@ -1 +0,0 @@
-<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html
deleted file mode 100644
index 118ccd3c..00000000
--- a/atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
- <div class="card">
- <div class="card-body">
- <table>
-
- <tr>
-
- {% if request.session.task_filter == "all" %}
- <td>Click to Filter</td>
- {% else %}
- <td>Click to Filter ({{request.session.task_filter}})</td>
- {% endif %}
- </tr>
-
- <tr>
- <td>
- {% include "taskdatabase/ingest/clear_filter_button.html" %}
- <a href="{% url 'task-set-ingest-filter' 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
- <a href="{% url 'task-set-ingest-filter' 'archiving' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
-
- </td>
- <td>
- {% include 'taskdatabase/filter/search.html' %}
- </td>
- </tr>
-
- </table>
- </div>
- </div>
-
-
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html
deleted file mode 100644
index a3659194..00000000
--- a/atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html
+++ /dev/null
@@ -1,34 +0,0 @@
-<tr>
- <th>
- <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
- SAS_ID (input)
- <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
- </th>
- <th>
- <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
- Project
- <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
- </th>
- <th>
- <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
- Workflow
- <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
- </th>
- <th>
- <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
- Filter
- <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
- </th>
- <th>
- <a href="{% url 'sort-tasks' '-priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
- Priority
- <a href="{% url 'sort-tasks' 'priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
- </th>
- <th>Status</th>
-
- <th>Completion</th>
- <th>
- SAS_ID (output) at LTA
- </th>
-
-</tr>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/page.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/page.html
deleted file mode 100644
index 70442ffc..00000000
--- a/atdb/taskdatabase/templates/taskdatabase/ingest2/page.html
+++ /dev/null
@@ -1,54 +0,0 @@
-{% extends 'taskdatabase/base.html' %}
-{% load static %}
-
-{% block myBlock %}
-
-<div class="container-fluid details-container">
-
- <div class="card">
- <div class="card-body">
- <div class="row">
- <div class="col-12">
- <h3>Ingest Queue</h3>
-
- The ingest queue shows SASids with archiving and queued (scrubbed) tasks which are ingesting into the LTA
-
- {% include 'taskdatabase/ingest2/filter_buttons.html' %}
-
- </div>
- </div>
- <div class="row">
- <div class="col-sm-12 col-md-12 col-lg-12">
- {% include 'taskdatabase/ingest2/pagination.html' %}
-
- {% if my_activities %}
- <div class="panel panel-success">
-
- <div class="panel-body">
-
- <table class="table table-striped table-bordered table-sm">
- <thead>
- {% include 'taskdatabase/ingest2/headers.html' %}
- </thead>
- <tbody>
- {% include 'taskdatabase/ingest2/activities.html' %}
- </tbody>
- </table>
- {% else %}
- <p>No Activities.</p>
- {% endif %}
-
- </div>
- </div>
- {% include 'taskdatabase/ingest2/pagination.html' %}
- </div>
- </div>
- </div>
- </div>
-</div>
-
-{% include 'taskdatabase/no_refresh.html' %}
-
-
-{% endblock %}
-
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/pagination.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/pagination.html
deleted file mode 100644
index 64ab02c0..00000000
--- a/atdb/taskdatabase/templates/taskdatabase/ingest2/pagination.html
+++ /dev/null
@@ -1,43 +0,0 @@
- <div class="btn-group" role="group" aria-label="Item pagination" style="margin-bottom: 1rem">
-
- {% if my_activities.has_previous %}
- <a href="?page=1" class="btn btn-outline-primary">First</a>
- <a href="?page={{ my_activities.previous_page_number }}" class="btn btn-outline-primary">Previous</a>
- {% else %}
- <button class="btn btn-outline-primary" disabled>First</button>
- <button class="btn btn-outline-primary" disabled>Previous</button>
- {% endif %}
-
- {% for page_number in my_activities.paginator.page_range %}
- {% if my_activities.number == page_number %}
- <button class="btn btn-outline-primary active">
- <span>{{ page_number }} <span class="sr-only">(current)</span></span>
- </button>
- <!-- first and last 20 pages -->
- {% elif my_activities.number < 10 and page_number < 20 or my_activities.number > my_activities.paginator.num_pages|add:"-10" and page_number > my_activities.paginator.num_pages|add:"-20" %}
- <a href="?page={{ page_number }}" class="btn btn-outline-primary">
- {{ page_number }}
- </a>
- <!-- 10 pages around current page -->
- {% elif page_number < my_activities.number and page_number > my_activities.number|add:"-10" or page_number > my_activities.number and page_number < my_activities.number|add:10 %}
- <a href="?page={{ page_number }}" class="btn btn-outline-primary">
- {{ page_number }}
- </a>
- {% endif %}
- {% endfor %}
-
- <!-- last page button or ellipsis if last page is too far -->
- {% if my_activities.number|add:10 < my_activities.paginator.num_pages %}
- <button class="btn btn-outline-primary" disabled>...</button>
- {% elif my_activities.number|add:10 == my_activities.paginator.num_pages %}
- <a href="?page={{ my_activities.paginator.num_pages }}" class="btn btn-outline-primary">{{ my_activities.paginator.num_pages }}</a>
- {% endif %}
-
- {% if my_activities.has_next %}
- <a href="?page={{ my_activities.next_page_number }}" class="btn btn-outline-primary">Next</a>
- <a href="?page={{ my_activities.paginator.num_pages }}" class="btn btn-outline-primary">Last</a>
- {% else %}
- <button class="btn btn-outline-primary" disabled>Next</button>
- <button class="btn btn-outline-primary" disabled>Last</button>
- {% endif %}
- </div>
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
index fca15520..a64240c3 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
@@ -1,16 +1,16 @@
- {% if task.sasid_is_verified %}
+ {% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
{% endif %}
- {% if task.sasid_is_verified %}
+ {% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> M</a>
{% endif %}
- {% if task.sasid_is_verified %}
+ {% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> G</a>
{% endif %}
- {% if task.sasid_is_verified %}
+ {% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
{% endif %}
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index a89c664a..b59f8b2d 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -20,7 +20,6 @@ urlpatterns = [
path('failures', views.ShowFailuresPage.as_view(), name='failures'),
path('discarded', views.ShowDiscardedPage.as_view(), name='discarded'),
path('ingest', views.ShowIngestQPage.as_view(), name='ingest'),
- path('ingest2', views.ShowIngestQPage2.as_view(), name='ingest2'),
path('finished', views.ShowFinishedPage.as_view(), name='finished'),
path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
@@ -131,4 +130,5 @@ urlpatterns = [
path('tasks/<int:pk>/query-purge/<purge_policy>/<query_params>', views.PurgeQuery, name='query-purge'),
path('tasks/associate-activities/', views.AssociateActivities, name='associate-activities'),
+ path('tasks/update-all-activities/', views.UpdateAllActivities(), name='update-all-activities'),
]
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index 32e41624..c29fcc75 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -499,44 +499,6 @@ class ShowIngestQPage(ListView):
return tasks
-class ShowIngestQPage2(ListView):
- """
- This shows aggregated tasks per sas_id that are queued for ingest or archiving
- Note that the global filter is also applied
- """
- template_name = 'taskdatabase/ingest2/page.html'
- context_object_name = 'my_activities'
-
- def get_queryset(self):
-
- activities = Activity.objects.filter(
- Q(ingestq_status__icontains=State.SCRUBBED.value) |
- (Q(ingested_fraction__gt=0) & Q(ingested_fraction__lt=100) )).order_by('-sas_id')
-
- # exclude the failed tasks
- paginator = Paginator(activities, config.TASKS_PER_PAGE_SMALL) # Show 10 tasks per page
- page = self.request.GET.get('page')
-
- try:
- # check if there was a page on the session, if so, use it.
- if page == None:
- page = self.request.session['page']
- self.request.session['page'] = None
- except:
- pass
-
- try:
- activities = paginator.page(page)
- except PageNotAnInteger:
- # If page is not an integer, deliver first page.
- activities = paginator.page(1)
- except EmptyPage:
- # If page is out of range (e.g. 9999), deliver last page of results.
- activities = paginator.page(paginator.num_pages)
-
- return activities
-
-
class ShowFinishedPage(ListView):
"""
This shows the tasks that are finished
@@ -1714,7 +1676,16 @@ def AssociateActivities(request):
tasks = Task.objects.all()
for task in tasks:
- activities.associate_activity(task)
+ activities.associate_task_with_activity(task)
logger.info(f'{task.id} => {task.sas_id}')
+ return redirect('index')
+
+def UpdateAllActivities(request):
+
+ tasks = Task.objects.all()
+ for task in tasks:
+ activities.update_activity(task)
+ logger.info(f'{task.id}')
+
return redirect('index')
\ No newline at end of file
--
GitLab