diff --git a/atdb/atdb/settings/dev.py b/atdb/atdb/settings/dev.py
index 6467a72df42559cd143c5c9bc97ff9736da5b347..dc1cae903f010e6f4bcde9f134001fa199b2a57f 100644
--- a/atdb/atdb/settings/dev.py
+++ b/atdb/atdb/settings/dev.py
@@ -13,7 +13,7 @@ DATABASES = {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'atdb_admin',
'PASSWORD': 'atdb123',
- 'NAME': 'atdb_ldv_20nov2023',
+ 'NAME': 'atdb_ldv_4jan2024',
'HOST': 'localhost',
'PORT': '5432',
},
diff --git a/atdb/taskdatabase/config.py b/atdb/taskdatabase/config.py
index f4ff5fc82e25e1c43d478623f7de42f7b063ff53..acfde381938613709c5336ef82e264602ef89aa2 100644
--- a/atdb/taskdatabase/config.py
+++ b/atdb/taskdatabase/config.py
@@ -1,3 +1,4 @@
VERSION = "Version 1.0.0 (14 jan 2020)"
TASKS_PER_PAGE = 50
+TASKS_PER_PAGE_SMALL = 10
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 197ff582c9aa7e5d3fa7eac4a99c6221d892ccfb..bc4758b28ead0f5cef12d70cba2777cb51295326 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -21,6 +21,7 @@ class State(Enum):
STORED = 'stored'
VALIDATED = "validated"
SCRUBBED = "scrubbed"
+ ARCHIVING = "archiving"
ARCHIVED = "archived"
FINISHED = "finished"
SUSPENDED = "suspended"
@@ -29,6 +30,7 @@ class State(Enum):
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
verified_statusses = ['stored','validated','scrubbed','archived','finished','suspended','discarded']
+ingest_statusses = ['scrubbed','archiving','archived']
class Workflow(models.Model):
description = models.CharField(max_length=500, blank=True, null=True)
@@ -324,6 +326,25 @@ class Task(models.Model):
return finished
+ @property
+ def sasid_ingested_fraction(self):
+ result = {}
+ statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finished': 0}
+
+ tasks = Task.objects.filter(sas_id=self.sas_id)
+
+ for task in tasks:
+ try:
+ statusses[task.status] = statusses[task.status] + 1
+ except:
+ pass
+
+ incomplete = int(statusses['scrubbed']) + int(statusses['archiving'])
+ complete = int(statusses['archived']) + int(statusses['finished'])
+ completion = round(complete / (incomplete + complete) * 100)
+ result['status'] = statusses
+ result['completion'] = completion
+ return result
@property
def task_type_join(self):
diff --git a/atdb/taskdatabase/templates/astronauth/navbar.html b/atdb/taskdatabase/templates/astronauth/navbar.html
index 02569569352e12bf6af7b0f0dd10e86071ab2bcd..fc7678131e5ac47ee6cba5278c0b99f2798021d5 100644
--- a/atdb/taskdatabase/templates/astronauth/navbar.html
+++ b/atdb/taskdatabase/templates/astronauth/navbar.html
@@ -28,6 +28,7 @@
<li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
<li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
+ <li><a class="nav-link" href="{% url 'ingest' %}">Ingest</a></li>
<li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
<li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
<li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>
diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html
index 1f4b938b974045fa803bcd39a14bc5d554d72802..900bad4a8c3bee9e96e4338ca7e993e292aa5926 100644
--- a/atdb/taskdatabase/templates/taskdatabase/index.html
+++ b/atdb/taskdatabase/templates/taskdatabase/index.html
@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %}
</div>
</div>
- <p class="footer"> Version 4 Jan 2024
+ <p class="footer"> Version 5 Jan 2024
</div>
{% include 'taskdatabase/refresh.html' %}
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html
new file mode 100644
index 0000000000000000000000000000000000000000..06d07ae139ed3196fa0ed99a9e9397d68b816aa6
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html
@@ -0,0 +1 @@
+<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
new file mode 100644
index 0000000000000000000000000000000000000000..292d782894828006cf07cd537f83ab446a9a9530
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
@@ -0,0 +1,32 @@
+
+
+ <div class="card">
+ <div class="card-body">
+ <table>
+
+ <tr>
+
+ {% if request.session.task_filter == "all" %}
+ <td>Click to Filter</td>
+ {% else %}
+ <td>Click to Filter ({{request.session.task_filter}})</td>
+ {% endif %}
+ </tr>
+
+ <tr>
+ <td>
+ {% include "taskdatabase/quality/clear_filter_button.html" %}
+ <a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued</a>
+ <a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+
+ </td>
+ <td>
+ {% include 'taskdatabase/filter/search.html' %}
+ </td>
+ </tr>
+
+ </table>
+ </div>
+ </div>
+
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/headers.html b/atdb/taskdatabase/templates/taskdatabase/ingest/headers.html
new file mode 100644
index 0000000000000000000000000000000000000000..806d6f73ae61d7e0317076eeb416f0ee557897dc
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/headers.html
@@ -0,0 +1,29 @@
+<tr>
+
+ <th>
+ <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Workflow
+ <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>Status</th>
+ <th>
+ <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Project
+ <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Filter
+ <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ SAS_ID (input)
+ <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>Completion</th>
+ <th>
+ SAS_ID (output) at LTA
+ </th>
+
+</tr>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/page.html b/atdb/taskdatabase/templates/taskdatabase/ingest/page.html
new file mode 100644
index 0000000000000000000000000000000000000000..8b4ad569a538a837e3aa568b97a8270e222ca2c6
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/page.html
@@ -0,0 +1,54 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+ <div class="card">
+ <div class="card-body">
+ <div class="row">
+ <div class="col-12">
+ <h3>Ingest Queue</h3>
+
+ These are the SAS_ID's queued for ingest (<i>scrubbed</i>), <i>archiving</i> or <i>archived</i> into the LTA.
+ <hr>
+ {% include 'taskdatabase/ingest/filter_buttons.html' %}
+
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-sm-12 col-md-12 col-lg-12">
+ {% include 'taskdatabase/pagination.html' %}
+
+ {% if my_tasks %}
+ <div class="panel panel-success">
+
+ <div class="panel-body">
+
+ <table class="table table-striped table-bordered table-sm">
+ <thead>
+ {% include 'taskdatabase/ingest/headers.html' %}
+ </thead>
+ <tbody>
+ {% include 'taskdatabase/ingest/tasks.html' %}
+ </tbody>
+ </table>
+ {% else %}
+ <p>No recent Tasks.</p>
+ {% endif %}
+
+ </div>
+ </div>
+ {% include 'taskdatabase/pagination.html' %}
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
+
+{% include 'taskdatabase/no_refresh.html' %}
+
+
+{% endblock %}
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
new file mode 100644
index 0000000000000000000000000000000000000000..814bb21c2ccfe9976b254c2ee29573a2c42fa1ad
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
@@ -0,0 +1,38 @@
+{% load static %}
+{% for task in my_tasks %}
+
+ {% if task.status != "removed_invisible" %}
+ <div class="row">
+ <tr>
+
+ <td>
+ <a class="open-modal btn btn-primary btn-sm"
+ href="{% url 'workflow-details' task.workflow.id %}"
+ data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+ target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
+ </a></td>
+ </td>
+ <td>{{ task.sasid_ingested_fraction.status }}</td>
+ <td>{{ task.project }}</td>
+
+ <td>{{ task.filter }} </td>
+ <td>{{ task.sas_id }}</td>
+ <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+ <td>
+ {% if task.sas_id_archived != None %}
+ <a href={{ task.path_to_lta }} target="_blank">
+ <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+ {{ task.sas_id_archived }}
+ </a>
+ {% else %}
+ -
+ {% endif %}
+ </td>
+ </tr>
+ </div>
+ {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal_no_close.html" %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 091e51c1f28c18f617606841b85346cc6e0f63b8..17c565bee11aefe4abe69271d5bae647368e777d 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -19,6 +19,7 @@ urlpatterns = [
path('validation', views.ShowValidationPage.as_view(), name='validation'),
path('failures', views.ShowFailuresPage.as_view(), name='failures'),
path('discarded', views.ShowDiscardedPage.as_view(), name='discarded'),
+ path('ingest', views.ShowIngestPage.as_view(), name='ingest'),
path('finished', views.ShowFinishedPage.as_view(), name='finished'),
path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index f912cfa0e183ebb12a36eee7b097160aa934bebc..b98f20a6b9bb870712cfbc538b43b9cbba9aad8b 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -335,20 +335,12 @@ class ShowValidationPage(ListView):
#@silk_profile(name='ShowValidationPage')
def get_queryset(self):
- # check filtered_tasks on the session
- # if it is at its max, then it is not a query targeted at 1 SAS_ID.
- # in that case clear the filter so that all the SAS_ID's show up on the validation page.
- #if self.request.session['filtered_tasks_as_list']:
- # filtered_tasks_on_session = len(self.request.session['filtered_tasks_as_list'])
- # if filtered_tasks_on_session == settings.QUERY_LIMIT_MULTI_CHANGE:
- # self.request.session['filtered_tasks_as_list'] = []
-
tasks = get_filtered_tasks(self.request, None, "sas_id").filter(status__icontains=State.STORED.value)
# exclude the failed tasks
tasks = tasks.exclude(status__icontains=State.FAILED.value)
- paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL) # Show 50 tasks per page
page = self.request.GET.get('page')
try:
@@ -385,11 +377,9 @@ class ShowFailuresPage(ListView):
#@silk_profile(name='ShowFailuresPage')
def get_queryset(self):
- #failed_tasks = Task.objects.filter(status__icontains=State.FAILED.value)
- #tasks = get_filtered_tasks(self.request, failed_tasks)
tasks = get_filtered_tasks(self.request).filter(status__icontains=State.FAILED.value)
- paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL) # Show 50 tasks per page
page = self.request.GET.get('page')
try:
@@ -452,6 +442,50 @@ class ShowDiscardedPage(ListView):
return tasks
+class ShowIngestPage(ListView):
+ """
+ This shows aggregated tasks per sas_id that are queued for ingest or archiving
+ Note that the global filter is also applied
+ """
+ template_name = 'taskdatabase/ingest/page.html'
+ context_object_name = 'my_tasks'
+
+ # @silk_profile(name='ShowIngestPage')
+ def get_queryset(self):
+
+ ingest_tasks = Task.objects.filter(
+ Q(status__icontains=State.SCRUBBED.value) |
+ Q(status__icontains=State.ARCHIVING.value) |
+ Q(status__icontains=State.ARCHIVED.value))
+
+ tasks = get_filtered_tasks(self.request, ingest_tasks, "sas_id")
+
+ # exclude the failed tasks
+ tasks = tasks.exclude(status__icontains=State.FAILED.value)
+
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL) # Show 10 tasks per page
+ page = self.request.GET.get('page')
+
+ try:
+ # check if there was a page on the session, if so, use it.
+ if page == None:
+ page = self.request.session['page']
+ self.request.session['page'] = None
+ except:
+ pass
+
+ try:
+ tasks = paginator.page(page)
+ except PageNotAnInteger:
+ # If page is not an integer, deliver first page.
+ tasks = paginator.page(1)
+ except EmptyPage:
+ # If page is out of range (e.g. 9999), deliver last page of results.
+ tasks = paginator.page(paginator.num_pages)
+
+ return tasks
+
+
class ShowFinishedPage(ListView):
"""
This shows the tasks that are archived
@@ -1282,6 +1316,8 @@ def TaskSetFilter(request, filter, redirect_to_page):
if redirect_to_page == 'quality':
return redirect('quality')
+ if redirect_to_page == 'ingest':
+ return redirect('ingest')
return redirect_with_params('index', '?page=1')
@@ -1311,12 +1347,15 @@ def TaskClearFilter(request, redirect_to_page):
request.session['search_box'] = ''
request.session['filtered'] = False
+
if redirect_to_page == 'quality':
return redirect('quality')
if redirect_to_page == 'query':
return redirect('query')
if redirect_to_page == 'discarded':
return redirect('discarded')
+ if redirect_to_page == 'ingest':
+ return redirect('ingest')
return redirect_with_params('index', '?page=1')