add Ingest page

atdb/atdb/settings/dev.py

@@ -13,7 +13,7 @@ DATABASES = {
          'ENGINE': 'django.db.backends.postgresql_psycopg2',
          'USER': 'atdb_admin',
          'PASSWORD': 'atdb123',
-         'NAME': 'atdb_ldv_20nov2023',
+         'NAME': 'atdb_ldv_4jan2024',
          'HOST': 'localhost',
          'PORT': '5432',
     },

atdb/taskdatabase/config.py

 VERSION = "Version 1.0.0 (14 jan 2020)"
 TASKS_PER_PAGE = 50
+TASKS_PER_PAGE_SMALL = 10
 

atdb/taskdatabase/models.py

@@ -21,6 +21,7 @@ class State(Enum):
     STORED = 'stored'
     VALIDATED = "validated"
     SCRUBBED = "scrubbed"
+    ARCHIVING = "archiving"
     ARCHIVED = "archived"
     FINISHED = "finished"
     SUSPENDED = "suspended"
@@ -29,6 +30,7 @@ class State(Enum):
 
 datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
 verified_statusses = ['stored','validated','scrubbed','archived','finished','suspended','discarded']
+ingest_statusses = ['scrubbed','archiving','archived']
 
 class Workflow(models.Model):
     description = models.CharField(max_length=500, blank=True, null=True)
@@ -324,6 +326,25 @@ class Task(models.Model):
 
         return finished
 
+    @property
+    def sasid_ingested_fraction(self):
+        result = {}
+        statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finished': 0}
+
+        tasks = Task.objects.filter(sas_id=self.sas_id)
+
+        for task in tasks:
+            try:
+                statusses[task.status] = statusses[task.status] + 1
+            except:
+                pass
+
+        incomplete = int(statusses['scrubbed']) + int(statusses['archiving'])
+        complete = int(statusses['archived']) + int(statusses['finished'])
+        completion = round(complete / (incomplete + complete) * 100)
+        result['status'] = statusses
+        result['completion'] = completion
+        return result
 
     @property
     def task_type_join(self):

atdb/taskdatabase/templates/astronauth/navbar.html

@@ -28,6 +28,7 @@
 
                 <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
                 <li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
+                <li><a class="nav-link" href="{% url 'ingest' %}">Ingest</a></li>
                 <li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
                 <li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
                 <li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -31,7 +31,7 @@
             {% include 'taskdatabase/pagination.html' %}
         </div>
     </div>
-    <p class="footer"> Version 4 Jan 2024
+    <p class="footer"> Version 5 Jan 2024
 </div>
 
 {% include 'taskdatabase/refresh.html' %}

atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html

+<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>

atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html

+
+
+        <div class="card">
+         <div class="card-body">
+           <table>
+
+             <tr>
+
+                 {% if request.session.task_filter == "all" %}
+                    <td>Click to Filter</td>
+                 {% else %}
+                    <td>Click to Filter ({{request.session.task_filter}})</td>
+                 {% endif %}
+             </tr>
+
+             <tr>
+               <td>
+                   {% include "taskdatabase/quality/clear_filter_button.html" %}
+                    <a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued</a>
+                    <a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+
+               </td>
+                 <td>
+                     {% include 'taskdatabase/filter/search.html' %}
+                 </td>
+             </tr>
+
+           </table>
+         </div>
+        </div>
+
+

atdb/taskdatabase/templates/taskdatabase/ingest/headers.html

+<tr>
+
+    <th>
+        <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Workflow
+        <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>Status</th>
+    <th>
+        <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Project
+        <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Filter
+        <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        SAS_ID (input)
+        <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>Completion</th>
+    <th>
+        SAS_ID (output) at LTA
+    </th>
+
+</tr>
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/ingest/page.html

+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+      <div class="card">
+       <div class="card-body">
+        <div class="row">
+            <div class="col-12">
+                <h3>Ingest Queue</h3>
+
+                These are the SAS_ID's queued for ingest (<i>scrubbed</i>), <i>archiving</i> or <i>archived</i> into the LTA.
+                <hr>
+                {% include 'taskdatabase/ingest/filter_buttons.html' %}&nbsp;
+
+            </div>
+        </div>
+        <div class="row">
+        <div class="col-sm-12 col-md-12 col-lg-12">
+            {% include 'taskdatabase/pagination.html' %}
+            &nbsp;
+            {% if my_tasks %}
+            <div class="panel panel-success">
+
+                <div class="panel-body">
+
+                    <table class="table table-striped table-bordered table-sm">
+                        <thead>
+                        {% include 'taskdatabase/ingest/headers.html' %}
+                        </thead>
+                        <tbody>
+                        {% include 'taskdatabase/ingest/tasks.html' %}
+                        </tbody>
+                    </table>
+                    {% else %}
+                    <p>No recent Tasks.</p>
+                    {% endif %}
+
+                </div>
+            </div>
+            {% include 'taskdatabase/pagination.html' %}
+           </div>
+        </div>
+       </div>
+      </div>
+</div>
+
+{% include 'taskdatabase/no_refresh.html' %}
+
+
+{% endblock %}
+

atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html

+{% load static %}
+{% for task in my_tasks %}
+
+        {% if task.status != "removed_invisible" %}
+        <div class="row">
+            <tr>
+
+                <td>
+                <a class="open-modal btn btn-primary btn-sm"
+                       href="{% url 'workflow-details' task.workflow.id %}"
+                       data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+                       target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
+                    </a></td>
+                </td>
+                <td>{{ task.sasid_ingested_fraction.status }}</td>
+                <td>{{ task.project }}</td>
+
+                <td>{{ task.filter }} </td>
+                <td>{{ task.sas_id }}</td>
+                <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+                <td>
+                    {% if task.sas_id_archived != None %}
+                      <a href={{ task.path_to_lta }} target="_blank">
+                          <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+                          {{ task.sas_id_archived }}
+                      </a>&nbsp;
+                    {% else %}
+                    -
+                    {% endif %}
+                </td>
+           </tr>
+        </div>
+        {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal_no_close.html" %}
\ No newline at end of file

atdb/taskdatabase/urls.py

@@ -19,6 +19,7 @@ urlpatterns = [
     path('validation', views.ShowValidationPage.as_view(), name='validation'),
     path('failures', views.ShowFailuresPage.as_view(), name='failures'),
     path('discarded', views.ShowDiscardedPage.as_view(), name='discarded'),
+    path('ingest', views.ShowIngestPage.as_view(), name='ingest'),
     path('finished', views.ShowFinishedPage.as_view(), name='finished'),
 
     path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),

atdb/taskdatabase/views.py

@@ -335,20 +335,12 @@ class ShowValidationPage(ListView):
     #@silk_profile(name='ShowValidationPage')
     def get_queryset(self):
 
-        # check filtered_tasks on the session
-        # if it is at its max, then it is not a query targeted at 1 SAS_ID.
-        # in that case clear the filter so that all the SAS_ID's show up on the validation page.
-        #if self.request.session['filtered_tasks_as_list']:
-        #    filtered_tasks_on_session = len(self.request.session['filtered_tasks_as_list'])
-        #    if filtered_tasks_on_session == settings.QUERY_LIMIT_MULTI_CHANGE:
-        #        self.request.session['filtered_tasks_as_list'] = []
-
         tasks = get_filtered_tasks(self.request, None, "sas_id").filter(status__icontains=State.STORED.value)
 
         # exclude the failed tasks
         tasks = tasks.exclude(status__icontains=State.FAILED.value)
 
-        paginator = Paginator(tasks, config.TASKS_PER_PAGE)  # Show 50 tasks per page
+        paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL)  # Show 50 tasks per page
         page = self.request.GET.get('page')
 
         try:
@@ -385,11 +377,9 @@ class ShowFailuresPage(ListView):
     #@silk_profile(name='ShowFailuresPage')
     def get_queryset(self):
 
-        #failed_tasks = Task.objects.filter(status__icontains=State.FAILED.value)
-        #tasks = get_filtered_tasks(self.request, failed_tasks)
         tasks = get_filtered_tasks(self.request).filter(status__icontains=State.FAILED.value)
 
-        paginator = Paginator(tasks, config.TASKS_PER_PAGE)  # Show 50 tasks per page
+        paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL)  # Show 50 tasks per page
         page = self.request.GET.get('page')
 
         try:
@@ -452,6 +442,50 @@ class ShowDiscardedPage(ListView):
         return tasks
 
 
+class ShowIngestPage(ListView):
+    """
+     This shows aggregated tasks per sas_id that are queued for ingest or archiving
+     Note that the global filter is also applied
+     """
+    template_name = 'taskdatabase/ingest/page.html'
+    context_object_name = 'my_tasks'
+
+    # @silk_profile(name='ShowIngestPage')
+    def get_queryset(self):
+
+        ingest_tasks = Task.objects.filter(
+            Q(status__icontains=State.SCRUBBED.value) |
+            Q(status__icontains=State.ARCHIVING.value) |
+            Q(status__icontains=State.ARCHIVED.value))
+
+        tasks = get_filtered_tasks(self.request, ingest_tasks, "sas_id")
+
+        # exclude the failed tasks
+        tasks = tasks.exclude(status__icontains=State.FAILED.value)
+
+        paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL)  # Show 10 tasks per page
+        page = self.request.GET.get('page')
+
+        try:
+            # check if there was a page on the session, if so, use it.
+            if page == None:
+                page = self.request.session['page']
+                self.request.session['page'] = None
+        except:
+            pass
+
+        try:
+            tasks = paginator.page(page)
+        except PageNotAnInteger:
+            # If page is not an integer, deliver first page.
+            tasks = paginator.page(1)
+        except EmptyPage:
+            # If page is out of range (e.g. 9999), deliver last page of results.
+            tasks = paginator.page(paginator.num_pages)
+
+        return tasks
+
+
 class ShowFinishedPage(ListView):
     """
     This shows the tasks that are archived
@@ -1282,6 +1316,8 @@ def TaskSetFilter(request, filter, redirect_to_page):
 
     if redirect_to_page == 'quality':
         return redirect('quality')
+    if redirect_to_page == 'ingest':
+        return redirect('ingest')
 
     return redirect_with_params('index', '?page=1')
 
@@ -1311,12 +1347,15 @@ def TaskClearFilter(request, redirect_to_page):
     request.session['search_box'] = ''
     request.session['filtered'] = False
 
+
     if redirect_to_page == 'quality':
         return redirect('quality')
     if redirect_to_page == 'query':
         return redirect('query')
     if redirect_to_page == 'discarded':
         return redirect('discarded')
+    if redirect_to_page == 'ingest':
+        return redirect('ingest')
     return redirect_with_params('index', '?page=1')