experimenting with activities

atdb/atdb/settings/base.py

@@ -209,7 +209,7 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto
 STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
 AGGREGATES = ['failed','active','total']
 
-QUERY_LIMIT_MULTI_CHANGE = 10000
+QUERY_LIMIT_MULTI_CHANGE = 5000
 MAX_MONITORING_HISTORY_HOURS = 7 * 24
 SERVICES_LATE_WARNING_SECONDS = 1800
 

atdb/taskdatabase/templates/astronauth/navbar.html

@@ -29,6 +29,7 @@
                 <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
                 <li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
                 <li><a class="nav-link" href="{% url 'ingest' %}">IngestQ</a></li>
+                <li><a class="nav-link" href="{% url 'ingest2' %}">IngestQ2</a></li>
                 <li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
                 <li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
                 <li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>

atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html

+{% load static %}
+{% for activity in my_activities %}
+        {% if activity.status != "removed_invisible" %}
+        <div class="row">
+            <tr>
+                <td>{{ activity.sas_id }}</td>
+                <td>{{ activity.project }}</td>
+                <td>
+                <a class="open-modal btn btn-primary btn-sm"
+                       href="{% url 'workflow-details' activity.workflow.id %}"
+                       data-popup-url="{% url 'workflow-details' activity.workflow.id %}"
+                       target="_blank"><i class="fas fa-project-diagram"></i> {{ activity.workflow.id }}
+                    </a></td>
+                </td>
+                 <td>{{ activity.filter }} </td>
+                <td>
+                {% if user.is_authenticated %}
+                    <a href="{% url 'task-change-priority-sasid' activity.sas_id '-10' my_activities.number %}" class="btn btn-warning btn-sm" role="button">-10</a>
+                {% endif %}
+                {{ task.priority }}
+                {% if user.is_authenticated %}
+                    <a href="{% url 'task-change-priority-sasid' activity.sas_id '10' my_activities.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
+                {% endif %}
+                </td>
+                <td>{{ activity.ingestq_status }}</td>
+
+                <td>{{ activity.ingested_fraction }}%</td>
+                <td>
+                    {% if activity.has_archived != None %}
+                      <a href={{ activity.archive.path_to_lta }} target="_blank">
+                          <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+                          {{ activity.has_archived }}
+                      </a>&nbsp;
+                    {% else %}
+                    -
+                    {% endif %}
+                </td>
+           </tr>
+        </div>
+        {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal_no_close.html" %}
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html

+<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>

atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html

+
+
+        <div class="card">
+         <div class="card-body">
+           <table>
+
+             <tr>
+
+                 {% if request.session.task_filter == "all" %}
+                    <td>Click to Filter</td>
+                 {% else %}
+                    <td>Click to Filter ({{request.session.task_filter}})</td>
+                 {% endif %}
+             </tr>
+
+             <tr>
+               <td>
+                   {% include "taskdatabase/ingest/clear_filter_button.html" %}
+                    <a href="{% url 'task-set-ingest-filter' 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
+                    <a href="{% url 'task-set-ingest-filter' 'archiving' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+
+               </td>
+                 <td>
+                     {% include 'taskdatabase/filter/search.html' %}
+                 </td>
+             </tr>
+
+           </table>
+         </div>
+        </div>
+
+

atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html

+<tr>
+    <th>
+        <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        SAS_ID (input)
+        <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Project
+        <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Workflow
+        <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Filter
+        <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Priority
+        <a href="{% url 'sort-tasks' 'priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>Status</th>
+
+    <th>Completion</th>
+    <th>
+        SAS_ID (output) at LTA
+    </th>
+
+</tr>
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/ingest2/page.html

+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+      <div class="card">
+       <div class="card-body">
+        <div class="row">
+            <div class="col-12">
+                <h3>Ingest Queue</h3>
+
+                The ingest queue shows SASids with archiving and queued (scrubbed) tasks which are ingesting into the LTA
+
+                {% include 'taskdatabase/ingest2/filter_buttons.html' %}&nbsp;
+
+            </div>
+        </div>
+        <div class="row">
+        <div class="col-sm-12 col-md-12 col-lg-12">
+            {% include 'taskdatabase/pagination.html' %}
+            &nbsp;
+            {% if my_activities %}
+            <div class="panel panel-success">
+
+                <div class="panel-body">
+
+                    <table class="table table-striped table-bordered table-sm">
+                        <thead>
+                        {% include 'taskdatabase/ingest2/headers.html' %}
+                        </thead>
+                        <tbody>
+                        {% include 'taskdatabase/ingest2/activities.html' %}
+                        </tbody>
+                    </table>
+                    {% else %}
+                    <p>No Activities.</p>
+                    {% endif %}
+
+                </div>
+            </div>
+            {% include 'taskdatabase/pagination.html' %}
+           </div>
+        </div>
+       </div>
+      </div>
+</div>
+
+{% include 'taskdatabase/no_refresh.html' %}
+
+
+{% endblock %}
+

atdb/taskdatabase/urls.py

@@ -20,6 +20,7 @@ urlpatterns = [
     path('failures', views.ShowFailuresPage.as_view(), name='failures'),
     path('discarded', views.ShowDiscardedPage.as_view(), name='discarded'),
     path('ingest', views.ShowIngestQPage.as_view(), name='ingest'),
+    path('ingest2', views.ShowIngestQPage2.as_view(), name='ingest2'),
     path('finished', views.ShowFinishedPage.as_view(), name='finished'),
 
     path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),

atdb/taskdatabase/views.py

@@ -101,6 +101,8 @@ class ActivityFilter(filters.FilterSet):
             'project': ['exact', 'icontains'],
             'sas_id': ['exact', 'icontains', 'in'],
             'status': ['exact', 'icontains', 'in', 'startswith'],
+            'ingestq_status': ['icontains'],
+            'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'],
         }
 
 
@@ -497,6 +499,43 @@ class ShowIngestQPage(ListView):
 
         return tasks
 
+class ShowIngestQPage2(ListView):
+    """
+     This shows aggregated tasks per sas_id that are queued for ingest or archiving
+     Note that the global filter is also applied
+     """
+    template_name = 'taskdatabase/ingest2/page.html'
+    context_object_name = 'my_activities'
+
+    def get_queryset(self):
+
+        activities = Activity.objects.filter(
+            Q(ingestq_status__icontains=State.SCRUBBED.value) |
+            (Q(ingested_fraction__gt=0) & Q(ingested_fraction__lt=100) ))
+
+        # exclude the failed tasks
+        paginator = Paginator(activities, config.TASKS_PER_PAGE_SMALL)  # Show 10 tasks per page
+        page = self.request.GET.get('page')
+
+        try:
+            # check if there was a page on the session, if so, use it.
+            if page == None:
+                page = self.request.session['page']
+                self.request.session['page'] = None
+        except:
+            pass
+
+        try:
+            activities = paginator.page(page)
+        except PageNotAnInteger:
+            # If page is not an integer, deliver first page.
+            activities = paginator.page(1)
+        except EmptyPage:
+            # If page is out of range (e.g. 9999), deliver last page of results.
+            activities = paginator.page(paginator.num_pages)
+
+        return activities
+
 
 class ShowFinishedPage(ListView):
     """