diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py index 814f2d8b322a4388b52ad35d0e10af630f78c616..5511fbbd4269d57ba6b2acb32aeaedd459b97f1e 100644 --- a/atdb/atdb/settings/base.py +++ b/atdb/atdb/settings/base.py @@ -209,7 +209,7 @@ ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','sto STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived'] AGGREGATES = ['failed','active','total'] -QUERY_LIMIT_MULTI_CHANGE = 10000 +QUERY_LIMIT_MULTI_CHANGE = 5000 MAX_MONITORING_HISTORY_HOURS = 7 * 24 SERVICES_LATE_WARNING_SECONDS = 1800 diff --git a/atdb/taskdatabase/templates/astronauth/navbar.html b/atdb/taskdatabase/templates/astronauth/navbar.html index 944ae733ac31f03aa48347081ebbd43ba7ce316a..28b693bcd5dcbe93537d2c711cf17d5fe84ca4ee 100644 --- a/atdb/taskdatabase/templates/astronauth/navbar.html +++ b/atdb/taskdatabase/templates/astronauth/navbar.html @@ -29,6 +29,7 @@ <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li> <li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li> <li><a class="nav-link" href="{% url 'ingest' %}">IngestQ</a></li> + <li><a class="nav-link" href="{% url 'ingest2' %}">IngestQ2</a></li> <li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li> <li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li> <li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li> diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html new file mode 100644 index 0000000000000000000000000000000000000000..acc7ff8c32c4566e9f8685938cee719dc303ad71 --- /dev/null +++ b/atdb/taskdatabase/templates/taskdatabase/ingest2/activities.html @@ -0,0 +1,45 @@ +{% load static %} +{% for activity in my_activities %} + {% if activity.status != "removed_invisible" %} + <div class="row"> + <tr> + <td>{{ activity.sas_id }}</td> + <td>{{ activity.project }}</td> + <td> + <a class="open-modal btn btn-primary btn-sm" + href="{% url 'workflow-details' activity.workflow.id %}" + data-popup-url="{% url 'workflow-details' activity.workflow.id %}" + target="_blank"><i class="fas fa-project-diagram"></i> {{ activity.workflow.id }} + </a></td> + </td> + <td>{{ activity.filter }} </td> + <td> + {% if user.is_authenticated %} + <a href="{% url 'task-change-priority-sasid' activity.sas_id '-10' my_activities.number %}" class="btn btn-warning btn-sm" role="button">-10</a> + {% endif %} + {{ task.priority }} + {% if user.is_authenticated %} + <a href="{% url 'task-change-priority-sasid' activity.sas_id '10' my_activities.number %}" class="btn btn-warning btn-sm" role="button">+10</a> + {% endif %} + </td> + <td>{{ activity.ingestq_status }}</td> + + <td>{{ activity.ingested_fraction }}%</td> + <td> + {% if activity.has_archived != None %} + <a href={{ activity.archive.path_to_lta }} target="_blank"> + <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA"> + {{ activity.has_archived }} + </a> + {% else %} + - + {% endif %} + </td> + </tr> + </div> + {% endif %} + +{% endfor %} + +{% include "taskdatabase/modal/modal_script.html" %} +{% include "taskdatabase/modal/modal_no_close.html" %} \ No newline at end of file diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html new file mode 100644 index 0000000000000000000000000000000000000000..06d07ae139ed3196fa0ed99a9e9397d68b816aa6 --- /dev/null +++ b/atdb/taskdatabase/templates/taskdatabase/ingest2/clear_filter_button.html @@ -0,0 +1 @@ +<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a> diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html new file mode 100644 index 0000000000000000000000000000000000000000..118ccd3c2f664f1f55c376cb42a9ffe29de95fdd --- /dev/null +++ b/atdb/taskdatabase/templates/taskdatabase/ingest2/filter_buttons.html @@ -0,0 +1,32 @@ + + + <div class="card"> + <div class="card-body"> + <table> + + <tr> + + {% if request.session.task_filter == "all" %} + <td>Click to Filter</td> + {% else %} + <td>Click to Filter ({{request.session.task_filter}})</td> + {% endif %} + </tr> + + <tr> + <td> + {% include "taskdatabase/ingest/clear_filter_button.html" %} + <a href="{% url 'task-set-ingest-filter' 'scrubbed' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a> + <a href="{% url 'task-set-ingest-filter' 'archiving' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a> + + </td> + <td> + {% include 'taskdatabase/filter/search.html' %} + </td> + </tr> + + </table> + </div> + </div> + + diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html new file mode 100644 index 0000000000000000000000000000000000000000..a36591940f65a28de895445bf6232941cb864bda --- /dev/null +++ b/atdb/taskdatabase/templates/taskdatabase/ingest2/headers.html @@ -0,0 +1,34 @@ +<tr> + <th> + <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a> + SAS_ID (input) + <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a> + </th> + <th> + <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a> + Project + <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a> + </th> + <th> + <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a> + Workflow + <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a> + </th> + <th> + <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a> + Filter + <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a> + </th> + <th> + <a href="{% url 'sort-tasks' '-priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a> + Priority + <a href="{% url 'sort-tasks' 'priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a> + </th> + <th>Status</th> + + <th>Completion</th> + <th> + SAS_ID (output) at LTA + </th> + +</tr> \ No newline at end of file diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest2/page.html b/atdb/taskdatabase/templates/taskdatabase/ingest2/page.html new file mode 100644 index 0000000000000000000000000000000000000000..60539738958429fecee077cd146b3cdbcdad65d2 --- /dev/null +++ b/atdb/taskdatabase/templates/taskdatabase/ingest2/page.html @@ -0,0 +1,54 @@ +{% extends 'taskdatabase/base.html' %} +{% load static %} + +{% block myBlock %} + +<div class="container-fluid details-container"> + + <div class="card"> + <div class="card-body"> + <div class="row"> + <div class="col-12"> + <h3>Ingest Queue</h3> + + The ingest queue shows SASids with archiving and queued (scrubbed) tasks which are ingesting into the LTA + + {% include 'taskdatabase/ingest2/filter_buttons.html' %} + + </div> + </div> + <div class="row"> + <div class="col-sm-12 col-md-12 col-lg-12"> + {% include 'taskdatabase/pagination.html' %} + + {% if my_activities %} + <div class="panel panel-success"> + + <div class="panel-body"> + + <table class="table table-striped table-bordered table-sm"> + <thead> + {% include 'taskdatabase/ingest2/headers.html' %} + </thead> + <tbody> + {% include 'taskdatabase/ingest2/activities.html' %} + </tbody> + </table> + {% else %} + <p>No Activities.</p> + {% endif %} + + </div> + </div> + {% include 'taskdatabase/pagination.html' %} + </div> + </div> + </div> + </div> +</div> + +{% include 'taskdatabase/no_refresh.html' %} + + +{% endblock %} + diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py index 9ee0ea9c7ae4a143f62795a032dd35dcd96e84cc..7fc2d5966d2c7784ab72ad71eb38af862f7f1d45 100644 --- a/atdb/taskdatabase/urls.py +++ b/atdb/taskdatabase/urls.py @@ -20,6 +20,7 @@ urlpatterns = [ path('failures', views.ShowFailuresPage.as_view(), name='failures'), path('discarded', views.ShowDiscardedPage.as_view(), name='discarded'), path('ingest', views.ShowIngestQPage.as_view(), name='ingest'), + path('ingest2', views.ShowIngestQPage2.as_view(), name='ingest2'), path('finished', views.ShowFinishedPage.as_view(), name='finished'), path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'), diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py index f6325cc5abac438ea88e31ca55d1affb11b571ab..c41456585e3ec64d8ea66955638a7c3edc2b6e9f 100644 --- a/atdb/taskdatabase/views.py +++ b/atdb/taskdatabase/views.py @@ -101,6 +101,8 @@ class ActivityFilter(filters.FilterSet): 'project': ['exact', 'icontains'], 'sas_id': ['exact', 'icontains', 'in'], 'status': ['exact', 'icontains', 'in', 'startswith'], + 'ingestq_status': ['icontains'], + 'ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte'], } @@ -497,6 +499,43 @@ class ShowIngestQPage(ListView): return tasks +class ShowIngestQPage2(ListView): + """ + This shows aggregated tasks per sas_id that are queued for ingest or archiving + Note that the global filter is also applied + """ + template_name = 'taskdatabase/ingest2/page.html' + context_object_name = 'my_activities' + + def get_queryset(self): + + activities = Activity.objects.filter( + Q(ingestq_status__icontains=State.SCRUBBED.value) | + (Q(ingested_fraction__gt=0) & Q(ingested_fraction__lt=100) )) + + # exclude the failed tasks + paginator = Paginator(activities, config.TASKS_PER_PAGE_SMALL) # Show 10 tasks per page + page = self.request.GET.get('page') + + try: + # check if there was a page on the session, if so, use it. + if page == None: + page = self.request.session['page'] + self.request.session['page'] = None + except: + pass + + try: + activities = paginator.page(page) + except PageNotAnInteger: + # If page is not an integer, deliver first page. + activities = paginator.page(1) + except EmptyPage: + # If page is out of range (e.g. 9999), deliver last page of results. + activities = paginator.page(paginator.num_pages) + + return activities + class ShowFinishedPage(ListView): """