Merge branch 'master' into upgrade-to-django-4

# Conflicts:
#	atdb/taskdatabase/templates/taskdatabase/index.html

atdb/atdb/settings/dev.py

@@ -13,7 +13,7 @@ DATABASES = {
          'ENGINE': 'django.db.backends.postgresql_psycopg2',
          'USER': 'atdb_admin',
          'PASSWORD': 'atdb123',
-         'NAME': 'atdb_ldv_20nov2023',
+         'NAME': 'atdb_ldv_12jan2024',
          'HOST': 'localhost',
          'PORT': '5432',
     },

atdb/run_8000.bat

-SET DEBUG=True
-SET DATABASE_HOST=localhost
-SET DATABASE_PORT=5432
-SET DATABASE_NAME=atdb_ldv
-SET DATABASE_USER=atdb_admin
-SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
-SET LOGIN_REDIRECT_URL=/atdb/
-
-python manage.py runserver --settings=atdb.settings.dev

atdb/run_8001.bat

-SET DEBUG=True
-SET DATABASE_HOST=localhost
-SET DATABASE_PORT=5432
-SET DATABASE_NAME=atdb_ldv
-SET DATABASE_USER=atdb_admin
-SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
-SET LOGIN_REDIRECT_URL=/atdb/
-python manage.py runserver 0.0.0.0:8001 --settings=atdb.settings.dev

atdb/taskdatabase/config.py

-VERSION = "Version 1.0.0 (14 jan 2020)"
 TASKS_PER_PAGE = 50
+TASKS_PER_PAGE_SMALL = 10
 

atdb/taskdatabase/models.py

@@ -21,8 +21,10 @@ class State(Enum):
     STORED = 'stored'
     VALIDATED = "validated"
     SCRUBBED = "scrubbed"
+    ARCHIVING = "archiving"
     ARCHIVED = "archived"
     FINISHED = "finished"
+    FINISHING = "finishing"
     SUSPENDED = "suspended"
     DISCARDED = "discarded"
     FAILED = "failed"
@@ -291,6 +293,7 @@ class Task(models.Model):
         except:
             return None
 
+
     @property
     def sasid_is_verified(self):
         for task in Task.objects.filter(sas_id=self.sas_id):
@@ -324,6 +327,42 @@ class Task(models.Model):
 
         return finished
 
+    @property
+    def sasid_ingested_fraction(self):
+        """
+        This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
+        and a list of statusses of other tasks belonging to the same SAS_ID.
+        It is implemented as 'property', because then it can be used in html pages like this:
+           <td>{{ task.sasid_ingested_fraction.status }}</td>
+           <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+
+        A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
+        The division of those 2 are returned as a 'completed %'.
+        A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
+
+        """
+        result = {}
+        statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
+                     'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
+
+        tasks = Task.objects.filter(sas_id=self.sas_id)
+
+        for task in tasks:
+            try:
+                statusses[task.status] += 1
+            except:
+                pass
+
+        incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
+                     int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
+        complete = int(statusses['archived']) + int(statusses['finished'])
+        completion = round(complete / (incomplete + complete) * 100)
+
+        non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
+
+        result['status'] = non_zero_statusses
+        result['completion'] = completion
+        return result
 
     @property
     def task_type_join(self):

atdb/taskdatabase/services/calculated_qualities.py

@@ -125,6 +125,15 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
         # store the result in task.calculated_qualities (not yet saved in the database)
         qualities['per_sasid'] = calculated_quality_sasid
 
+        # save the new sas_id quality for all the other tasks (siblings) of this sas_id as well
+        for other_task in tasks_for_this_sasid:
+            # the task for which the quality is (re)calculated is saved later, but do save all its siblings
+            if other_task.id != task.id:
+                calc_q = other_task.calculated_qualities
+                calc_q['per_sasid'] = calculated_quality_sasid
+                other_task.calculated_qualities = calc_q
+                other_task.save()
+
     except Exception as error:
         logger.error(error)
 

atdb/taskdatabase/templates/astronauth/navbar.html

@@ -28,6 +28,7 @@
 
                 <li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
                 <li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
+                <li><a class="nav-link" href="{% url 'ingest' %}">IngestQ</a></li>
                 <li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
                 <li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
                 <li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>

atdb/taskdatabase/templates/taskdatabase/archived/clear_filter_button.html

+
+   <a href="{% url 'clear-filter' 'finished' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
+

atdb/taskdatabase/templates/taskdatabase/archived/filter_buttons.html

+
+
+        <div class="card">
+         <div class="card-body">
+           <table>
+
+             <tr>
+               <td>
+                   {% include "taskdatabase/archived/clear_filter_button.html" %}
+               </td>
+                 <td>
+                     {% include 'taskdatabase/filter/search.html' %}
+                 </td>
+             </tr>
+
+           </table>
+         </div>
+        </div>
+
+

atdb/taskdatabase/templates/taskdatabase/archived/page.html

@@ -10,8 +10,8 @@
             <div class="col-12">
                 <h3>Finished</h3>
                 This overview shows the tasks that are <b>finished</b> and ingested into the LTA.
-                <hr>
-                <td>{% include 'taskdatabase/filter/search.html' %}</td>
+
+                {% include 'taskdatabase/archived/filter_buttons.html' %}&nbsp;
             </div>
         </div>
         <div class="row">

atdb/taskdatabase/templates/taskdatabase/failures/clear_filter_button.html

+
+   <a href="{% url 'clear-filter' 'failures' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
+

atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html

+
+
+        <div class="card">
+         <div class="card-body">
+           <table>
+
+             <tr>
+
+                 {% if request.session.task_filter == "all" %}
+                    <td>Click to Filter</td>
+                 {% else %}
+                    <td>Click to Filter ({{request.session.task_filter}})</td>
+                 {% endif %}
+             </tr>
+
+             <tr>
+               <td>
+                   {% include "taskdatabase/failures/clear_filter_button.html" %}
+                    <a href="{% url 'task-set-filter' 'processed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed</a>
+                    <a href="{% url 'task-set-filter' 'stored' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored</a>
+                    <a href="{% url 'task-set-filter' 'scrubbing' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing</i></a>
+                    <a href="{% url 'task-set-filter' 'scrubbed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed</a>
+                    <a href="{% url 'task-set-filter' 'archiving' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving</i></a>
+                    <a href="{% url 'task-set-filter' 'archived' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived</a>
+                    <a href="{% url 'task-set-filter' 'finished' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished</a>
+
+               </td>
+                 <td>
+                     {% include 'taskdatabase/filter/search.html' %}
+                 </td>
+             </tr>
+
+           </table>
+         </div>
+        </div>
+
+

atdb/taskdatabase/templates/taskdatabase/failures/page.html

@@ -11,8 +11,9 @@
                 <h3>Failures</h3>
                 These are all the tasks that <b>failed</b>.
                 Click 'Retry' to restart the this step in the workflow (see 'Diagram' in top menu).
-                <td>{% include 'taskdatabase/filter/search.html' %}</td>
-                <hr>
+
+                {% include 'taskdatabase/failures/filter_buttons.html' %}&nbsp;
+
             </div>
         </div>
         <div class="row">

atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html

+<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>

atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html

+
+
+        <div class="card">
+         <div class="card-body">
+           <table>
+
+             <tr>
+
+                 {% if request.session.task_filter == "all" %}
+                    <td>Click to Filter</td>
+                 {% else %}
+                    <td>Click to Filter ({{request.session.task_filter}})</td>
+                 {% endif %}
+             </tr>
+
+             <tr>
+               <td>
+                   {% include "taskdatabase/ingest/clear_filter_button.html" %}
+                    <a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
+                    <a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+
+               </td>
+                 <td>
+                     {% include 'taskdatabase/filter/search.html' %}
+                 </td>
+             </tr>
+
+           </table>
+         </div>
+        </div>
+
+

atdb/taskdatabase/templates/taskdatabase/ingest/headers.html

+<tr>
+    <th>
+        <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        SAS_ID (input)
+        <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Project
+        <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Workflow
+        <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Filter
+        <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>
+        <a href="{% url 'sort-tasks' '-priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+        Priority
+        <a href="{% url 'sort-tasks' 'priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+    </th>
+    <th>Status</th>
+
+    <th>Completion</th>
+    <th>
+        SAS_ID (output) at LTA
+    </th>
+
+</tr>
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/ingest/page.html

+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+      <div class="card">
+       <div class="card-body">
+        <div class="row">
+            <div class="col-12">
+                <h3>Ingest Queue</h3>
+
+                The ingest queue shows SASids with archiving and queued (scrubbed) tasks which are ingesting into the LTA
+
+                {% include 'taskdatabase/ingest/filter_buttons.html' %}&nbsp;
+
+            </div>
+        </div>
+        <div class="row">
+        <div class="col-sm-12 col-md-12 col-lg-12">
+            {% include 'taskdatabase/pagination.html' %}
+            &nbsp;
+            {% if my_tasks %}
+            <div class="panel panel-success">
+
+                <div class="panel-body">
+
+                    <table class="table table-striped table-bordered table-sm">
+                        <thead>
+                        {% include 'taskdatabase/ingest/headers.html' %}
+                        </thead>
+                        <tbody>
+                        {% include 'taskdatabase/ingest/tasks.html' %}
+                        </tbody>
+                    </table>
+                    {% else %}
+                    <p>No recent Tasks.</p>
+                    {% endif %}
+
+                </div>
+            </div>
+            {% include 'taskdatabase/pagination.html' %}
+           </div>
+        </div>
+       </div>
+      </div>
+</div>
+
+{% include 'taskdatabase/no_refresh.html' %}
+
+
+{% endblock %}
+

atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html

+{% load static %}
+{% for task in my_tasks %}
+
+        {% if task.status != "removed_invisible" %}
+        <div class="row">
+            <tr>
+                <td>{{ task.sas_id }}</td>
+                <td>{{ task.project }}</td>
+                <td>
+                <a class="open-modal btn btn-primary btn-sm"
+                       href="{% url 'workflow-details' task.workflow.id %}"
+                       data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+                       target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
+                    </a></td>
+                </td>
+                 <td>{{ task.filter }} </td>
+                <td>
+                {% if user.is_authenticated %}
+                    <a href="{% url 'task-change-priority-sasid' task.pk '-10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">-10</a>
+                {% endif %}
+                {{ task.priority }}
+                {% if user.is_authenticated %}
+                    <a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
+                {% endif %}
+                </td>
+                <td>{{ task.sasid_ingested_fraction.status }}</td>
+
+                <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+                <td>
+                    {% if task.sas_id_archived != None %}
+                      <a href={{ task.path_to_lta }} target="_blank">
+                          <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+                          {{ task.sas_id_archived }}
+                      </a>&nbsp;
+                    {% else %}
+                    -
+                    {% endif %}
+                </td>
+           </tr>
+        </div>
+        {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal_no_close.html" %}
\ No newline at end of file

atdb/taskdatabase/templates/taskdatabase/validation/clear_filter_button.html

+
+   <a href="{% url 'clear-filter' 'validation' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
+