diff --git a/atdb/atdb/settings/dev.py b/atdb/atdb/settings/dev.py
index 6467a72df42559cd143c5c9bc97ff9736da5b347..32433ae2ce74beb62caaf4806f14d2cd8a6e570d 100644
--- a/atdb/atdb/settings/dev.py
+++ b/atdb/atdb/settings/dev.py
@@ -13,7 +13,7 @@ DATABASES = {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'USER': 'atdb_admin',
'PASSWORD': 'atdb123',
- 'NAME': 'atdb_ldv_20nov2023',
+ 'NAME': 'atdb_ldv_12jan2024',
'HOST': 'localhost',
'PORT': '5432',
},
diff --git a/atdb/docs/ATDB-LDV GUI.png b/atdb/docs/ATDB-LDV GUI.png
index fdeaf2e5ff4a6bb0856e4016ace4de329e89ff8f..2010d22a5113da517417fd1d2ee0a3ce7c6fa45f 100644
Binary files a/atdb/docs/ATDB-LDV GUI.png and b/atdb/docs/ATDB-LDV GUI.png differ
diff --git a/atdb/run_8000.bat b/atdb/run_8000.bat
deleted file mode 100644
index bb5d4220ee6ece9fc9ca1d749a9fc2871354a327..0000000000000000000000000000000000000000
--- a/atdb/run_8000.bat
+++ /dev/null
@@ -1,12 +0,0 @@
-SET DEBUG=True
-SET DATABASE_HOST=localhost
-SET DATABASE_PORT=5432
-SET DATABASE_NAME=atdb_ldv
-SET DATABASE_USER=atdb_admin
-SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
-SET LOGIN_REDIRECT_URL=/atdb/
-
-python manage.py runserver --settings=atdb.settings.dev
diff --git a/atdb/run_8001.bat b/atdb/run_8001.bat
deleted file mode 100644
index 3571ee3f0c8cd3f49a262fef814964abd6a10977..0000000000000000000000000000000000000000
--- a/atdb/run_8001.bat
+++ /dev/null
@@ -1,11 +0,0 @@
-SET DEBUG=True
-SET DATABASE_HOST=localhost
-SET DATABASE_PORT=5432
-SET DATABASE_NAME=atdb_ldv
-SET DATABASE_USER=atdb_admin
-SET DATABASE_PASSWORD=atdb123
-SET KEYCLOAK_URL=https://keycloak-sdc.astron.nl
-SET KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
-SET KEYCLOAK_CLIENT_SECRET=HA0Jwpdyxx8eriUHDaxhHWIXGKftJtAT
-SET LOGIN_REDIRECT_URL=/atdb/
-python manage.py runserver 0.0.0.0:8001 --settings=atdb.settings.dev
diff --git a/atdb/taskdatabase/config.py b/atdb/taskdatabase/config.py
index f4ff5fc82e25e1c43d478623f7de42f7b063ff53..f50b238accc44b6d61e409e4cf84f09328bd9000 100644
--- a/atdb/taskdatabase/config.py
+++ b/atdb/taskdatabase/config.py
@@ -1,3 +1,3 @@
-VERSION = "Version 1.0.0 (14 jan 2020)"
TASKS_PER_PAGE = 50
+TASKS_PER_PAGE_SMALL = 10
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index 197ff582c9aa7e5d3fa7eac4a99c6221d892ccfb..20b6ee2813528b3130bbf14cc52b1abb3e57a176 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -21,8 +21,10 @@ class State(Enum):
STORED = 'stored'
VALIDATED = "validated"
SCRUBBED = "scrubbed"
+ ARCHIVING = "archiving"
ARCHIVED = "archived"
FINISHED = "finished"
+ FINISHING = "finishing"
SUSPENDED = "suspended"
DISCARDED = "discarded"
FAILED = "failed"
@@ -291,6 +293,7 @@ class Task(models.Model):
except:
return None
+
@property
def sasid_is_verified(self):
for task in Task.objects.filter(sas_id=self.sas_id):
@@ -324,6 +327,42 @@ class Task(models.Model):
return finished
+ @property
+ def sasid_ingested_fraction(self):
+ """
+ This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
+ and a list of statusses of other tasks belonging to the same SAS_ID.
+ It is implemented as 'property', because then it can be used in html pages like this:
+ <td>{{ task.sasid_ingested_fraction.status }}</td>
+ <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+
+ A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
+ The division of those 2 are returned as a 'completed %'.
+ A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
+
+ """
+ result = {}
+ statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
+ 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
+
+ tasks = Task.objects.filter(sas_id=self.sas_id)
+
+ for task in tasks:
+ try:
+ statusses[task.status] += 1
+ except:
+ pass
+
+ incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
+ int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
+ complete = int(statusses['archived']) + int(statusses['finished'])
+ completion = round(complete / (incomplete + complete) * 100)
+
+ non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
+
+ result['status'] = non_zero_statusses
+ result['completion'] = completion
+ return result
@property
def task_type_join(self):
diff --git a/atdb/taskdatabase/services/calculated_qualities.py b/atdb/taskdatabase/services/calculated_qualities.py
index c064d461698adba12d76acefa097d1a35ea5e3f0..b6c72692642be584e366540fbf02fc5d1edfb0d0 100644
--- a/atdb/taskdatabase/services/calculated_qualities.py
+++ b/atdb/taskdatabase/services/calculated_qualities.py
@@ -125,6 +125,15 @@ def calculate_qualities(task, tasks_for_this_sasid, quality_thresholds):
# store the result in task.calculated_qualities (not yet saved in the database)
qualities['per_sasid'] = calculated_quality_sasid
+ # save the new sas_id quality for all the other tasks (siblings) of this sas_id as well
+ for other_task in tasks_for_this_sasid:
+ # the task for which the quality is (re)calculated is saved later, but do save all its siblings
+ if other_task.id != task.id:
+ calc_q = other_task.calculated_qualities
+ calc_q['per_sasid'] = calculated_quality_sasid
+ other_task.calculated_qualities = calc_q
+ other_task.save()
+
except Exception as error:
logger.error(error)
diff --git a/atdb/taskdatabase/templates/astronauth/navbar.html b/atdb/taskdatabase/templates/astronauth/navbar.html
index 02569569352e12bf6af7b0f0dd10e86071ab2bcd..944ae733ac31f03aa48347081ebbd43ba7ce316a 100644
--- a/atdb/taskdatabase/templates/astronauth/navbar.html
+++ b/atdb/taskdatabase/templates/astronauth/navbar.html
@@ -28,6 +28,7 @@
<li><a class="nav-link" href="{% url 'quality' %}">Quality</a></li>
<li><a class="nav-link" href="{% url 'validation' %}">Validation</a></li>
+ <li><a class="nav-link" href="{% url 'ingest' %}">IngestQ</a></li>
<li><a class="nav-link" href="{% url 'failures' %}">Failures</a></li>
<li><a class="nav-link" href="{% url 'discarded' %}">Discarded</a></li>
<li><a class="nav-link" href="{% url 'finished' %}">Finished</a></li>
diff --git a/atdb/taskdatabase/templates/taskdatabase/archived/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/archived/clear_filter_button.html
new file mode 100644
index 0000000000000000000000000000000000000000..b2da212620c1c42046a86495b44414e2a2c906d2
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/archived/clear_filter_button.html
@@ -0,0 +1,3 @@
+
+ <a href="{% url 'clear-filter' 'finished' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/archived/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/archived/filter_buttons.html
new file mode 100644
index 0000000000000000000000000000000000000000..f3439f4aff9852e1e1beaf8aedee6f9492048090
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/archived/filter_buttons.html
@@ -0,0 +1,20 @@
+
+
+ <div class="card">
+ <div class="card-body">
+ <table>
+
+ <tr>
+ <td>
+ {% include "taskdatabase/archived/clear_filter_button.html" %}
+ </td>
+ <td>
+ {% include 'taskdatabase/filter/search.html' %}
+ </td>
+ </tr>
+
+ </table>
+ </div>
+ </div>
+
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/archived/page.html b/atdb/taskdatabase/templates/taskdatabase/archived/page.html
index 6bd1f77d1a453d48bae4a7fd7ffee969929a51e8..27982943c1f2f5f8bff45a04acbf6b319f289a65 100644
--- a/atdb/taskdatabase/templates/taskdatabase/archived/page.html
+++ b/atdb/taskdatabase/templates/taskdatabase/archived/page.html
@@ -10,8 +10,8 @@
<div class="col-12">
<h3>Finished</h3>
This overview shows the tasks that are <b>finished</b> and ingested into the LTA.
- <hr>
- <td>{% include 'taskdatabase/filter/search.html' %}</td>
+
+ {% include 'taskdatabase/archived/filter_buttons.html' %}
</div>
</div>
<div class="row">
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/failures/clear_filter_button.html
new file mode 100644
index 0000000000000000000000000000000000000000..b5ca7f6a4699b40065b73be686699b3a61fb73e9
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/clear_filter_button.html
@@ -0,0 +1,3 @@
+
+ <a href="{% url 'clear-filter' 'failures' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html
new file mode 100644
index 0000000000000000000000000000000000000000..482c707d93fb405cf0e21279dbe658852951fc89
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/filter_buttons.html
@@ -0,0 +1,37 @@
+
+
+ <div class="card">
+ <div class="card-body">
+ <table>
+
+ <tr>
+
+ {% if request.session.task_filter == "all" %}
+ <td>Click to Filter</td>
+ {% else %}
+ <td>Click to Filter ({{request.session.task_filter}})</td>
+ {% endif %}
+ </tr>
+
+ <tr>
+ <td>
+ {% include "taskdatabase/failures/clear_filter_button.html" %}
+ <a href="{% url 'task-set-filter' 'processed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">processed</a>
+ <a href="{% url 'task-set-filter' 'stored' 'failures' %}" class="btn btn-secondary btn-sm" role="button">stored</a>
+ <a href="{% url 'task-set-filter' 'scrubbing' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing</i></a>
+ <a href="{% url 'task-set-filter' 'scrubbed' 'failures' %}" class="btn btn-secondary btn-sm" role="button">scrubbed</a>
+ <a href="{% url 'task-set-filter' 'archiving' 'failures' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving</i></a>
+ <a href="{% url 'task-set-filter' 'archived' 'failures' %}" class="btn btn-secondary btn-sm" role="button">archived</a>
+ <a href="{% url 'task-set-filter' 'finished' 'failures' %}" class="btn btn-secondary btn-sm" role="button">finished</a>
+
+ </td>
+ <td>
+ {% include 'taskdatabase/filter/search.html' %}
+ </td>
+ </tr>
+
+ </table>
+ </div>
+ </div>
+
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/page.html b/atdb/taskdatabase/templates/taskdatabase/failures/page.html
index cc3032c85dd74a1e1adacef56a72680ca7a8412c..161f50b9825c3ee60bf7e4473906c44a284e803f 100644
--- a/atdb/taskdatabase/templates/taskdatabase/failures/page.html
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/page.html
@@ -11,8 +11,9 @@
<h3>Failures</h3>
These are all the tasks that <b>failed</b>.
Click 'Retry' to restart the this step in the workflow (see 'Diagram' in top menu).
- <td>{% include 'taskdatabase/filter/search.html' %}</td>
- <hr>
+
+ {% include 'taskdatabase/failures/filter_buttons.html' %}
+
</div>
</div>
<div class="row">
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html
new file mode 100644
index 0000000000000000000000000000000000000000..06d07ae139ed3196fa0ed99a9e9397d68b816aa6
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/clear_filter_button.html
@@ -0,0 +1 @@
+<a href="{% url 'clear-filter' 'ingest' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
new file mode 100644
index 0000000000000000000000000000000000000000..782d2ec1b8d3cc5d9d4a1f2017873af694b56ef6
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/filter_buttons.html
@@ -0,0 +1,32 @@
+
+
+ <div class="card">
+ <div class="card-body">
+ <table>
+
+ <tr>
+
+ {% if request.session.task_filter == "all" %}
+ <td>Click to Filter</td>
+ {% else %}
+ <td>Click to Filter ({{request.session.task_filter}})</td>
+ {% endif %}
+ </tr>
+
+ <tr>
+ <td>
+ {% include "taskdatabase/ingest/clear_filter_button.html" %}
+ <a href="{% url 'task-set-filter' 'scrubbed' 'ingest' %}" class="btn btn-secondary btn-sm" role="button">Queued (scrubbed)</a>
+ <a href="{% url 'task-set-filter' 'archiving' 'ingest' %}" class="btn btn-secondary btn-sm" role="button"><i>Archiving</i></a>
+
+ </td>
+ <td>
+ {% include 'taskdatabase/filter/search.html' %}
+ </td>
+ </tr>
+
+ </table>
+ </div>
+ </div>
+
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/headers.html b/atdb/taskdatabase/templates/taskdatabase/ingest/headers.html
new file mode 100644
index 0000000000000000000000000000000000000000..a36591940f65a28de895445bf6232941cb864bda
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/headers.html
@@ -0,0 +1,34 @@
+<tr>
+ <th>
+ <a href="{% url 'sort-tasks' '-sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ SAS_ID (input)
+ <a href="{% url 'sort-tasks' 'sas_id' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Project
+ <a href="{% url 'sort-tasks' 'project' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Workflow
+ <a href="{% url 'sort-tasks' 'workflow' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Filter
+ <a href="{% url 'sort-tasks' 'filter' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>
+ <a href="{% url 'sort-tasks' '-priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-up"></i></a>
+ Priority
+ <a href="{% url 'sort-tasks' 'priority' 'ingest' %}" class="btn btn-light btn-sm" role="button"><i class="fas fa-sort-down"></i></a>
+ </th>
+ <th>Status</th>
+
+ <th>Completion</th>
+ <th>
+ SAS_ID (output) at LTA
+ </th>
+
+</tr>
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/page.html b/atdb/taskdatabase/templates/taskdatabase/ingest/page.html
new file mode 100644
index 0000000000000000000000000000000000000000..cfcbcbd895b40c75236f8221e43e603dbd58c86b
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/page.html
@@ -0,0 +1,54 @@
+{% extends 'taskdatabase/base.html' %}
+{% load static %}
+
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+
+ <div class="card">
+ <div class="card-body">
+ <div class="row">
+ <div class="col-12">
+ <h3>Ingest Queue</h3>
+
+ The ingest queue shows SASids with archiving and queued (scrubbed) tasks which are ingesting into the LTA
+
+ {% include 'taskdatabase/ingest/filter_buttons.html' %}
+
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-sm-12 col-md-12 col-lg-12">
+ {% include 'taskdatabase/pagination.html' %}
+
+ {% if my_tasks %}
+ <div class="panel panel-success">
+
+ <div class="panel-body">
+
+ <table class="table table-striped table-bordered table-sm">
+ <thead>
+ {% include 'taskdatabase/ingest/headers.html' %}
+ </thead>
+ <tbody>
+ {% include 'taskdatabase/ingest/tasks.html' %}
+ </tbody>
+ </table>
+ {% else %}
+ <p>No recent Tasks.</p>
+ {% endif %}
+
+ </div>
+ </div>
+ {% include 'taskdatabase/pagination.html' %}
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
+
+{% include 'taskdatabase/no_refresh.html' %}
+
+
+{% endblock %}
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
new file mode 100644
index 0000000000000000000000000000000000000000..1a61c352add2df52faf58fa3b64a6a529f9cd86a
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
@@ -0,0 +1,46 @@
+{% load static %}
+{% for task in my_tasks %}
+
+ {% if task.status != "removed_invisible" %}
+ <div class="row">
+ <tr>
+ <td>{{ task.sas_id }}</td>
+ <td>{{ task.project }}</td>
+ <td>
+ <a class="open-modal btn btn-primary btn-sm"
+ href="{% url 'workflow-details' task.workflow.id %}"
+ data-popup-url="{% url 'workflow-details' task.workflow.id %}"
+ target="_blank"><i class="fas fa-project-diagram"></i> {{ task.workflow.id }}
+ </a></td>
+ </td>
+ <td>{{ task.filter }} </td>
+ <td>
+ {% if user.is_authenticated %}
+ <a href="{% url 'task-change-priority-sasid' task.pk '-10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">-10</a>
+ {% endif %}
+ {{ task.priority }}
+ {% if user.is_authenticated %}
+ <a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
+ {% endif %}
+ </td>
+ <td>{{ task.sasid_ingested_fraction.status }}</td>
+
+ <td>{{ task.sasid_ingested_fraction.completion }}%</td>
+ <td>
+ {% if task.sas_id_archived != None %}
+ <a href={{ task.path_to_lta }} target="_blank">
+ <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
+ {{ task.sas_id_archived }}
+ </a>
+ {% else %}
+ -
+ {% endif %}
+ </td>
+ </tr>
+ </div>
+ {% endif %}
+
+{% endfor %}
+
+{% include "taskdatabase/modal/modal_script.html" %}
+{% include "taskdatabase/modal/modal_no_close.html" %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/clear_filter_button.html b/atdb/taskdatabase/templates/taskdatabase/validation/clear_filter_button.html
new file mode 100644
index 0000000000000000000000000000000000000000..6787f10079150443357d4fdd515a27beb73264f1
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/clear_filter_button.html
@@ -0,0 +1,3 @@
+
+ <a href="{% url 'clear-filter' 'validation' %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-window-close"></i> Clear Filter</a>
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/validation/filter_buttons.html
new file mode 100644
index 0000000000000000000000000000000000000000..54e0fffdaee2fc21ad999d1a78a81ee2293c769b
--- /dev/null
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/filter_buttons.html
@@ -0,0 +1,21 @@
+
+
+ <div class="card">
+ <div class="card-body">
+ <table>
+
+ <tr>
+ <td>
+ {% include "taskdatabase/validation/clear_filter_button.html" %}
+
+ </td>
+ <td>
+ {% include 'taskdatabase/filter/search.html' %}
+ </td>
+ </tr>
+
+ </table>
+ </div>
+ </div>
+
+
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/page.html b/atdb/taskdatabase/templates/taskdatabase/validation/page.html
index b2434b97b96fefe312110eaa38d3dd2bb450985e..195d5557185a9a48a6fca2b79cb799ef805a9591 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/page.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/page.html
@@ -13,9 +13,7 @@
These are the SAS_ID's in status <b>stored</b> that can be validated. Click one of the quality buttons to validate.
- <td>{% include 'taskdatabase/filter/search.html' %}</td>
-
- <hr>
+ {% include 'taskdatabase/validation/filter_buttons.html' %}
</div>
</div>
<div class="row">
diff --git a/atdb/taskdatabase/tests/test_calculated_qualities.py b/atdb/taskdatabase/tests/test_calculated_qualities.py
index 4aa8adf7773f1d0adb97dafa75fb96ec13f73f88..3563d46c082cc8f9a8609f35c1439da6b3fee92d 100644
--- a/atdb/taskdatabase/tests/test_calculated_qualities.py
+++ b/atdb/taskdatabase/tests/test_calculated_qualities.py
@@ -70,11 +70,12 @@ class TestCalculatedQualities(TestCase):
# only 4 of the 7 tasks should now have calculated_qualities
count = 0
- for task in Task.objects.all():
+ tasks = Task.objects.all()
+ for task in tasks:
if task.calculated_qualities['per_sasid']:
count += 1
- self.assertEqual(count,5)
+ self.assertEqual(count,6)
def test_calculated_qualities(self):
diff --git a/atdb/taskdatabase/tests/test_ingest_fraction.py b/atdb/taskdatabase/tests/test_ingest_fraction.py
new file mode 100644
index 0000000000000000000000000000000000000000..b50472c8eb86a5b8f03c9729dba2e1d48471e675
--- /dev/null
+++ b/atdb/taskdatabase/tests/test_ingest_fraction.py
@@ -0,0 +1,32 @@
+from django.test import TestCase
+
+from taskdatabase.models import Task, Workflow
+
+class TestIngestFraction(TestCase):
+
+ def setUp(self):
+ # create a list of Tasks with various values of rfi_percent to test the quality algorithms
+ workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation")
+ workflow_requantisation.save()
+
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='stored', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='archiving', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='archived', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a',sas_id=54321, status='finishing', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a', sas_id=54321, status='finished', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='b',sas_id=54321, status='finished', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a', sas_id=54321, status='discarded', workflow=workflow_requantisation)
+ Task.objects.get_or_create(filter='a', sas_id=54321, status='archived_failed', workflow=workflow_requantisation)
+ def test_ingest_fraction(self):
+
+ # collapse all tasks into a single task for this sas_id
+ task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0]
+
+ # get the list of statusses and level of completion
+ statusses = task.sasid_ingested_fraction['status']
+ completion = task.sasid_ingested_fraction['completion']
+
+ self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1})
+ self.assertEqual(completion,38)
diff --git a/atdb/taskdatabase/tests/test_priority.py b/atdb/taskdatabase/tests/test_priority.py
new file mode 100644
index 0000000000000000000000000000000000000000..26556a17dc6f564abcd879330835340134bd9173
--- /dev/null
+++ b/atdb/taskdatabase/tests/test_priority.py
@@ -0,0 +1,59 @@
+from django.test import TestCase
+from django.urls import reverse
+from django.contrib.auth.models import User
+from django.contrib.auth.decorators import login_required
+from django.test import Client
+
+from taskdatabase.models import Task, Workflow
+
+class ChangePriorityViewTest(TestCase):
+ def setUp(self):
+ # Create a test user
+ self.user = User.objects.create_user(username='testuser', password='testpassword')
+
+ # Create a test workflow
+ workflow_requantisation = Workflow(workflow_uri="psrfits_requantisation")
+ workflow_requantisation.save()
+
+ # Create a test task
+ self.task = Task.objects.create(filter='a',sas_id=54321, status='stored', workflow=workflow_requantisation, priority=1)
+ self.task = Task.objects.create(filter='b',sas_id=54321, status='stored', workflow=workflow_requantisation, priority=1)
+
+
+ # Login the test user
+ self.client = Client()
+ self.client.login(username='testuser', password='testpassword')
+
+ def test_change_priority(self):
+ # Set up the URL for the view
+ url = reverse('task-change-priority', kwargs={'pk': self.task.pk, 'priority_change': 2, 'page': 1})
+
+ # Call the view using the client
+ response = self.client.get(url)
+
+ # Check if the task priority has been updated
+ updated_task = Task.objects.get(pk=self.task.pk)
+ self.assertEqual(updated_task.priority, 3)
+
+
+ def test_change_priority_negative_priority(self):
+ # Set up the URL for the view with a negative priority_change value
+ url = reverse('task-change-priority', kwargs={'pk': self.task.pk, 'priority_change': -2, 'page': 0})
+
+ # Call the view using the client
+ response = self.client.get(url)
+
+ # Check if the task priority is set to 0 when priority becomes negative
+ updated_task = Task.objects.get(pk=self.task.pk)
+ self.assertEqual(updated_task.priority, 0)
+
+ def test_change_priority_sasid(self):
+ # Set up the URL for the view
+ url = reverse('task-change-priority-sasid', kwargs={'pk': self.task.pk, 'priority_change': 2, 'page': 1})
+
+ # Call the view using the client
+ response = self.client.get(url)
+
+ # Check if the task priority has been updated
+ updated_task = Task.objects.get(pk=self.task.pk)
+ self.assertEqual(updated_task.priority, 3)
\ No newline at end of file
diff --git a/atdb/taskdatabase/tests/test_views_ingest_page.py b/atdb/taskdatabase/tests/test_views_ingest_page.py
new file mode 100644
index 0000000000000000000000000000000000000000..7bc76349063a3dd2ca89709b5ef94760b91faa2e
--- /dev/null
+++ b/atdb/taskdatabase/tests/test_views_ingest_page.py
@@ -0,0 +1,31 @@
+from django.test import TestCase
+from django.urls import reverse
+
+from taskdatabase.models import Task, Workflow
+class IngestPageViewTest(TestCase):
+
+ @classmethod
+ def setUpTestData(cls):
+
+ # Set up non-modified objects used by all test methods
+ workflow = Workflow()
+ workflow.save()
+
+ # create a list of Tasks
+ Task.objects.get_or_create(sas_id=12345, status='scrubbed', workflow = workflow)
+ Task.objects.get_or_create(sas_id=12345, status='scrubbed', workflow = workflow)
+ Task.objects.get_or_create(sas_id=12345, status='archiving', workflow = workflow)
+ Task.objects.get_or_create(sas_id=12345, status='finished', workflow=workflow)
+
+ def test_url_exists_at_desired_location(self):
+ response = self.client.get('/atdb/ingest')
+ self.assertEqual(response.status_code, 200)
+
+ def test_url_accessible_by_name(self):
+ response = self.client.get(reverse('ingest'))
+ self.assertEqual(response.status_code, 200)
+
+ def test_uses_correct_template(self):
+ response = self.client.get(reverse('ingest'))
+ self.assertEqual(response.status_code, 200)
+ self.assertTemplateUsed(response, 'taskdatabase/ingest/page.html')
\ No newline at end of file
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 091e51c1f28c18f617606841b85346cc6e0f63b8..f292639079a899f970f26da8bc94f9dd50d373d5 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -19,6 +19,7 @@ urlpatterns = [
path('validation', views.ShowValidationPage.as_view(), name='validation'),
path('failures', views.ShowFailuresPage.as_view(), name='failures'),
path('discarded', views.ShowDiscardedPage.as_view(), name='discarded'),
+ path('ingest', views.ShowIngestQPage.as_view(), name='ingest'),
path('finished', views.ShowFinishedPage.as_view(), name='finished'),
path('task_details/<int:id>/<page>', views.TaskDetails, name='task-details'),
@@ -105,6 +106,8 @@ urlpatterns = [
path('tasks/<int:pk>/change_priority/<priority_change>/<page>', views.ChangePriority, name='task-change-priority'),
path('tasks/<int:pk>/change_priority/<priority_change>', views.ChangePriority, name='task-change-priority'),
+ path('tasks/<int:pk>/change_priority_sasid/<priority_change>/<page>', views.ChangePrioritySasID, name='task-change-priority-sasid'),
+
path('tasks/sort-tasks/<sort>/<redirect_to_page>', views.SortTasks, name='sort-tasks'),
path('tasks/set_filter/<filter>/<redirect_to_page>', views.TaskSetFilter, name='task-set-filter'),
path('tasks/set_active_filter/<redirect_to_page>', views.TaskSetActiveFilter, name='task-set-active-filter'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index f912cfa0e183ebb12a36eee7b097160aa934bebc..b54b8e711f0511cef726a0c7d720fa991d27315a 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -335,20 +335,12 @@ class ShowValidationPage(ListView):
#@silk_profile(name='ShowValidationPage')
def get_queryset(self):
- # check filtered_tasks on the session
- # if it is at its max, then it is not a query targeted at 1 SAS_ID.
- # in that case clear the filter so that all the SAS_ID's show up on the validation page.
- #if self.request.session['filtered_tasks_as_list']:
- # filtered_tasks_on_session = len(self.request.session['filtered_tasks_as_list'])
- # if filtered_tasks_on_session == settings.QUERY_LIMIT_MULTI_CHANGE:
- # self.request.session['filtered_tasks_as_list'] = []
-
tasks = get_filtered_tasks(self.request, None, "sas_id").filter(status__icontains=State.STORED.value)
# exclude the failed tasks
tasks = tasks.exclude(status__icontains=State.FAILED.value)
- paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL) # Show 50 tasks per page
page = self.request.GET.get('page')
try:
@@ -385,11 +377,9 @@ class ShowFailuresPage(ListView):
#@silk_profile(name='ShowFailuresPage')
def get_queryset(self):
- #failed_tasks = Task.objects.filter(status__icontains=State.FAILED.value)
- #tasks = get_filtered_tasks(self.request, failed_tasks)
tasks = get_filtered_tasks(self.request).filter(status__icontains=State.FAILED.value)
- paginator = Paginator(tasks, config.TASKS_PER_PAGE) # Show 50 tasks per page
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL) # Show 50 tasks per page
page = self.request.GET.get('page')
try:
@@ -452,9 +442,54 @@ class ShowDiscardedPage(ListView):
return tasks
+class ShowIngestQPage(ListView):
+ """
+ This shows aggregated tasks per sas_id that are queued for ingest or archiving
+ Note that the global filter is also applied
+ """
+ template_name = 'taskdatabase/ingest/page.html'
+ context_object_name = 'my_tasks'
+
+ # @silk_profile(name='ShowIngestPage')
+ def get_queryset(self):
+
+ ingest_tasks = Task.objects.only('workflow','project','filter').filter(
+ Q(status__icontains=State.SCRUBBED.value) |
+ Q(status__icontains=State.ARCHIVING.value) |
+ Q(status__icontains=State.ARCHIVED.value) |
+ Q(status__icontains=State.FINISHING.value))
+
+ tasks = get_filtered_tasks(self.request, ingest_tasks, "sas_id")
+
+ # exclude the failed tasks
+ tasks = tasks.exclude(status__icontains=State.FAILED.value)
+
+ paginator = Paginator(tasks, config.TASKS_PER_PAGE_SMALL) # Show 10 tasks per page
+ page = self.request.GET.get('page')
+
+ try:
+ # check if there was a page on the session, if so, use it.
+ if page == None:
+ page = self.request.session['page']
+ self.request.session['page'] = None
+ except:
+ pass
+
+ try:
+ tasks = paginator.page(page)
+ except PageNotAnInteger:
+ # If page is not an integer, deliver first page.
+ tasks = paginator.page(1)
+ except EmptyPage:
+ # If page is out of range (e.g. 9999), deliver last page of results.
+ tasks = paginator.page(paginator.num_pages)
+
+ return tasks
+
+
class ShowFinishedPage(ListView):
"""
- This shows the tasks that are archived
+ This shows the tasks that are finished
Note that the global filter is also applied
"""
template_name = 'taskdatabase/archived/page.html'
@@ -534,7 +569,8 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
if pre_filtered_tasks:
filtered_tasks = pre_filtered_tasks
else:
- filtered_tasks = Task.objects.all()
+ #filtered_tasks = Task.objects.all()
+ filtered_tasks = Task.objects.defer('inputs','outputs')
# check if there is a status filter active
try:
@@ -566,7 +602,7 @@ def get_filtered_tasks(request, pre_filtered_tasks=None, distinct=None):
Q(status__in=search) |
Q(project__icontains=search))
- if (Task.objects.all().count() == filtered_tasks.count()):
+ if (Task.objects.defer('inputs','outputs','metrics','remarks','meta_scheduling').count() == filtered_tasks.count()):
request.session['filtered'] = False
else:
request.session['filtered'] = True
@@ -1282,6 +1318,10 @@ def TaskSetFilter(request, filter, redirect_to_page):
if redirect_to_page == 'quality':
return redirect('quality')
+ if redirect_to_page == 'ingest':
+ return redirect('ingest')
+ if redirect_to_page == 'failures':
+ return redirect('failures')
return redirect_with_params('index', '?page=1')
@@ -1311,18 +1351,15 @@ def TaskClearFilter(request, redirect_to_page):
request.session['search_box'] = ''
request.session['filtered'] = False
- if redirect_to_page == 'quality':
- return redirect('quality')
- if redirect_to_page == 'query':
- return redirect('query')
- if redirect_to_page == 'discarded':
- return redirect('discarded')
- return redirect_with_params('index', '?page=1')
+ try:
+ return redirect(redirect_to_page)
+ except:
+ return redirect_with_params('index', '?page=1')
+
@login_required
def ChangePriority(request, pk, priority_change, page=0):
- model = Task
task = Task.objects.get(pk=pk)
priority = task.priority + int(priority_change)
@@ -1339,6 +1376,25 @@ def ChangePriority(request, pk, priority_change, page=0):
# redirect to tasks list
return redirect_with_params('index', '?page=' + page)
+@login_required
+def ChangePrioritySasID(request, pk, priority_change, page=0):
+
+ task = Task.objects.get(pk=pk)
+ tasks = Task.objects.filter(sas_id=task.sas_id)
+
+ for task in tasks:
+ if task.status not in ['discarded','suspended']:
+ priority = task.priority + int(priority_change)
+
+ if priority < 0:
+ priority = 0
+
+ task.priority = priority
+ task.save()
+
+ return redirect_with_params('ingest', '?page=' + page)
+
+
def SortTasks(request, sort, redirect_to_page):
# store the sort field on the session