diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index 814f2d8b322a4388b52ad35d0e10af630f78c616..07a3e77d0303df8d44bdf02c10ebf5c3f8235e93 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -204,9 +204,9 @@ LOGOUT_REDIRECT_URL = FORCE_SCRIPT_NAME + 'atdb'
STATIC_URL = FORCE_SCRIPT_NAME + 'atdb/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
-ALL_STATUSSES = ['defining','defined','staging','staged','processing','processed','storing','stored','scrubbing','scrubbed','validated', 'archiving','archived','finished']
-ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
-STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
+ALL_STATUSSES = ['defining','defined','staging','staged','processing','processed','storing','stored','scrubbing','scrubbed','validated', 'pre_archiving','pre_archived', 'archiving','archived','finishing','finished']
+ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','storing','stored','scrubbing','scrubbed','pre_archiving','pre_archived', 'archiving','archived']
+STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','pre_archiving','pre_archived', 'archiving','archived']
AGGREGATES = ['failed','active','total']
QUERY_LIMIT_MULTI_CHANGE = 10000
diff --git a/atdb/atdb/static/taskdatabase/style.css b/atdb/atdb/static/taskdatabase/style.css
index e3b90a877ba6478a82cf72635f7f15d96d85d61b..5a9474dc821e986f99447d450413324c3d2bf9b7 100644
--- a/atdb/atdb/static/taskdatabase/style.css
+++ b/atdb/atdb/static/taskdatabase/style.css
@@ -3,12 +3,12 @@ TD {
font-size: 12pt;
}
-.defining,.staging,.fetching,.processing,.storing,.scrub,.scrubbing,.archiving {
+.defining,.staging,.fetching,.processing,.storing,.scrub,.scrubbing,.archiving,.discarding,.pre_archiving {
font-style: italic;
color: green;
}
-.defined,.staged,.fetched,.processed,.stored,.validated,.scrubbed,.archived,.finished {
+.defined,.staged,.fetched,.processed,.stored,.validated,.scrubbed,.archived,.pre_archived,.finished {
background-color: lemonchiffon;
color: blue;
}
diff --git a/atdb/docs/ATDB-LDV - Deployment Diagram.png b/atdb/docs/ATDB-LDV - Deployment Diagram.png
index 04c3f179adecc17fdd11e7994051a736a6cf85c3..ecd5a74403744f17804218cbc03d4500133c59d2 100644
Binary files a/atdb/docs/ATDB-LDV - Deployment Diagram.png and b/atdb/docs/ATDB-LDV - Deployment Diagram.png differ
diff --git a/atdb/docs/ATDB-LDV Workflow Diagram.png b/atdb/docs/ATDB-LDV Workflow Diagram.png
index daf41b9972c5c8bc5d3c596875e30a1c49d9c5c7..eeed7db92827385b647afd4d7cd5bbe68545db9b 100644
Binary files a/atdb/docs/ATDB-LDV Workflow Diagram.png and b/atdb/docs/ATDB-LDV Workflow Diagram.png differ
diff --git a/atdb/taskdatabase/migrations/0044_alter_job_type.py b/atdb/taskdatabase/migrations/0044_alter_job_type.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d8e604cec77601af7327da226059538f2ac0db9
--- /dev/null
+++ b/atdb/taskdatabase/migrations/0044_alter_job_type.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.0 on 2024-02-27 08:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('taskdatabase', '0043_activity_is_combined_activity_is_validated'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='job',
+ name='type',
+ field=models.CharField(blank=True, db_index=True, default=None, max_length=30, null=True),
+ ),
+ ]
diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py
index c8547417ba18b30e7e1642d65ec021f3dd095dd1..fd3e3662ee764876b3617cfab9243a96614a9612 100644
--- a/atdb/taskdatabase/models.py
+++ b/atdb/taskdatabase/models.py
@@ -522,7 +522,7 @@ class Configuration(models.Model):
class Job(models.Model):
- type = models.CharField(db_index=True, max_length=20, default=None,null=True, blank=True)
+ type = models.CharField(db_index=True, max_length=30, default=None,null=True, blank=True)
task_id = models.IntegerField(null=True, blank=True)
job_id = models.IntegerField(null=True, blank=True)
timestamp = models.DateTimeField(default=timezone.now, blank=True)
diff --git a/atdb/taskdatabase/services/activities_handler.py b/atdb/taskdatabase/services/activities_handler.py
index 086542ddf6f35b4217ad3585d8533ab640895bf8..fa44983408cd844a87f09f477e6a036b85c6821f 100644
--- a/atdb/taskdatabase/services/activities_handler.py
+++ b/atdb/taskdatabase/services/activities_handler.py
@@ -18,8 +18,9 @@ def calculate_ingested_fraction(this_task):
"""
result = {}
- statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
- 'suspended': 0, 'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
+ statusses = {'scrubbed': 0, 'pre_archiving': 0, 'pre_archived': 0,'archiving': 0, 'archived': 0,
+ 'finishing': 0, 'finished': 0, 'suspended': 0, 'discarded': 0,
+ 'pre_archived_failed': 0, 'archived_failed': 0, 'finished_failed': 0}
tasks = Task.objects.filter(sas_id=this_task.sas_id)
@@ -29,8 +30,9 @@ def calculate_ingested_fraction(this_task):
except:
pass
- incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) + \
- int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
+ incomplete = int(statusses['scrubbed']) + int(statusses['pre_archiving']) + int(statusses['pre_archived']) + \
+ int(statusses['archiving']) +int(statusses['finishing']) + int(statusses['suspended']) + \
+ int(statusses['pre_archived_failed']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
complete = int(statusses['archived']) + int(statusses['finished'])
completion = round(complete / (incomplete + complete) * 100)
@@ -95,7 +97,8 @@ def update_activity(task):
pass
# calculate the fraction and list of statusses of ingested tasks of this SAS_ID
- if task.status in [State.SCRUBBED.value, State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]:
+ if task.status in [State.SCRUBBED.value, State.PRE_ARCHIVING.value, State.PRE_ARCHIVED.value,
+ State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]:
logger.info(f'- calculate_ingested_fraction')
result = calculate_ingested_fraction(task)
activity.ingested_fraction = result['completion']
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 23651ae0db817fc212fddd3b82bdae281881c78e..7651909e711f59dbab809ca77b4bcfc8ce55ec5a 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -533,7 +533,7 @@ def construct_tasks_per_workflow_html(request, workflow_results):
for workflow_result in workflow_results:
link = construct_link_to_workflow_api(request, workflow_result)
- values = "<tr class='info'><td colspan='6'><b>" + link + "</b></td>"
+ values = "<tr class='info'><td colspan='9'><b>" + link + "</b></td>"
# add sizes
values += "<td><b>size to process:</b> " + str(human_readable(workflow_result['size_to_process'])) + "</td>"
diff --git a/atdb/taskdatabase/services/common.py b/atdb/taskdatabase/services/common.py
index 3cf84aa882a7337561fcb52065e371de3d6104cf..b6895f7abd9ccce9545610082ce2e8de1322a2dc 100644
--- a/atdb/taskdatabase/services/common.py
+++ b/atdb/taskdatabase/services/common.py
@@ -15,6 +15,8 @@ class State(Enum):
STORED = 'stored'
VALIDATED = "validated"
SCRUBBED = "scrubbed"
+ PRE_ARCHIVING = "pre_archiving"
+ PRE_ARCHIVED = "pre_archived"
ARCHIVING = "archiving"
ARCHIVED = "archived"
FINISHED = "finished"
@@ -23,8 +25,8 @@ class State(Enum):
DISCARDED = "discarded"
FAILED = "failed"
-verified_statusses = [State.STORED.value, State.VALIDATED.value, State.SCRUBBED.value, State.ARCHIVED.value,
- State.FINISHED.value, State.SUSPENDED.value, State.DISCARDED.value]
+verified_statusses = [State.STORED.value, State.VALIDATED.value, State.SCRUBBED.value, State.PRE_ARCHIVED,
+ State.ARCHIVED.value, State.FINISHED.value, State.SUSPENDED.value, State.DISCARDED.value]
class SummaryFlavour(Enum):
diff --git a/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png b/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png
index a91ad4d06bc1e7d1395198cfee37d7e40669b8f9..0035bc3bb0fbaa4f4814048548d46304af040a54 100644
Binary files a/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png and b/atdb/taskdatabase/static/taskdatabase/ATDB-LDV Workflow Diagram.png differ
diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/retry_buttons.html b/atdb/taskdatabase/templates/taskdatabase/failures/retry_buttons.html
index fc22415e1ae9c367e4f44d3a8719893e6b06a078..6126d241702fe94b93d4ed964ceeed2fc52a12b3 100644
--- a/atdb/taskdatabase/templates/taskdatabase/failures/retry_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/failures/retry_buttons.html
@@ -13,9 +13,12 @@
{% if task.status == "scrubbed_failed" or task.status == "scrubbing_failed" %}
<a href="{% url 'task-retry-view' task.pk 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
{% endif %}
- {% if task.status == "archived_failed" or task.status == "archiving_failed" %}
+ {% if task.status == "pre_archived_failed" %}
<a href="{% url 'task-retry-view' task.pk 'scrubbed' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
{% endif %}
- {% if task.status == "finished_failed" or task.status == "finishing_failed" %}
+ {% if task.status == "archived_failed" %}
+ <a href="{% url 'task-retry-view' task.pk 'pre_archived' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
+ {% endif %}
+ {% if task.status == "finished_failed" %}
<a href="{% url 'task-retry-view' task.pk 'finishing' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><b><i class="fas fa-sync-alt"></i> Retry</b></a>
{% endif %}
\ No newline at end of file
diff --git a/atdb/taskdatabase/templates/taskdatabase/filter/filter_buttons.html b/atdb/taskdatabase/templates/taskdatabase/filter/filter_buttons.html
index c94156723da3ebed70a16ed0893f762dc166c3fe..e9a725fe5d96abe45f57f175f135ae16ec563361 100644
--- a/atdb/taskdatabase/templates/taskdatabase/filter/filter_buttons.html
+++ b/atdb/taskdatabase/templates/taskdatabase/filter/filter_buttons.html
@@ -19,21 +19,18 @@
<a href="{% url 'task-set-active-filter' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-layer-group"></i> Active</a>
<a href="{% url 'task-set-filter' 'failed' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-layer-group"></i> Failed</a>
<a href="{% url 'task-set-onhold-filter' True 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i class="fas fa-layer-group"></i> On Hold</a>
-
+ <a href="{% url 'task-set-filter' 'defining' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>defining</i></a>
<a href="{% url 'task-set-filter' 'defined' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">defined</a>
<a href="{% url 'task-set-filter' 'staging' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>staging</i></a>
<a href="{% url 'task-set-filter' 'staged' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">staged</a>
- <a href="{% url 'task-set-filter' 'fetching' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>fetching</i></a>
<a href="{% url 'task-set-filter' 'fetched' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">fetched</a>
<a href="{% url 'task-set-filter' 'processing' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>processing</i></a>
<a href="{% url 'task-set-filter' 'processed' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">processed</a>
- <a href="{% url 'task-set-filter' 'storing' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>storing</i></a>
<a href="{% url 'task-set-filter' 'stored' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">stored</a>
<a href="{% url 'task-set-filter' 'validated' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">validated</a>
- <a href="{% url 'task-set-filter' 'scrubbing' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>scrubbing</i></a>
<a href="{% url 'task-set-filter' 'scrubbed' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">scrubbed</a>
- <a href="{% url 'task-set-filter' 'archiving' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>archiving</i></a>
- <a href="{% url 'task-set-filter' 'archived' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">archived</a>
+ <a href="{% url 'task-set-filter' 'archiving' 'tasks' %}" class="btn btn-secondary btn-sm" role="button"><i>(pre) archiving</i></a>
+ <a href="{% url 'task-set-filter' 'archived' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">(pre) archived</a>
<a href="{% url 'task-set-filter' 'finished' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">finished</a>
<a href="{% url 'task-set-filter' 'suspended' 'tasks' %}" class="btn btn-secondary btn-sm" role="button">suspended</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/query/set_multi_status_panel.html b/atdb/taskdatabase/templates/taskdatabase/query/set_multi_status_panel.html
index a4a5fc5ecde8b5d76d0d1e6f5e1d4c51a7fdb184..8e1a5d866ce1c826d102a2c93feb63f26a6d5f1b 100644
--- a/atdb/taskdatabase/templates/taskdatabase/query/set_multi_status_panel.html
+++ b/atdb/taskdatabase/templates/taskdatabase/query/set_multi_status_panel.html
@@ -20,6 +20,7 @@
<a href="{% url 'task-multi-setstatus' 'stored' request.GET %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> stored</a>
<a href="{% url 'task-multi-setstatus' 'validated' request.GET %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-sync-alt"></i> validated</a>
<a href="{% url 'task-multi-setstatus' 'scrubbed' request.GET %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> scrubbed</a>
+ <a href="{% url 'task-multi-setstatus' 'pre_archived' request.GET %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> pre_archived</a>
<a href="{% url 'task-multi-setstatus' 'archived' request.GET %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> archived</a>
<a href="{% url 'task-multi-setstatus' 'finished' request.GET %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> finished</a>
<a href="{% url 'task-multi-setstatus' 'suspended' request.GET %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-sync-alt"></i> suspended</a>
diff --git a/atdb/taskdatabase/templates/taskdatabase/query/status_buttons_per_row.html b/atdb/taskdatabase/templates/taskdatabase/query/status_buttons_per_row.html
index 3ff63b2e42c8f9c9fc5bf30a79b2677851c9bc6d..99ce59603f0fc26e28e4d40b87d47a9776f2bdcc 100644
--- a/atdb/taskdatabase/templates/taskdatabase/query/status_buttons_per_row.html
+++ b/atdb/taskdatabase/templates/taskdatabase/query/status_buttons_per_row.html
@@ -8,6 +8,7 @@
<a href="{% url 'task-setstatus' record.pk 'stored' request.GET %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> stored</a>
<a href="{% url 'task-setstatus' record.pk 'validated' request.GET %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-sync-alt"></i> validated</a>
<a href="{% url 'task-setstatus' record.pk 'scrubbed' request.GET %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> scrubbed</a>
+ <a href="{% url 'task-setstatus' record.pk 'pre_archived' request.GET %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> pre_archived</a>
<a href="{% url 'task-setstatus' record.pk 'archived' request.GET %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> archived</a>
<a href="{% url 'task-setstatus' record.pk 'finished' request.GET %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> finished</a>
<a href="{% url 'task-setstatus' record.pk 'suspended' request.GET %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-sync-alt"></i> suspended</a>