Skip to content
Snippets Groups Projects
Commit b15464d0 authored by Nico Vermaas's avatar Nico Vermaas
Browse files

Merge branch 'SDC-1188-SAS_ID-table' into 'master' 

SDC-1188 - final merge

See merge request !341
parents 407ed199 7ecedf62
No related branches found
No related tags found
1 merge request!341SDC-1188 - final merge
Pipeline #72851 passed
Stage: test
Stage: build
Stage: deploy_to_test
Stage: deploy_to_production
Show whitespace changes
Inline Side-by-side
Showing
with 187 additions and 264 deletions
atdb/atdb/settings/dev.py
+ 1
− 1
View file @ b15464d0
...@@ -14,7 +14,7 @@ DATABASES = { ...@@ -14,7 +14,7 @@ DATABASES = {
'USER': 'atdb_admin', 'USER': 'atdb_admin',
'PASSWORD': 'atdb123', 'PASSWORD': 'atdb123',
#'NAME': 'atdb_ldv_19jan2024', #'NAME': 'atdb_ldv_19jan2024',
'NAME': 'atdb_ldv_16feb2024', 'NAME': 'atdb_ldv_17feb2024',
'HOST': 'localhost', 'HOST': 'localhost',
'PORT': '5432', 'PORT': '5432',
}, },
......
atdb/taskdatabase/models.py
+ 1
− 113
View file @ b15464d0
...@@ -202,8 +202,7 @@ class Task(models.Model): ...@@ -202,8 +202,7 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id) tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds) self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
# make sure that every task has an activity (backward compatibility) # make sure that every task has an activity (also for backward compatibility)
# TODO: uncomment to enable SDC-1188 functionality for deploy STEP 2
associate_task_with_activity(self) associate_task_with_activity(self)
# remark: # remark:
...@@ -340,25 +339,6 @@ class Task(models.Model): ...@@ -340,25 +339,6 @@ class Task(models.Model):
except: except:
return None return None
# keep the old mechanism in comments to test/evaluate
# TODO: remove when it is no longer used by the GUI
# --- <CUT> ---
@property
def sas_id_has_archived(self):
"""
check if any task belonging to this sas_id already has an output SAS_ID at the LTA
"""
try:
for task in Task.objects.filter(sas_id=self.sas_id):
try:
if task.archive['sas_id_archived']:
return task.archive['sas_id_archived']
except:
pass
except:
return None
# --- </CUT> ---
@property @property
def path_to_lta(self): def path_to_lta(self):
""" """
...@@ -369,98 +349,6 @@ class Task(models.Model): ...@@ -369,98 +349,6 @@ class Task(models.Model):
except: except:
return None return None
# keep the old mechanism in comments to test/evaluate, remove later when it works
# TODO: remove when it is no longer used by the GUI
# ---- <CUT> ----
@property
def sasid_path_to_lta(self):
"""
check if any task belonging to this sas_id already has a 'path_to_lta' setting
"""
try:
for task in Task.objects.filter(sas_id=self.sas_id):
try:
if task.archive['path_to_lta']:
return task.archive['path_to_lta']
except:
# if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
pass
except:
return None
@property
def sasid_is_verified(self):
for task in Task.objects.filter(sas_id=self.sas_id):
if task.status not in verified_statusses:
return False
return True
@property
def sasid_finished_fraction(self):
size_archived = 0
size_remaining = 0
total_size = 0
tasks = Task.objects.filter(sas_id=self.sas_id)
for task in tasks:
if task.status == State.FINISHED.value:
size_archived = size_archived + task.size_to_process
else:
size_remaining = size_remaining + task.size_to_process
total_size = total_size + task.size_to_process
finished = {}
try:
finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
except:
finished['fraction'] = -1
finished['total_size'] = total_size
finished['remaining'] = size_remaining
return finished
@property
def sasid_ingested_fraction(self):
"""
This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
and a list of statusses of other tasks belonging to the same SAS_ID.
It is implemented as 'property', because then it can be used in html pages like this:
<td>{{ task.sasid_ingested_fraction.status }}</td>
<td>{{ task.sasid_ingested_fraction.completion }}%</td>
A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
The division of those 2 are returned as a 'completed %'.
A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
"""
result = {}
statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
tasks = Task.objects.filter(sas_id=self.sas_id)
for task in tasks:
try:
statusses[task.status] += 1
except:
pass
incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
complete = int(statusses['archived']) + int(statusses['finished'])
completion = round(complete / (incomplete + complete) * 100)
non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
result['status'] = non_zero_statusses
result['completion'] = completion
return result
# ---- </CUT> ----
@property @property
def task_type_join(self): def task_type_join(self):
try: try:
......
atdb/taskdatabase/services/signals.py
+ 0
− 1
View file @ b15464d0
...@@ -65,7 +65,6 @@ def handle_post_save(sender, **kwargs): ...@@ -65,7 +65,6 @@ def handle_post_save(sender, **kwargs):
""" """
task = kwargs.get('instance') task = kwargs.get('instance')
# TODO: uncomment to enable SDC-1188 functionality
update_activity(task) update_activity(task)
......
atdb/taskdatabase/templates/taskdatabase/archived/tasks.html
+ 0
− 14
View file @ b15464d0
...@@ -52,19 +52,6 @@ ...@@ -52,19 +52,6 @@
{{ task.sas_id }} {{ task.sas_id }}
</td> </td>
<td> <td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
{% if task.sas_id_archived != None %}
<a href={{ task.path_to_lta }} target="_blank">
<img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
{{ task.sas_id_archived }}
</a>&nbsp;
{% else %}
-
{% endif %}
<!-- new activity mechanism, to enable SDC-1188
{% if task.activity.archive.sas_id_archived != None %} {% if task.activity.archive.sas_id_archived != None %}
<a href={{ task.path_to_lta }} target="_blank"> <a href={{ task.path_to_lta }} target="_blank">
<img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA"> <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
...@@ -73,7 +60,6 @@ ...@@ -73,7 +60,6 @@
{% else %} {% else %}
- -
{% endif %} {% endif %}
-->
</td> </td>
<td> <td>
......
atdb/taskdatabase/templates/taskdatabase/failures/tasks.html
+ 1
− 9
View file @ b15464d0
...@@ -34,17 +34,9 @@ ...@@ -34,17 +34,9 @@
<td>{{ task.project }}</td> <td>{{ task.project }}</td>
<td>{{ task.sas_id }}</td> <td>{{ task.sas_id }}</td>
<td>{{ task.filter }} </td> <td>{{ task.filter }} </td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works --> <td>{{ task.activity.finished_fraction|floatformat:"0" }}% of {{ task.activity.total_size|filesizeformat }}</td>
<td>{{ task.sasid_finished_fraction.fraction }}% of {{ task.sasid_finished_fraction.total_size|filesizeformat }}</td>
<td>{{ task.size_to_process|filesizeformat }} / {{ task.sasid_finished_fraction.remaining|filesizeformat }}</td>
<!-- new activity mechanism, to enable SDC-1188
<td>{{ task.activity.finished_fraction }}% of {{ task.activity.total_size|filesizeformat }}</td>
<td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td> <td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td>
-->
<td> <td>
{% include "taskdatabase/failures/retry_buttons.html" %} {% include "taskdatabase/failures/retry_buttons.html" %}
......
atdb/taskdatabase/templates/taskdatabase/index.html
+ 1
− 1
View file @ b15464d0
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
{% include 'taskdatabase/pagination.html' %} {% include 'taskdatabase/pagination.html' %}
</div> </div>
</div> </div>
<p class="footer"> Version 16 Feb 2024 (7:00) <p class="footer"> Version 19 Feb 2024
</div> </div>
{% include 'taskdatabase/refresh.html' %} {% include 'taskdatabase/refresh.html' %}
......
atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html
+ 2
− 20
View file @ b15464d0
...@@ -23,26 +23,8 @@ ...@@ -23,26 +23,8 @@
<a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a> <a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a>
{% endif %} {% endif %}
</td> </td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
<td>{{ task.sasid_ingested_fraction.status }}</td>
<td>{{ task.sasid_ingested_fraction.completion }}%</td>
<td>
{% if task.sas_id_has_archived != None %}
<a href={{ task.sasid_path_to_lta }} target="_blank">
<img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA">
{{ task.sas_id_has_archived }}
</a>&nbsp;
{% else %}
-
{% endif %}
</td>
<!-- new activity mechanism, to enable SDC-1188
<td>{{ task.activity.ingestq_status }}</td> <td>{{ task.activity.ingestq_status }}</td>
<td>{{ task.activity.ingested_fraction }}%</td> <td>{{ task.activity.ingested_fraction|floatformat:"0" }}%</td>
<td> <td>
{% if task.activity.has_archived != None %} {% if task.activity.has_archived != None %}
<a href={{ task.sasid_path_to_lta }} target="_blank"> <a href={{ task.sasid_path_to_lta }} target="_blank">
...@@ -53,7 +35,7 @@ ...@@ -53,7 +35,7 @@
- -
{% endif %} {% endif %}
</td> </td>
-->
</tr> </tr>
</div> </div>
{% endif %} {% endif %}
......
atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html
+ 5
− 3
View file @ b15464d0
...@@ -4,8 +4,9 @@ ...@@ -4,8 +4,9 @@
<h4>Task: {{ task.id }} ({{ task.task_type }})</h4> <h4>Task: {{ task.id }} ({{ task.task_type }})</h4>
<table class="table table-striped"> <table class="table table-striped">
<tbody> <tbody>
<tr><td><b>id</b></td><td><a href="{% url 'task-detail-view-api' task.pk %}" target="_blank">{{ task.id }} </a></td></tr> <tr><td><b>id (api)</b></td><td><a href="{% url 'task-detail-view-api' task.pk %}" target="_blank">{{ task.id }} </a></td></tr>
<tr><td><b>slurm jobs</b></td><td><a href="{% url 'job-list-view-api' %}?task_id={{ task.id }}" {{ task.id }} target="_blank">jobs for task {{ task.id }} </a> {{ task.get_jobs_statusses }}</td></tr>
<tr><td><b>slurm jobs (api)</b></td><td><a href="{% url 'job-list-view-api' %}?task_id={{ task.id }}" {{ task.id }} target="_blank">jobs for task {{ task.id }} </a> {{ task.get_jobs_statusses }}</td></tr>
<tr><td><b>priority</b></td><td> <tr><td><b>priority</b></td><td>
{% if user.is_authenticated %} {% if user.is_authenticated %}
...@@ -77,7 +78,8 @@ ...@@ -77,7 +78,8 @@
</tr> </tr>
<tr><td><b>filter</b></td><td>{{ task.filter }}</td></tr> <tr><td><b>filter</b></td><td>{{ task.filter }}</td></tr>
<tr><td><b>project</b></td><td>{{ task.project }}</td></tr> <tr><td><b>project</b></td><td>{{ task.project }}</td></tr>
<tr><td><b>sas_id</b></td><td>{{ task.sas_id }}</td></tr> <tr><td><b>sas_id (api)</b></td><td><a href="{% url 'activity-detail-view-api' task.activity.id %}" target="_blank">{{ task.activity }} </a></td></tr>
<tr><td><b>creationTime</b></td><td>{{ task.creationTime|date:"Y-m-d H:i:s" }}</td></tr> <tr><td><b>creationTime</b></td><td>{{ task.creationTime|date:"Y-m-d H:i:s" }}</td></tr>
<tr><td><b>purge_policy</b></td><td>{{ task.purge_policy }}</td></tr> <tr><td><b>purge_policy</b></td><td>{{ task.purge_policy }}</td></tr>
......
atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
+ 0
− 6
View file @ b15464d0
...@@ -69,13 +69,7 @@ ...@@ -69,13 +69,7 @@
{% endif %} {% endif %}
</td> </td>
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
<td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }}</td>
<!-- new activity mechanism, to enable SDC-1188
<td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td> <td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td>
-->
<td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td> <td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td>
<td>{% include "taskdatabase/validation/validation_buttons.html" %}</td> <td>{% include "taskdatabase/validation/validation_buttons.html" %}</td>
<td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td> <td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td>
......
atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html
+ 0
− 19
View file @ b15464d0
<!-- keep the old mechanism in comments to test/evaluate, remove later when it works -->
{% if task.sasid_is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
{% endif %}
{% if task.sasid_is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> M</a>
{% endif %}
{% if task.sasid_is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> G</a>
{% endif %}
{% if task.sasid_is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
{% endif %}
<!-- new activity mechanism, to enable SDC-1188
{% if task.activity.is_verified %} {% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a> <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a>
{% endif %} {% endif %}
...@@ -32,4 +14,3 @@ ...@@ -32,4 +14,3 @@
{% if task.activity.is_verified %} {% if task.activity.is_verified %}
<a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a> <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a>
{% endif %} {% endif %}
-->
\ No newline at end of file
atdb/taskdatabase/tests/test_ingest_fraction.py
+ 12
− 12
View file @ b15464d0
...@@ -20,15 +20,15 @@ class TestIngestFraction(TestCase): ...@@ -20,15 +20,15 @@ class TestIngestFraction(TestCase):
Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation) Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation) Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation)
# def test_ingest_fraction(self): def test_ingest_fraction(self):
#
# # collapse all tasks into a single task for this sas_id # collapse all tasks into a single task for this sas_id
# task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0] task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0]
# task.save() task.save()
#
# # get the list of statusses and level of completion # get the list of statusses and level of completion
# statusses = task.activity.ingestq_status statusses = task.activity.ingestq_status
# completion = task.activity.ingested_fraction completion = task.activity.ingested_fraction
#
# self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1}) self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1})
# self.assertEqual(completion,38) self.assertEqual(completion,38)
atdb/taskdatabase/tests/test_update_activity.py 0 → 100644
+ 139
− 0
View file @ b15464d0
from django.test import TestCase
import json
from taskdatabase.models import Task, Workflow, Activity
class TestUpdateActivity(TestCase):
def setUp(self):
"""
initialize test data
"""
self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation")
self.workflow_requantisation.save()
self.task1 = Task.objects.create(sas_id=12345,
filter="test_blabla",
status='stored',
workflow=self.workflow_requantisation,
calculated_qualities={"per_task": "good", "per_sasid": "good"})
self.task2 = Task.objects.create(sas_id=12345,
status='scrubbed',
workflow=self.workflow_requantisation,
calculated_qualities={"per_task": "good", "per_sasid": "good"})
self.task3 = Task.objects.create(sas_id=12345,
status='archived',
workflow=self.workflow_requantisation,
archive={
"path_to_lta": "https://lta.lofar.eu//Lofar?project=ALL&mode=show_dataproducts_pipe&product=PulsarPipeline&pipeline_object_id=1101AB934B386BD5E063164A17AC38B9",
"lta_object_id": "1101AB934B386BD5E063164A17AC38B9",
"sas_id_archived": "1219995"
},
size_to_process=1000,
size_processed=500)
self.task4 = Task.objects.create(sas_id=12345,
status='finished',
workflow=self.workflow_requantisation,
archive={
"path_to_lta": "https://lta.lofar.eu//Lofar?project=ALL&mode=show_dataproducts_pipe&product=PulsarPipeline&pipeline_object_id=1101AB934B386BD5E063164A17AC38B9",
"lta_object_id": "1101AB934B386BD5E063164A17AC38B9",
"sas_id_archived": "1219995"
},
size_to_process=1000,
size_processed=500)
self.task5 = Task.objects.create(sas_id=12345,
status='archived_failed',
workflow=self.workflow_requantisation,
size_to_process=1000,
size_processed=500)
def test_created_activity(self):
"""
test if activity is created
"""
activity = self.task1.activity
# test if an activity with the correct sas_id was created
actual = activity.sas_id
self.assertEqual(actual, 12345)
def test_stored(self):
"""
test if the activity gets the calculated quality of the sas_id of the stored task
"""
activity = self.task1.activity
actual = activity.calculated_quality
self.assertEqual(actual, "good")
def test_scrubbed(self):
"""
test if the activity gets the ingested_fraction of 0 when scrubbed
"""
activity = self.task2.activity
actual = activity.ingestq_status
self.assertEqual(actual, {'scrubbed': 1})
actual = activity.ingested_fraction
self.assertEqual(actual, 0)
def test_archived(self):
"""
test if the activity gets the ingested_fraction of an archived task
"""
activity = self.task3.activity
actual = activity.ingestq_status
self.assertEqual(actual, {'scrubbed': 1, 'archived': 1})
actual = activity.archive['sas_id_archived']
self.assertEqual(actual, "1219995")
def test_finished(self):
"""
test if the activity gets the proper values from 'archived' json of a finished task
"""
activity = self.task4.activity
actual = activity.ingestq_status
self.assertEqual(actual, {'scrubbed': 1, 'archived': 1})
actual = activity.archive['sas_id_archived']
self.assertEqual(actual, "1219995")
def test_failed(self):
"""
test if the activity gets the ingested_fraction of an archived task
"""
activity = self.task5.activity
actual = activity.finished_fraction
self.assertEqual(actual, 33)
actual = activity.total_size
self.assertEqual(actual, 3000)
actual = activity.remaining
self.assertEqual(actual, 2000)
def test_filter_and_workflow(self):
"""
test if the activity gets the filter and workflow_id of updated tasks
"""
activity = self.task1.activity
actual = activity.filter
self.assertEqual(actual, "test_blabla")
actual = activity.workflow_id
self.assertEqual(actual, 22)
atdb/taskdatabase/urls.py
+ 1
− 4
View file @ b15464d0
...@@ -132,8 +132,5 @@ urlpatterns = [ ...@@ -132,8 +132,5 @@ urlpatterns = [
#some migration and repair endpoints #some migration and repair endpoints
path('tasks/repair/associate-activities/', views.AssociateActivities, name='associate-activities'), path('tasks/repair/associate-activities/', views.AssociateActivities, name='associate-activities'),
path('tasks/repair/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'), path('tasks/repair/update-activity/<sas_id>', views.UpdateActivitySasId, name='update-activity-sasid'),
path('tasks/repair/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'),
path('tasks/repair/update-ingestq-tasks/<batch_size>', views.UpdateIngestQTasks, name='update-ingestq-tasks'),
path('tasks/repair/update-finished-tasks/', views.UpdateFinishedTasks, name='update-finished-tasks'),
] ]
atdb/taskdatabase/views.py
+ 24
− 61
View file @ b15464d0
...@@ -9,6 +9,7 @@ from django.contrib import messages ...@@ -9,6 +9,7 @@ from django.contrib import messages
from rest_framework import generics from rest_framework import generics
from rest_framework.response import Response from rest_framework.response import Response
from django.http import JsonResponse
from django_filters import rest_framework as filters from django_filters import rest_framework as filters
from django_filters.views import FilterView from django_filters.views import FilterView
...@@ -70,6 +71,7 @@ class TaskFilter(filters.FilterSet): ...@@ -70,6 +71,7 @@ class TaskFilter(filters.FilterSet):
# http://localhost:8000/atdb/tasks/?predecessor__isnull=True # http://localhost:8000/atdb/tasks/?predecessor__isnull=True
'predecessor': ['isnull'], 'predecessor': ['isnull'],
'predecessor__status': ['exact', 'icontains', 'in', 'startswith'], 'predecessor__status': ['exact', 'icontains', 'in', 'startswith'],
'activity' : ['isnull'],
'activity__id': ['exact'], 'activity__id': ['exact'],
'activity__ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte','isnull'], 'activity__ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte','isnull'],
} }
...@@ -1701,72 +1703,33 @@ def AssociateActivities(request): ...@@ -1701,72 +1703,33 @@ def AssociateActivities(request):
connect_signals() connect_signals()
return redirect('index') return redirect('index')
@staff_member_required
def UpdateAllActivities(request):
all_activities = Activity.objects.all() def UpdateActivitySasId(request, sas_id):
# find a task for every activity # this function is called externally to avoid worker timeouts
total = all_activities.count() # http://localhost:8000/atdb/tasks/repair/update-activity/600907
i = 0
for activity in all_activities:
try: try:
i += 1 activity = Activity.objects.get(sas_id=sas_id)
task = Task.objects.filter(sas_id=activity.sas_id)[0]
activities_handler.update_activity(task)
logger.info(f'{i} of {total}')
except Exception as error:
logger.error(error)
# tasks = Task.objects.all()
# total = tasks.count()
# i = 0
# for task in tasks:
# i+=1
# activities.update_activity(task)
# logger.info(f'{i} of {total}')
return redirect('index') tasks = Task.objects.filter(sas_id=activity.sas_id)
@staff_member_required
def UpdateFailedTasks(request):
tasks = Task.objects.filter(status__icontains="failed")
total = tasks.count() total = tasks.count()
i = 0 i = 0
for task in tasks:
i+=1
activities_handler.update_activity(task)
logger.info(f'{i} of {total}')
return redirect('index')
@staff_member_required
def UpdateIngestQTasks(request, batch_size):
tasks = Task.objects.only('sas_id').filter(
Q(status__icontains=State.SCRUBBED.value) |
Q(status__icontains=State.ARCHIVING.value) |
Q(status__icontains=State.ARCHIVED.value) |
Q(status__icontains=State.FINISHING.value)).filter(activity__ingested_fraction__isnull=True)[:int(batch_size)]
total = tasks.count()
i = 0
for task in tasks: for task in tasks:
i += 1 i += 1
if task.status in ['stored', 'validated','scrubbed','archiving', 'archived','finished'] or 'failed' in task.status:
activities_handler.update_activity(task) activities_handler.update_activity(task)
logger.info(f'{i} of {total}')
return redirect('index') logger.info(f'{i} of {total} for sas_id = {sas_id}')
@staff_member_required logger.info(f'UpdateActivitySasId {sas_id}')
def UpdateFinishedTasks(request): except Exception as error:
tasks = Task.objects.only('sas_id').filter(status=State.FINISHED.value) logger.error(error)
total = tasks.count() return JsonResponse({
i = 0 'error': str(error)
for task in tasks: })
i+=1
activities_handler.update_activity(task) return JsonResponse({
logger.info(f'{i} of {total}') 'total': total
})
return redirect('index')
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment