diff --git a/atdb/atdb/settings/dev.py b/atdb/atdb/settings/dev.py index fbbded20d0fe15e2470edb5ca18ffbb6513f50a3..7174497c1ebde58462fdc3d3c038df2d89103804 100644 --- a/atdb/atdb/settings/dev.py +++ b/atdb/atdb/settings/dev.py @@ -14,7 +14,7 @@ DATABASES = { 'USER': 'atdb_admin', 'PASSWORD': 'atdb123', #'NAME': 'atdb_ldv_19jan2024', - 'NAME': 'atdb_ldv_16feb2024', + 'NAME': 'atdb_ldv_17feb2024', 'HOST': 'localhost', 'PORT': '5432', }, diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py index fd330201c6934b9bd1b4574987adce12a542192e..51f56a7fb0f12e13223ad6f62bad815ad9c59a73 100644 --- a/atdb/taskdatabase/models.py +++ b/atdb/taskdatabase/models.py @@ -202,8 +202,7 @@ class Task(models.Model): tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id) self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds) - # make sure that every task has an activity (backward compatibility) - # TODO: uncomment to enable SDC-1188 functionality for deploy STEP 2 + # make sure that every task has an activity (also for backward compatibility) associate_task_with_activity(self) # remark: @@ -340,25 +339,6 @@ class Task(models.Model): except: return None - # keep the old mechanism in comments to test/evaluate - # TODO: remove when it is no longer used by the GUI - # --- <CUT> --- - @property - def sas_id_has_archived(self): - """ - check if any task belonging to this sas_id already has an output SAS_ID at the LTA - """ - try: - for task in Task.objects.filter(sas_id=self.sas_id): - try: - if task.archive['sas_id_archived']: - return task.archive['sas_id_archived'] - except: - pass - except: - return None - # --- </CUT> --- - @property def path_to_lta(self): """ @@ -369,98 +349,6 @@ class Task(models.Model): except: return None - # keep the old mechanism in comments to test/evaluate, remove later when it works - # TODO: remove when it is no longer used by the GUI - # ---- <CUT> ---- - @property - def sasid_path_to_lta(self): - """ - check if any task belonging to this sas_id already has a 'path_to_lta' setting - """ - try: - for task in Task.objects.filter(sas_id=self.sas_id): - try: - if task.archive['path_to_lta']: - return task.archive['path_to_lta'] - except: - # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task - pass - except: - return None - - - @property - def sasid_is_verified(self): - for task in Task.objects.filter(sas_id=self.sas_id): - if task.status not in verified_statusses: - return False - return True - - @property - def sasid_finished_fraction(self): - size_archived = 0 - size_remaining = 0 - total_size = 0 - - tasks = Task.objects.filter(sas_id=self.sas_id) - - for task in tasks: - if task.status == State.FINISHED.value: - size_archived = size_archived + task.size_to_process - else: - size_remaining = size_remaining + task.size_to_process - total_size = total_size + task.size_to_process - - finished = {} - try: - finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100) - except: - finished['fraction'] = -1 - - finished['total_size'] = total_size - finished['remaining'] = size_remaining - - return finished - - @property - def sasid_ingested_fraction(self): - """ - This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID - and a list of statusses of other tasks belonging to the same SAS_ID. - It is implemented as 'property', because then it can be used in html pages like this: - <td>{{ task.sasid_ingested_fraction.status }}</td> - <td>{{ task.sasid_ingested_fraction.completion }}%</td> - - A selection of statusses are considered 'queued', and another selection is considered 'ingested'. - The division of those 2 are returned as a 'completed %'. - A limited list of statusses for the other tasks that belong to this SAS_ID is also returned. - - """ - result = {} - statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0, - 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0} - - tasks = Task.objects.filter(sas_id=self.sas_id) - - for task in tasks: - try: - statusses[task.status] += 1 - except: - pass - - incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\ - int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed']) - complete = int(statusses['archived']) + int(statusses['finished']) - completion = round(complete / (incomplete + complete) * 100) - - non_zero_statusses = {key: value for key, value in statusses.items() if value != 0} - - result['status'] = non_zero_statusses - result['completion'] = completion - return result - - # ---- </CUT> ---- - @property def task_type_join(self): try: diff --git a/atdb/taskdatabase/services/signals.py b/atdb/taskdatabase/services/signals.py index ef36178a3ae995eabf030bba7d20bc8f9ec2bd86..2442e3d78cb5bdb571aeab3969bc2a6f182dd420 100644 --- a/atdb/taskdatabase/services/signals.py +++ b/atdb/taskdatabase/services/signals.py @@ -65,7 +65,6 @@ def handle_post_save(sender, **kwargs): """ task = kwargs.get('instance') - # TODO: uncomment to enable SDC-1188 functionality update_activity(task) diff --git a/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html b/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html index 53433d5239ee985357a5584419d73ce235e5a7e8..0116beff2d324b6988d16e675ae68463980a7d23 100644 --- a/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html +++ b/atdb/taskdatabase/templates/taskdatabase/archived/tasks.html @@ -52,19 +52,6 @@ {{ task.sas_id }} </td> <td> - -<!-- keep the old mechanism in comments to test/evaluate, remove later when it works --> - - {% if task.sas_id_archived != None %} - <a href={{ task.path_to_lta }} target="_blank"> - <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA"> - {{ task.sas_id_archived }} - </a> - {% else %} - - - {% endif %} - -<!-- new activity mechanism, to enable SDC-1188 {% if task.activity.archive.sas_id_archived != None %} <a href={{ task.path_to_lta }} target="_blank"> <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA"> @@ -73,7 +60,6 @@ {% else %} - {% endif %} ---> </td> <td> diff --git a/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html b/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html index 21ba2ddbe3ef45ce6cc134941126931bdcdddfce..55d0fb82f200b840a69c0a81e8dba3ca2072dd10 100644 --- a/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html +++ b/atdb/taskdatabase/templates/taskdatabase/failures/tasks.html @@ -34,17 +34,9 @@ <td>{{ task.project }}</td> <td>{{ task.sas_id }}</td> - <td>{{ task.filter }} </td> -<!-- keep the old mechanism in comments to test/evaluate, remove later when it works --> - - <td>{{ task.sasid_finished_fraction.fraction }}% of {{ task.sasid_finished_fraction.total_size|filesizeformat }}</td> - <td>{{ task.size_to_process|filesizeformat }} / {{ task.sasid_finished_fraction.remaining|filesizeformat }}</td> - -<!-- new activity mechanism, to enable SDC-1188 - <td>{{ task.activity.finished_fraction }}% of {{ task.activity.total_size|filesizeformat }}</td> + <td>{{ task.activity.finished_fraction|floatformat:"0" }}% of {{ task.activity.total_size|filesizeformat }}</td> <td>{{ task.size_to_process|filesizeformat }} / {{ task.activity.remaining|filesizeformat }}</td> ---> <td> {% include "taskdatabase/failures/retry_buttons.html" %} diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html index f4fb3278583426d1aef9b3cb9603ff1be856b9d6..d9a8482a724472f6636c0183ba9522edad85f4a0 100644 --- a/atdb/taskdatabase/templates/taskdatabase/index.html +++ b/atdb/taskdatabase/templates/taskdatabase/index.html @@ -31,7 +31,7 @@ {% include 'taskdatabase/pagination.html' %} </div> </div> - <p class="footer"> Version 16 Feb 2024 (7:00) + <p class="footer"> Version 19 Feb 2024 </div> {% include 'taskdatabase/refresh.html' %} diff --git a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html index 79a13d9f9c7dcf656f7d49d1feec2111a1ed8bfb..5900a5c45dd1e3287600e70734677c5ad8bd85a7 100644 --- a/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html +++ b/atdb/taskdatabase/templates/taskdatabase/ingest/tasks.html @@ -23,26 +23,8 @@ <a href="{% url 'task-change-priority-sasid' task.pk '10' my_tasks.number %}" class="btn btn-warning btn-sm" role="button">+10</a> {% endif %} </td> - -<!-- keep the old mechanism in comments to test/evaluate, remove later when it works --> - - <td>{{ task.sasid_ingested_fraction.status }}</td> - - <td>{{ task.sasid_ingested_fraction.completion }}%</td> - <td> - {% if task.sas_id_has_archived != None %} - <a href={{ task.sasid_path_to_lta }} target="_blank"> - <img src="{% static 'taskdatabase/ldvlogo_small.png' %}" height="20" alt="link to LTA"> - {{ task.sas_id_has_archived }} - </a> - {% else %} - - - {% endif %} - </td> - -<!-- new activity mechanism, to enable SDC-1188 <td>{{ task.activity.ingestq_status }}</td> - <td>{{ task.activity.ingested_fraction }}%</td> + <td>{{ task.activity.ingested_fraction|floatformat:"0" }}%</td> <td> {% if task.activity.has_archived != None %} <a href={{ task.sasid_path_to_lta }} target="_blank"> @@ -53,7 +35,7 @@ - {% endif %} </td> ---> + </tr> </div> {% endif %} diff --git a/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html b/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html index 65f8cc4a39af3f24679db64aeacedb1d6fc941cc..c475e3771eaa3453b37dd633185c5e575cc0d0d5 100644 --- a/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html +++ b/atdb/taskdatabase/templates/taskdatabase/tasks/details_card.html @@ -4,8 +4,9 @@ <h4>Task: {{ task.id }} ({{ task.task_type }})</h4> <table class="table table-striped"> <tbody> - <tr><td><b>id</b></td><td><a href="{% url 'task-detail-view-api' task.pk %}" target="_blank">{{ task.id }} </a></td></tr> - <tr><td><b>slurm jobs</b></td><td><a href="{% url 'job-list-view-api' %}?task_id={{ task.id }}" {{ task.id }} target="_blank">jobs for task {{ task.id }} </a> {{ task.get_jobs_statusses }}</td></tr> + <tr><td><b>id (api)</b></td><td><a href="{% url 'task-detail-view-api' task.pk %}" target="_blank">{{ task.id }} </a></td></tr> + + <tr><td><b>slurm jobs (api)</b></td><td><a href="{% url 'job-list-view-api' %}?task_id={{ task.id }}" {{ task.id }} target="_blank">jobs for task {{ task.id }} </a> {{ task.get_jobs_statusses }}</td></tr> <tr><td><b>priority</b></td><td> {% if user.is_authenticated %} @@ -77,7 +78,8 @@ </tr> <tr><td><b>filter</b></td><td>{{ task.filter }}</td></tr> <tr><td><b>project</b></td><td>{{ task.project }}</td></tr> - <tr><td><b>sas_id</b></td><td>{{ task.sas_id }}</td></tr> + <tr><td><b>sas_id (api)</b></td><td><a href="{% url 'activity-detail-view-api' task.activity.id %}" target="_blank">{{ task.activity }} </a></td></tr> + <tr><td><b>creationTime</b></td><td>{{ task.creationTime|date:"Y-m-d H:i:s" }}</td></tr> <tr><td><b>purge_policy</b></td><td>{{ task.purge_policy }}</td></tr> diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html index 160703f6831a4b627396d0091a5817947b70752e..c0d65878f8ef2e760696366db7a437123ec25791 100644 --- a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html +++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html @@ -69,13 +69,7 @@ {% endif %} </td> -<!-- keep the old mechanism in comments to test/evaluate, remove later when it works --> - - <td class="{{ task.calculated_qualities.per_sasid }}">{{ task.calculated_qualities.per_sasid|default_if_none:"-" }}</td> - -<!-- new activity mechanism, to enable SDC-1188 <td class="{{ task.activity.calculated_quality }}">{{ task.activity.calculated_quality|default_if_none:"-" }}</td> ---> <td class="{{ task.quality }}">{{ task.quality|default_if_none:"-" }}</td> <td>{% include "taskdatabase/validation/validation_buttons.html" %}</td> <td><a href="{% url 'task-discard-view-sasid' task.pk 'discard' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-trash-alt"></i></a></td> diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html index e5f633f78d8952acb549ed39924edaaf65fe14c3..a64240c3277d926ea624fadfb055d2e5764824d9 100644 --- a/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html +++ b/atdb/taskdatabase/templates/taskdatabase/validation/validation_buttons.html @@ -1,22 +1,4 @@ -<!-- keep the old mechanism in comments to test/evaluate, remove later when it works --> - {% if task.sasid_is_verified %} - <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a> - {% endif %} - - {% if task.sasid_is_verified %} - <a href="{% url 'task-validate-sasid' task.pk 'moderate' 'validated' my_tasks.number %}" class="btn btn-warning btn-sm" role="button"><i class="fas fa-check"></i> M</a> - {% endif %} - - {% if task.sasid_is_verified %} - <a href="{% url 'task-validate-sasid' task.pk 'good' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> G</a> - {% endif %} - - {% if task.sasid_is_verified %} - <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a> - {% endif %} - -<!-- new activity mechanism, to enable SDC-1188 {% if task.activity.is_verified %} <a href="{% url 'task-validate-sasid' task.pk 'poor' 'validated' my_tasks.number %}" class="btn btn-danger btn-sm" role="button"><i class="fas fa-check"></i> P</a> {% endif %} @@ -32,4 +14,3 @@ {% if task.activity.is_verified %} <a href="{% url 'task-validate-sasid' task.pk 'calculated' 'validated' my_tasks.number %}" class="btn btn-success btn-sm" role="button"><i class="fas fa-check"></i> Validate</a> {% endif %} ---> \ No newline at end of file diff --git a/atdb/taskdatabase/tests/test_ingest_fraction.py b/atdb/taskdatabase/tests/test_ingest_fraction.py index 0061ccc552aaecd43d9556a1ec3ee884e1a1d93a..98807eea173affaef8f7914d5e619abec4ee8268 100644 --- a/atdb/taskdatabase/tests/test_ingest_fraction.py +++ b/atdb/taskdatabase/tests/test_ingest_fraction.py @@ -20,15 +20,15 @@ class TestIngestFraction(TestCase): Task.objects.get_or_create(filter='a',sas_id=54321, status='scrubbed', workflow=workflow_requantisation) Task.objects.get_or_create(filter='b',sas_id=54321, status='scrubbed', workflow=workflow_requantisation) - # def test_ingest_fraction(self): - # - # # collapse all tasks into a single task for this sas_id - # task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0] - # task.save() - # - # # get the list of statusses and level of completion - # statusses = task.activity.ingestq_status - # completion = task.activity.ingested_fraction - # - # self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1}) - # self.assertEqual(completion,38) + def test_ingest_fraction(self): + + # collapse all tasks into a single task for this sas_id + task = Task.objects.filter(sas_id=54321).distinct('sas_id')[0] + task.save() + + # get the list of statusses and level of completion + statusses = task.activity.ingestq_status + completion = task.activity.ingested_fraction + + self.assertEqual(statusses, {'scrubbed': 2, 'archiving': 1, 'archived': 1, 'finishing': 1, 'finished': 2, 'discarded': 1, 'archived_failed': 1}) + self.assertEqual(completion,38) diff --git a/atdb/taskdatabase/tests/test_update_activity.py b/atdb/taskdatabase/tests/test_update_activity.py new file mode 100644 index 0000000000000000000000000000000000000000..cd015954369450045a47354dda962c44d2ecf10a --- /dev/null +++ b/atdb/taskdatabase/tests/test_update_activity.py @@ -0,0 +1,139 @@ +from django.test import TestCase +import json +from taskdatabase.models import Task, Workflow, Activity + +class TestUpdateActivity(TestCase): + + def setUp(self): + """ + initialize test data + """ + self.workflow_requantisation = Workflow(id=22, workflow_uri="psrfits_requantisation") + self.workflow_requantisation.save() + + self.task1 = Task.objects.create(sas_id=12345, + filter="test_blabla", + status='stored', + workflow=self.workflow_requantisation, + calculated_qualities={"per_task": "good", "per_sasid": "good"}) + self.task2 = Task.objects.create(sas_id=12345, + status='scrubbed', + workflow=self.workflow_requantisation, + calculated_qualities={"per_task": "good", "per_sasid": "good"}) + self.task3 = Task.objects.create(sas_id=12345, + status='archived', + workflow=self.workflow_requantisation, + archive={ + "path_to_lta": "https://lta.lofar.eu//Lofar?project=ALL&mode=show_dataproducts_pipe&product=PulsarPipeline&pipeline_object_id=1101AB934B386BD5E063164A17AC38B9", + "lta_object_id": "1101AB934B386BD5E063164A17AC38B9", + "sas_id_archived": "1219995" + }, + size_to_process=1000, + size_processed=500) + self.task4 = Task.objects.create(sas_id=12345, + status='finished', + workflow=self.workflow_requantisation, + archive={ + "path_to_lta": "https://lta.lofar.eu//Lofar?project=ALL&mode=show_dataproducts_pipe&product=PulsarPipeline&pipeline_object_id=1101AB934B386BD5E063164A17AC38B9", + "lta_object_id": "1101AB934B386BD5E063164A17AC38B9", + "sas_id_archived": "1219995" + }, + size_to_process=1000, + size_processed=500) + self.task5 = Task.objects.create(sas_id=12345, + status='archived_failed', + workflow=self.workflow_requantisation, + size_to_process=1000, + size_processed=500) + + def test_created_activity(self): + """ + test if activity is created + """ + activity = self.task1.activity + + # test if an activity with the correct sas_id was created + actual = activity.sas_id + self.assertEqual(actual, 12345) + + + def test_stored(self): + """ + test if the activity gets the calculated quality of the sas_id of the stored task + """ + activity = self.task1.activity + + actual = activity.calculated_quality + self.assertEqual(actual, "good") + + + def test_scrubbed(self): + """ + test if the activity gets the ingested_fraction of 0 when scrubbed + """ + activity = self.task2.activity + + actual = activity.ingestq_status + self.assertEqual(actual, {'scrubbed': 1}) + + actual = activity.ingested_fraction + self.assertEqual(actual, 0) + + + def test_archived(self): + """ + test if the activity gets the ingested_fraction of an archived task + """ + activity = self.task3.activity + + actual = activity.ingestq_status + self.assertEqual(actual, {'scrubbed': 1, 'archived': 1}) + + actual = activity.archive['sas_id_archived'] + self.assertEqual(actual, "1219995") + + + def test_finished(self): + """ + test if the activity gets the proper values from 'archived' json of a finished task + """ + + activity = self.task4.activity + + actual = activity.ingestq_status + self.assertEqual(actual, {'scrubbed': 1, 'archived': 1}) + + actual = activity.archive['sas_id_archived'] + self.assertEqual(actual, "1219995") + + + def test_failed(self): + """ + test if the activity gets the ingested_fraction of an archived task + """ + + activity = self.task5.activity + + actual = activity.finished_fraction + self.assertEqual(actual, 33) + + actual = activity.total_size + self.assertEqual(actual, 3000) + + actual = activity.remaining + self.assertEqual(actual, 2000) + + + def test_filter_and_workflow(self): + """ + test if the activity gets the filter and workflow_id of updated tasks + """ + + activity = self.task1.activity + + actual = activity.filter + self.assertEqual(actual, "test_blabla") + + actual = activity.workflow_id + self.assertEqual(actual, 22) + diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py index bf8ed0a4087b273b865f695267985371d1a628f7..906ffe5edfa99879158841e8b5d803431e96a006 100644 --- a/atdb/taskdatabase/urls.py +++ b/atdb/taskdatabase/urls.py @@ -132,8 +132,5 @@ urlpatterns = [ #some migration and repair endpoints path('tasks/repair/associate-activities/', views.AssociateActivities, name='associate-activities'), - path('tasks/repair/update-all-activities/', views.UpdateAllActivities, name='update-all-activities'), - path('tasks/repair/update-failed-tasks/', views.UpdateFailedTasks, name='update-failed-tasks'), - path('tasks/repair/update-ingestq-tasks/<batch_size>', views.UpdateIngestQTasks, name='update-ingestq-tasks'), - path('tasks/repair/update-finished-tasks/', views.UpdateFinishedTasks, name='update-finished-tasks'), + path('tasks/repair/update-activity/<sas_id>', views.UpdateActivitySasId, name='update-activity-sasid'), ] diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py index 44a6189a7dd9f47e7837e34d31ca7984c0a8017b..eb462904c64a075019df95d1337a09913418f55d 100644 --- a/atdb/taskdatabase/views.py +++ b/atdb/taskdatabase/views.py @@ -9,6 +9,7 @@ from django.contrib import messages from rest_framework import generics from rest_framework.response import Response +from django.http import JsonResponse from django_filters import rest_framework as filters from django_filters.views import FilterView @@ -70,6 +71,7 @@ class TaskFilter(filters.FilterSet): # http://localhost:8000/atdb/tasks/?predecessor__isnull=True 'predecessor': ['isnull'], 'predecessor__status': ['exact', 'icontains', 'in', 'startswith'], + 'activity' : ['isnull'], 'activity__id': ['exact'], 'activity__ingested_fraction' : ['exact','lt', 'lte', 'gt', 'gte','isnull'], } @@ -1701,72 +1703,33 @@ def AssociateActivities(request): connect_signals() return redirect('index') -@staff_member_required -def UpdateAllActivities(request): - - all_activities = Activity.objects.all() - # find a task for every activity - total = all_activities.count() - i = 0 - - for activity in all_activities: - try: - i += 1 - task = Task.objects.filter(sas_id=activity.sas_id)[0] - activities_handler.update_activity(task) - logger.info(f'{i} of {total}') - except Exception as error: - logger.error(error) - - # tasks = Task.objects.all() - # total = tasks.count() - # i = 0 - # for task in tasks: - # i+=1 - # activities.update_activity(task) - # logger.info(f'{i} of {total}') - - return redirect('index') -@staff_member_required -def UpdateFailedTasks(request): +def UpdateActivitySasId(request, sas_id): + # this function is called externally to avoid worker timeouts + # http://localhost:8000/atdb/tasks/repair/update-activity/600907 - tasks = Task.objects.filter(status__icontains="failed") - total = tasks.count() - i = 0 - for task in tasks: - i+=1 - activities_handler.update_activity(task) - logger.info(f'{i} of {total}') - - return redirect('index') + try: + activity = Activity.objects.get(sas_id=sas_id) -@staff_member_required -def UpdateIngestQTasks(request, batch_size): - tasks = Task.objects.only('sas_id').filter( - Q(status__icontains=State.SCRUBBED.value) | - Q(status__icontains=State.ARCHIVING.value) | - Q(status__icontains=State.ARCHIVED.value) | - Q(status__icontains=State.FINISHING.value)).filter(activity__ingested_fraction__isnull=True)[:int(batch_size)] + tasks = Task.objects.filter(sas_id=activity.sas_id) + total = tasks.count() + i = 0 - total = tasks.count() - i = 0 - for task in tasks: - i+=1 + for task in tasks: + i += 1 + if task.status in ['stored', 'validated','scrubbed','archiving', 'archived','finished'] or 'failed' in task.status: + activities_handler.update_activity(task) - activities_handler.update_activity(task) - logger.info(f'{i} of {total}') + logger.info(f'{i} of {total} for sas_id = {sas_id}') - return redirect('index') + logger.info(f'UpdateActivitySasId {sas_id}') + except Exception as error: + logger.error(error) + return JsonResponse({ + 'error': str(error) + }) -@staff_member_required -def UpdateFinishedTasks(request): - tasks = Task.objects.only('sas_id').filter(status=State.FINISHED.value) - total = tasks.count() - i = 0 - for task in tasks: - i+=1 - activities_handler.update_activity(task) - logger.info(f'{i} of {total}') + return JsonResponse({ + 'total': total + }) - return redirect('index') \ No newline at end of file