Skip to content
Snippets Groups Projects
Commit 77e562af authored by Nico Vermaas's avatar Nico Vermaas
Browse files

simplify activity a bit

parent 2077a23d
No related branches found
No related tags found
1 merge request!339SDC-1188 - STEP 1 of 3 (the database)
Pipeline #71665 failed
Stage: test
Stage: build
Stage: deploy_to_test
Stage: deploy_to_production
Hide whitespace changes
Inline Side-by-side
atdb/taskdatabase/migrations/0037_remove_activity_has_summary.py 0 → 100644
+ 17
− 0
View file @ 77e562af
# Generated by Django 5.0 on 2024-02-09 09:42
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('taskdatabase', '0036_remove_activity_output_sas_id_activity_remaining_and_more'),
]
operations = [
migrations.RemoveField(
model_name='activity',
name='has_summary',
),
]
atdb/taskdatabase/migrations/0038_remove_activity_priority_remove_activity_workflow_and_more.py 0 → 100644
+ 26
− 0
View file @ 77e562af
# Generated by Django 5.0 on 2024-02-09 12:18
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('taskdatabase', '0037_remove_activity_has_summary'),
]
operations = [
migrations.RemoveField(
model_name='activity',
name='priority',
),
migrations.RemoveField(
model_name='activity',
name='workflow',
),
migrations.AddField(
model_name='activity',
name='workflow_id',
field=models.IntegerField(null=True),
),
]
atdb/taskdatabase/models.py
+ 111
− 111
View file @ 77e562af
......@@ -12,7 +12,6 @@ from .services.common import State, verified_statusses
logger = logging.getLogger(__name__)
# constants
datetime_format_string = '%Y-%m-%dT%H:%M:%SZ'
......@@ -80,20 +79,14 @@ def convert_summary_to_list_for_template(task):
class Activity(models.Model):
"""
update when tasks status changes from ??? to ???:
- to '???' : check for incoming 'archive' json from archiver
- to 'stored' : calculate qualities
- to ??? : calculate finished_fraction
- to ??? : calculate ingested_fraction
Aggregation per SAS_ID level
"""
# fields used in overview pages
sas_id = models.CharField(db_index=True, verbose_name="SAS_ID", max_length=15, blank=True, null=True)
workflow = models.ForeignKey(Workflow, related_name='activities', on_delete=models.CASCADE, null=True, blank=True)
#workflow = models.ForeignKey(Workflow, related_name='activities', on_delete=models.DO_NOTHING, null=True, blank=True)
workflow_id = models.IntegerField(null=True)
project = models.CharField(max_length=100, blank=True, null=True, default="unknown")
filter = models.CharField(max_length=30, blank=True, null=True)
priority = models.IntegerField(default=100, null=True)
status = models.CharField(db_index=True, default="unknown", max_length=50, blank=True, null=True)
calculated_quality = models.CharField(max_length=10, blank=True, null=True)
......@@ -101,8 +94,6 @@ class Activity(models.Model):
# this is the JSON blob that is filled in by the ldv_archiver during the ingest process
archive = models.JSONField(null=True, blank=True)
has_summary = models.BooleanField(default=False)
is_verified = models.BooleanField(default=False)
finished_fraction = models.FloatField(blank=True, null=True)
ingested_fraction = models.FloatField(blank=True, null=True)
......@@ -123,6 +114,9 @@ class Activity(models.Model):
except:
return None
def __str__(self):
return str(self.sas_id)
class Task(models.Model):
......@@ -190,6 +184,10 @@ class Task(models.Model):
tasks_for_this_sasid = Task.objects.filter(sas_id=self.sas_id)
self.calculated_qualities = qualities.calculate_qualities(self, tasks_for_this_sasid, quality_thresholds)
# remark:
# a post_save signal is triggered by this save()
# to update the associated 'activity' with relevant aggregated information
super(Task, self).save(*args, **kwargs)
......@@ -320,20 +318,21 @@ class Task(models.Model):
except:
return None
@property
def sas_id_has_archived(self):
"""
check if any task belonging to this sas_id already has an output SAS_ID at the LTA
"""
try:
for task in Task.objects.filter(sas_id=self.sas_id):
try:
if task.archive['sas_id_archived']:
return task.archive['sas_id_archived']
except:
pass
except:
return None
# keep the old mechanism in comments to test/evaluate, remove later when it works
# @property
# def sas_id_has_archived(self):
# """
# check if any task belonging to this sas_id already has an output SAS_ID at the LTA
# """
# try:
# for task in Task.objects.filter(sas_id=self.sas_id):
# try:
# if task.archive['sas_id_archived']:
# return task.archive['sas_id_archived']
# except:
# pass
# except:
# return None
@property
def path_to_lta(self):
......@@ -345,92 +344,93 @@ class Task(models.Model):
except:
return None
@property
def sasid_path_to_lta(self):
"""
check if any task belonging to this sas_id already has a 'path_to_lta' setting
"""
try:
for task in Task.objects.filter(sas_id=self.sas_id):
try:
if task.archive['path_to_lta']:
return task.archive['path_to_lta']
except:
# if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
pass
except:
return None
@property
def sasid_is_verified(self):
for task in Task.objects.filter(sas_id=self.sas_id):
if task.status not in verified_statusses:
return False
return True
@property
def sasid_finished_fraction(self):
size_archived = 0
size_remaining = 0
total_size = 0
tasks = Task.objects.filter(sas_id=self.sas_id)
for task in tasks:
if task.status == State.FINISHED.value:
size_archived = size_archived + task.size_to_process
else:
size_remaining = size_remaining + task.size_to_process
total_size = total_size + task.size_to_process
finished = {}
try:
finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
except:
finished['fraction'] = -1
finished['total_size'] = total_size
finished['remaining'] = size_remaining
return finished
@property
def sasid_ingested_fraction(self):
"""
This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
and a list of statusses of other tasks belonging to the same SAS_ID.
It is implemented as 'property', because then it can be used in html pages like this:
<td>{{ task.sasid_ingested_fraction.status }}</td>
<td>{{ task.sasid_ingested_fraction.completion }}%</td>
A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
The division of those 2 are returned as a 'completed %'.
A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
"""
result = {}
statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
tasks = Task.objects.filter(sas_id=self.sas_id)
for task in tasks:
try:
statusses[task.status] += 1
except:
pass
incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
complete = int(statusses['archived']) + int(statusses['finished'])
completion = round(complete / (incomplete + complete) * 100)
non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
# keep the old mechanism in comments to test/evaluate, remove later when it works
# @property
# def sasid_path_to_lta(self):
# """
# check if any task belonging to this sas_id already has a 'path_to_lta' setting
# """
# try:
# for task in Task.objects.filter(sas_id=self.sas_id):
# try:
# if task.archive['path_to_lta']:
# return task.archive['path_to_lta']
# except:
# # if 'path_to_lta' is not found, or 'archive' is empty, continue to the next task
# pass
# except:
# return None
result['status'] = non_zero_statusses
result['completion'] = completion
return result
#
# @property
# def sasid_is_verified(self):
# for task in Task.objects.filter(sas_id=self.sas_id):
# if task.status not in verified_statusses:
# return False
# return True
#
# @property
# def sasid_finished_fraction(self):
# size_archived = 0
# size_remaining = 0
# total_size = 0
#
# tasks = Task.objects.filter(sas_id=self.sas_id)
#
# for task in tasks:
# if task.status == State.FINISHED.value:
# size_archived = size_archived + task.size_to_process
# else:
# size_remaining = size_remaining + task.size_to_process
# total_size = total_size + task.size_to_process
#
# finished = {}
# try:
# finished['fraction'] = round((size_archived / (size_remaining + size_archived)) * 100)
# except:
# finished['fraction'] = -1
#
# finished['total_size'] = total_size
# finished['remaining'] = size_remaining
#
# return finished
#
# @property
# def sasid_ingested_fraction(self):
# """
# This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID
# and a list of statusses of other tasks belonging to the same SAS_ID.
# It is implemented as 'property', because then it can be used in html pages like this:
# <td>{{ task.sasid_ingested_fraction.status }}</td>
# <td>{{ task.sasid_ingested_fraction.completion }}%</td>
#
# A selection of statusses are considered 'queued', and another selection is considered 'ingested'.
# The division of those 2 are returned as a 'completed %'.
# A limited list of statusses for the other tasks that belong to this SAS_ID is also returned.
#
# """
# result = {}
# statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0,
# 'suspended': 0,'discarded': 0, 'archived_failed': 0, 'finished_failed': 0}
#
# tasks = Task.objects.filter(sas_id=self.sas_id)
#
# for task in tasks:
# try:
# statusses[task.status] += 1
# except:
# pass
#
# incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) +\
# int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed'])
# complete = int(statusses['archived']) + int(statusses['finished'])
# completion = round(complete / (incomplete + complete) * 100)
#
# non_zero_statusses = {key: value for key, value in statusses.items() if value != 0}
#
# result['status'] = non_zero_statusses
# result['completion'] = completion
# return result
@property
def task_type_join(self):
......
atdb/taskdatabase/serializers.py
+ 0
− 1
View file @ 77e562af
......@@ -95,7 +95,6 @@ class TaskReadSerializer(serializers.ModelSerializer):
required=False,
)
class Meta:
model = Task
fields = ['id','task_type','creationTime','filter',
......
atdb/taskdatabase/services/activities.py
+ 6
− 4
View file @ 77e562af
......@@ -76,13 +76,15 @@ def associate_task_with_activity(task, save_task=True):
logger.info(f'create new activity for sas_id {task.sas_id}')
activity = Activity(sas_id=task.sas_id,
workflow=task.workflow,
project=task.project,
filter=task.filter,
priority=task.priority,
archive=task.archive)
filter=task.filter)
activity.save()
# update activity
activity.filter = task.filter
activity.workflow_id = task.workflow.id
task.activity = activity
if save_task:
task.save()
......
atdb/taskdatabase/views.py
+ 2
− 1
View file @ 77e562af
......@@ -99,7 +99,8 @@ class ActivityFilter(filters.FilterSet):
fields = {
'sas_id': ['exact', 'icontains', 'in'],
'filter': ['exact', 'icontains'],
'workflow__id': ['exact', 'icontains'],
#'workflow__id': ['exact', 'icontains'],
'workflow_id': ['exact', 'icontains'],
'project': ['exact', 'icontains'],
'sas_id': ['exact', 'icontains', 'in'],
'status': ['exact', 'icontains', 'in', 'startswith'],
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment