import logging; from .common import State from taskdatabase.models import Task, Activity logger = logging.getLogger(__name__) def calculate_ingested_fraction(this_task): """ This 'property' of a task returns the fraction of queued/ingested tasks per SAS_ID and a list of statusses of other tasks belonging to the same SAS_ID. It is implemented as 'property', because then it can be used in html pages like this: <td>{{ task.sasid_ingested_fraction.status }}</td> <td>{{ task.sasid_ingested_fraction.completion }}%</td> A selection of statusses are considered 'queued', and another selection is considered 'ingested'. The division of those 2 are returned as a 'completed %'. A limited list of statusses for the other tasks that belong to this SAS_ID is also returned. """ result = {} statusses = {'scrubbed': 0, 'archiving': 0, 'archived': 0, 'finishing': 0, 'finished': 0, 'suspended': 0, 'discarded': 0, 'archived_failed': 0, 'finished_failed': 0} tasks = Task.objects.filter(sas_id=this_task.sas_id) for task in tasks: try: statusses[task.status] += 1 except: pass incomplete = int(statusses['scrubbed']) + int(statusses['archiving']) + int(statusses['finishing']) + \ int(statusses['suspended']) + int(statusses['archived_failed']) + int(statusses['finished_failed']) complete = int(statusses['archived']) + int(statusses['finished']) completion = round(complete / (incomplete + complete) * 100) non_zero_statusses = {key: value for key, value in statusses.items() if value != 0} result['status'] = non_zero_statusses result['completion'] = completion return result def associate_activity(task, save_task=True): if not task.activity: try: activity = Activity.objects.get(sas_id=task.sas_id) except: # no activity exists yet, create it logger.info(f'create new activity for sas_id {task.sas_id}') activity = Activity(sas_id=task.sas_id, workflow=task.workflow, project=task.project, filter=task.filter, priority=task.priority, archive=task.archive) activity.save() task.activity = activity if save_task: task.save() return task.activity def update_activity(task): """ The activity (SAS_ID level) is updated whenever a task change status. Depending on the type of status change, certain calculations and updates are performed. Doing this on status change, instead of on-the-fly when a user enters a page, balances the load. - to '???' : check for incoming 'archive' json from archiver - to STORED : calculate quality - to ??? : calculate finished_fraction - to SCRUBBED, ARCHIVING, ARCHIVED, FINISHED : calculate ingested_fraction """ logger.info(f'update_activity for task {task.id} with sas_id {task.sas_id} and status {task.status}') # do not save the task, # because this function is called from signals where the task.save is delayed on purpose to avoid recursion activity = associate_activity(task, save_task=False) # depending on the status transition, perform calculations if task.status == State.STORED.value: logger.info(f'- calculate_qualities') # quality is calculated per task and per sas_id, reget the quality per sas_id try: activity.calculated_quality = task.calculated_qualities['per_sasid'] activity.save() except: pass # calculate the fraction and list of statusses of ingested tasks of this SAS_ID if task.status in [State.SCRUBBED.value, State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]: logger.info(f'- calculate_ingested_fraction') result = calculate_ingested_fraction(task) activity.ingested_fraction = result['completion'] activity.ingestq_status = result['status'] activity.save() # check of any task of this activity already has LTA information. If so, copy to the activity level if task.status in [State.ARCHIVING.value, State.ARCHIVED.value, State.FINISHING.value]: logger.info(f'- add archive json') for t in Task.objects.filter(sas_id=task.sas_id): try: if t.archive['sas_id_archived']: activity.archive = t.archive break except: pass activity.save()