from django.db import models from django.urls import reverse from django.utils.timezone import datetime # constants datetime_format_string = '%Y-%m-%dT%H:%M:%SZ' class Workflow(models.Model): workflow_uri = models.CharField(unique=True, max_length=30, blank=True, null=True) repository = models.CharField(max_length=100, blank=True, null=True) commit_id = models.CharField(max_length=15, blank=True, null=True) path = models.CharField(max_length=100, blank=True, null=True) oi_size_fraction = models.FloatField(blank=True, null=True) meta_scheduling = models.JSONField(null=True, blank=True) def __str__(self): return str(self.id) # convert the quality information from the JSONfield into a easy parsable list for the template def convert_quality_to_list_for_template(task): list = [] try: list.append(str(task.quality['uv-coverage'])) list.append(str(task.quality['sensitivity'])) list.append(str(task.quality['observing-conditions'])) list.append("(link)") list.append("(link)") list.append("(link)") list.append("(link)") except Exception as err: pass return list def convert_quality_to_shortlist_for_template(task): list = [] try: list.append(str(task.quality['uv-coverage'])) list.append(str(task.quality['sensitivity'])) list.append(str(task.quality['observing-conditions'])) except Exception as err: pass return list class Task(models.Model): # Task control properties task_type = models.CharField(max_length=20, default="regular") filter = models.CharField(max_length=30, blank=True, null=True) #environment = models.JSONField(null=True, blank=True) environment = models.CharField(max_length=255, blank=True, null=True) new_status = models.CharField(max_length=50, default="defining", null=True) status = models.CharField(db_index=True, default="unknown", max_length=50,blank=True, null=True) resume = models.BooleanField(verbose_name="Resume", default=True) creationTime = models.DateTimeField(verbose_name="CreationTime",default=datetime.utcnow, blank=True) priority = models.IntegerField(default=100, null=True) purge_policy = models.CharField(max_length=5, default="no", blank=True, null=True) stage_request_id = models.IntegerField(null=True) # LOFAR properties project = models.CharField(max_length=100, blank=True, null=True, default="unknown") sas_id = models.CharField(verbose_name="SAS_ID",max_length=15, blank=True, null=True) inputs = models.JSONField(null=True, blank=True) outputs = models.JSONField(null=True, blank=True) metrics = models.JSONField(null=True, blank=True) meta_scheduling = models.JSONField(null=True, blank=True) size_to_process = models.PositiveBigIntegerField(default=0, null=True, blank=True) size_processed = models.PositiveBigIntegerField(default=0, null=True, blank=True) total_processing_time = models.IntegerField(default=0, null=True, blank=True) # relationships workflow = models.ForeignKey(Workflow, related_name='tasks', on_delete=models.SET_NULL, null=True, blank=True) predecessor = models.ForeignKey('self', related_name='successors', on_delete=models.SET_NULL, null=True, blank=True) def __str__(self): return str(self.id) + ' - (' + self.task_type + ') - ' + str(self.sas_id) # this translates a view-name (from urls.py) back to a url, to avoid hardcoded url's in the html templates # bad : <td><a href="/atdb/taaks/{{ task.id }}/" target="_blank">{{ task.taskID }} </a> </td> # good: <td><a href="{{ task.get_absolute_url }}" target="_blank">{{ task.taskID }} </a> </td> def get_absolute_url(self): return reverse('task-detail-view-api', kwargs={'pk': self.pk}) @property def predecessor_status(self): try: return self.predecessor.status except: return "no_predecessor" @property def has_quality(self): # todo: check if there is a 'quality' structure in the 'task.outputs'? try: quality = self.outputs['quality'] return True except: try: quality = self.outputs[0]['quality'] return True except: return False @property def quality(self): # todo: check if there is a 'quality' structure in the 'task.outputs'? try: return self.outputs['quality'] except: try: return self.outputs[0]['quality'] except: return None @property def quality_as_list(self): try: q = convert_quality_to_list_for_template(self) return q except: return None @property def quality_as_shortlist(self): try: q = convert_quality_to_shortlist_for_template(self) return q except: return None class LogEntry(models.Model): cpu_cycles = models.IntegerField(null=True,blank=True) wall_clock_time = models.IntegerField(null=True,blank=True) url_to_log_file = models.CharField(max_length=100, blank=True, null=True) service = models.CharField(max_length=30, blank=True, null=True) step_name = models.CharField(max_length=30, blank=True, null=True) timestamp = models.DateTimeField(blank=True, null=True) status = models.CharField(max_length=50,default="defined", blank=True, null=True) description = models.CharField(max_length=100, blank=True, null=True) size_processed = models.PositiveBigIntegerField(default=0, null=True, blank=True) # relationships task = models.ForeignKey(Task, related_name='log_entries', on_delete=models.CASCADE, null=False) def __str__(self): return str(self.id)+ ' - '+ str(self.task)+' - '+self.status class Status(models.Model): name = models.CharField(max_length=50, default="unknown") timestamp = models.DateTimeField(default=datetime.utcnow, blank=True) # relationships task = models.ForeignKey(Task, related_name='status_history', on_delete=models.CASCADE, null=False) # the representation of the value in the REST API def __str__(self): formatedDate = self.timestamp.strftime(datetime_format_string) return str(self.name)+' ('+str(formatedDate)+')' class Configuration(models.Model): filter = models.CharField(max_length=30, blank=True, null=True) key = models.CharField(max_length=50) value = models.CharField(max_length=255) # the representation of the value in the REST API def __str__(self): return str(self.key) class Job(models.Model): type = models.CharField(db_index=True, max_length=20, default=None,null=True, blank=True) task_id = models.IntegerField(null=True, blank=True) job_id = models.IntegerField(null=True, blank=True) metadata = models.JSONField(null=True, blank=True) # the representation of the value in the REST API def __str__(self): return 'task_id:'+str(self.task_id)+', job_id:'+str(self.job_id) class PostProcessingRule(models.Model): aggregation_key = models.CharField(db_index=True, max_length=20, default=None,null=True, blank=True) trigger_status = models.CharField(db_index=True, default="unknown", max_length=50, blank=True, null=True) # relationships workflow_to_process = models.ForeignKey(Workflow, related_name='to_process', on_delete=models.SET_NULL, null=True, blank=True) workflow_to_apply = models.ForeignKey(Workflow, related_name='to_apply',on_delete=models.SET_NULL, null=True, blank=True) # the representation of the value in the REST API def __str__(self): return str(self.aggregation_key)+' - '+str(self.trigger_status) class LatestMonitor(models.Model): name = models.CharField(max_length=50, default="unknown") type = models.CharField(max_length=20, default="ldv-service", null=True, blank=True) timestamp = models.DateTimeField(default=datetime.utcnow, blank=True) hostname = models.CharField(max_length=50, default="unknown") process_id = models.IntegerField(null=True, blank=True) description = models.CharField(max_length=255, blank=True, null=True) status = models.CharField(max_length=50, default="ok", null=True) metadata = models.JSONField(null=True, blank=True) # the representation of the value in the REST API def __str__(self): return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status class Monitor(models.Model): name = models.CharField(max_length=50, default="unknown") type = models.CharField(max_length=20, default="ldv-service", null=True, blank=True) timestamp = models.DateTimeField(default=datetime.utcnow, blank=True) hostname = models.CharField(max_length=50, default="unknown") process_id = models.IntegerField(null=True, blank=True) description = models.CharField(max_length=255, blank=True, null=True) status = models.CharField(max_length=50, default="ok", null=True) metadata = models.JSONField(null=True, blank=True) def save(self, *args, **kwargs): # check if this combination of service name + hostname already exists # in the LatestMonitor, and update if it is newer. try: latestMonitor = LatestMonitor.objects.get(name=self.name,hostname=self.hostname) latestMonitor.delete() except: pass # this combination of name and hostname didn't yet exist, create it. latestMonitor = LatestMonitor( name=self.name, type=self.type, timestamp=self.timestamp, hostname = self.hostname, process_id = self.process_id, description = self.description, status = self.status, metadata = self.metadata ) latestMonitor.save() # finally save the Monitor info itself also super(Monitor, self).save(*args, **kwargs) # the representation of the value in the REST API def __str__(self): return str(self.name) + ' - ('+ self.hostname+') - '+str(self.timestamp) + ' - ' + self.status