From a3a92e7416ae0c2cb35bd3b71109422b21d011ee Mon Sep 17 00:00:00 2001
From: goei <JsXLRu>
Date: Thu, 22 Oct 2020 17:28:50 +0200
Subject: [PATCH] TMSS-324 Add SIPIdentifier table. Add global_sip as
foreignkey to the SubTask, Dataproduct and SAP table. Remove identifier from
SAP specifications_doc because its in model now.
---
.../tmss/tmssapp/migrations/0001_initial.py | 81 +++++++++++++------
.../src/tmss/tmssapp/models/scheduling.py | 27 +++++++
.../tmss/tmssapp/schemas/sap_template-1.json | 12 ---
.../tmss/tmssapp/serializers/scheduling.py | 14 ++++
SAS/TMSS/src/tmss/tmssapp/subtasks.py | 10 ++-
.../src/tmss/tmssapp/viewsets/scheduling.py | 6 ++
SAS/TMSS/src/tmss/urls.py | 1 +
7 files changed, 111 insertions(+), 40 deletions(-)
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index e3ccbb0b1e8..a9eb77f656b 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2020-10-16 10:16
+# Generated by Django 3.0.7 on 2020-10-22 07:24
from django.conf import settings
import django.contrib.postgres.fields
@@ -62,7 +62,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('rcus', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=128)),
('inputs', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, size=128)),
],
@@ -78,7 +78,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('location', models.CharField(help_text='Human-readable location of the cluster.', max_length=128)),
('archive_site', models.BooleanField(default=False, help_text='TRUE if this cluster is an archive site, FALSE if not (f.e. a local cluster, or user-owned cluster).')),
],
@@ -94,7 +94,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -117,7 +117,7 @@ class Migration(migrations.Migration):
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128, primary_key=True, serialize=False)),
('start', models.DateTimeField(help_text='Moment at which the cycle starts, that is, when its projects can run.')),
('stop', models.DateTimeField(help_text='Moment at which the cycle officially ends.')),
@@ -186,7 +186,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -215,7 +215,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -344,7 +344,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('capacity', models.BigIntegerField(help_text='Capacity in bytes')),
('directory', models.CharField(help_text='Root directory under which we are allowed to write our data.', max_length=1024)),
],
@@ -369,7 +369,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('create_function', models.CharField(help_text='Python function to call to execute the generator.', max_length=128)),
@@ -393,7 +393,7 @@ class Migration(migrations.Migration):
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128, primary_key=True, serialize=False)),
('priority_rank', models.FloatField(help_text='Priority of this project w.r.t. other projects. Projects can interrupt observations of lower-priority projects.')),
('trigger_priority', models.IntegerField(default=1000, help_text='Priority of this project w.r.t. triggers.')),
@@ -438,7 +438,7 @@ class Migration(migrations.Migration):
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128, primary_key=True, serialize=False)),
],
options={
@@ -475,7 +475,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -500,7 +500,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -525,7 +525,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('generator_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Parameters for the generator (NULLable).', null=True)),
],
options={
@@ -540,7 +540,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling and/or quality requirements for this scheduling unit (IMMUTABLE).')),
('do_cancel', models.BooleanField()),
],
@@ -556,7 +556,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling and/or quality requirements for this run.')),
('generator_instance_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Parameter value that generated this run draft (NULLable).', null=True)),
('scheduling_constraints_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling Constraints for this run.', null=True)),
@@ -573,7 +573,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
('template', django.contrib.postgres.fields.jsonb.JSONField(help_text='JSON-data compliant with the JSON-schema in the scheduling_unit_template. This observation strategy template like a predefined recipe with all the correct settings, and defines which parameters the user can alter.')),
],
@@ -589,7 +589,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -597,6 +597,20 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='SIPidentifier',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('source', models.CharField(help_text='Source name', max_length=128)),
+ ('unique_identifier', models.CharField(help_text='Unique global identifier.', max_length=255, unique=True)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='StationType',
fields=[
@@ -679,7 +693,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('queue', models.BooleanField(default=False)),
@@ -714,7 +728,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schedulings for this task (IMMUTABLE).')),
('do_cancel', models.BooleanField(help_text='Cancel this task.')),
],
@@ -742,7 +756,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Specifications for this task.')),
],
options={
@@ -783,7 +797,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
@@ -821,7 +835,7 @@ class Migration(migrations.Migration):
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
- ('description', models.CharField(blank=True, help_text='A longer description of this object.', max_length=255)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('validation_code_js', models.CharField(blank=True, default='', help_text='JavaScript code for additional (complex) validation.', max_length=128)),
@@ -1057,6 +1071,11 @@ class Migration(migrations.Migration):
name='created_or_updated_by_user',
field=models.ForeignKey(editable=False, help_text='The user who created / updated the subtask.', null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL),
),
+ migrations.AddField(
+ model_name='subtask',
+ name='global_sip',
+ field=models.ForeignKey(editable=False, help_text='The global unique identifier for LTA SIP.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SIPidentifier'),
+ ),
migrations.AddField(
model_name='subtask',
name='schedule_method',
@@ -1077,6 +1096,10 @@ class Migration(migrations.Migration):
name='task_blueprint',
field=models.ForeignKey(help_text='Task Blueprint to which this Subtask belongs.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='subtasks', to='tmssapp.TaskBlueprint'),
),
+ migrations.AddIndex(
+ model_name='sipidentifier',
+ index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sip_tags_bbce92_gin'),
+ ),
migrations.AddConstraint(
model_name='schedulingunittemplate',
constraint=models.UniqueConstraint(fields=('name', 'version'), name='schedulingunittemplate_unique_name_version'),
@@ -1149,6 +1172,11 @@ class Migration(migrations.Migration):
model_name='saptemplate',
constraint=models.UniqueConstraint(fields=('name', 'version'), name='saptemplate_unique_name_version'),
),
+ migrations.AddField(
+ model_name='sap',
+ name='global_sip',
+ field=models.ForeignKey(editable=False, help_text='The global unique identifier for LTA SIP.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SIPidentifier'),
+ ),
migrations.AddField(
model_name='sap',
name='specifications_template',
@@ -1281,6 +1309,11 @@ class Migration(migrations.Migration):
name='feedback_template',
field=models.ForeignKey(help_text='Schema used for feedback_doc.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.DataproductFeedbackTemplate'),
),
+ migrations.AddField(
+ model_name='dataproduct',
+ name='global_sip',
+ field=models.ForeignKey(editable=False, help_text='The global unique identifier for LTA SIP.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SIPidentifier'),
+ ),
migrations.AddField(
model_name='dataproduct',
name='producer',
@@ -1289,7 +1322,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='dataproduct',
name='sap',
- field=models.ForeignKey(help_text='SAP this dataproduct was generated out of (NULLable).', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='SAP_dataproducts', to='tmssapp.SAP'),
+ field=models.ForeignKey(help_text='SAP this dataproduct was generated out of (NULLable).', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='dataproducts', to='tmssapp.SAP'),
),
migrations.AddField(
model_name='dataproduct',
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index a6932d705e3..ddbd8e1f730 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -23,6 +23,7 @@ from lofar.sas.tmss.client.tmssbuslistener import DEFAULT_TMSS_SUBTASK_NOTIFICAT
from lofar.common.util import single_line_with_single_spaces
from django.conf import settings
from lofar.sas.resourceassignment.resourceassignmentservice.rpc import RADBRPC
+import uuid
#
# I/O
#
@@ -31,6 +32,19 @@ from lofar.sas.resourceassignment.resourceassignmentservice.rpc import RADBRPC
# Choices
#
+
+def generate_unique_identifier_for_SIP_when_needed(model):
+ """
+ Create an Unique Identifier for given model class if not exist (None)
+ UUID is 16 bytes per definition so have to store as string.
+ This string should have integer values as defined for SIP validation in SIP xsd schema (IdentifierType)
+ """
+ print("generate_unique_identifier_for_SIP_when_needed for %s %s" % (str(type(model)), model.id))
+ if model.id is not None and model.global_sip is None:
+ unique_id = uuid.uuid4().int
+ model.global_sip = SIPidentifier.objects.create(unique_identifier=str(unique_id), source="TMSS")
+
+
class SubtaskState(AbstractChoice):
"""Defines the model and predefined list of possible SubtaskStatusChoice's for Subtask.
The items in the Choices class below are automagically populated into the database via a data migration."""
@@ -150,6 +164,7 @@ class Subtask(BasicCommon):
# resource_claim = ForeignKey("ResourceClaim", null=False, on_delete=PROTECT) # todo <-- how is this external reference supposed to work?
created_or_updated_by_user = ForeignKey(User, null=True, editable=False, on_delete=PROTECT, help_text='The user who created / updated the subtask.')
raw_feedback = CharField(null=True, max_length=1048576, help_text='The raw feedback for this Subtask')
+ global_sip = ForeignKey('SIPidentifier', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -194,6 +209,7 @@ class Subtask(BasicCommon):
creating = self._state.adding # True on create, False on update
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ generate_unique_identifier_for_SIP_when_needed(self)
if self.state.value == SubtaskState.Choices.SCHEDULED.value and self.__original_state_id == SubtaskState.Choices.SCHEDULING.value:
if self.start_time is None:
@@ -288,12 +304,15 @@ class SubtaskOutput(BasicCommon):
class SAP(BasicCommon):
specifications_doc = JSONField(help_text='SAP properties.')
specifications_template = ForeignKey('SAPTemplate', null=False, on_delete=CASCADE, help_text='Schema used for specifications_doc.')
+ global_sip = ForeignKey('SIPidentifier', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ generate_unique_identifier_for_SIP_when_needed(self)
super().save(force_insert, force_update, using, update_fields)
+
class Dataproduct(BasicCommon):
"""
A data product represents an atomic dataset, produced and consumed by subtasks. The consumed dataproducts are those
@@ -316,13 +335,16 @@ class Dataproduct(BasicCommon):
feedback_doc = JSONField(help_text='Dataproduct properties, as reported by the producing process.')
feedback_template = ForeignKey('DataproductFeedbackTemplate', on_delete=PROTECT, help_text='Schema used for feedback_doc.')
sap = ForeignKey('SAP', on_delete=PROTECT, null=True, related_name="dataproducts", help_text='SAP this dataproduct was generated out of (NULLable).')
+ global_sip = ForeignKey('SIPidentifier', editable=False, null=True, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
annotate_validate_add_defaults_to_doc_using_template(self, 'feedback_doc', 'feedback_template')
+ generate_unique_identifier_for_SIP_when_needed(self)
super().save(force_insert, force_update, using, update_fields)
+
class AntennaSet(NamedCommon):
station_type = ForeignKey('StationType', null=False, on_delete=PROTECT)
rcus = ArrayField(IntegerField(), size=128, blank=False)
@@ -368,3 +390,8 @@ class DataproductHash(BasicCommon):
algorithm = ForeignKey('Algorithm', null=False, on_delete=PROTECT, help_text='Algorithm used (MD5, AES256).')
hash = CharField(max_length=128, help_text='Hash value.')
+
+class SIPidentifier(BasicCommon):
+ source = CharField(max_length=128, help_text='Source name')
+ # the identifier can not be a BigInteger because its 8 bytes and UUID is by defintion 16 byte, so store as char
+ unique_identifier = CharField(max_length=255, unique=True, help_text='Unique global identifier.')
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/sap_template-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/sap_template-1.json
index b4f6eb64f1e..e80661a829d 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/sap_template-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/sap_template-1.json
@@ -5,17 +5,6 @@
"type": "object",
"default": {},
"properties": {
- "identifiers": {
- "type": "object",
- "properties": {
- "SIP": {
- "type": "string",
- "default": ""
- }
- },
- "additionalProperties": false,
- "default": {}
- },
"measurement_type": {
"type": "string",
"enum": ["calibrator", "target"],
@@ -59,7 +48,6 @@
}
},
"required": [
- "identifiers",
"name",
"pointing",
"time",
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
index 091a2352b42..a2a10449dae 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/scheduling.py
@@ -160,3 +160,17 @@ class SAPTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.SAPTemplate
fields = '__all__'
+
+
+class SAPSerializer(RelationalHyperlinkedModelSerializer):
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
+
+ class Meta:
+ model = models.SAP
+ fields = '__all__'
+
+
+class SIPidentifierSerializer(RelationalHyperlinkedModelSerializer):
+ class Meta:
+ model = models.SIPidentifier
+ fields = '__all__'
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 94e2ce2064b..07d3a41cda1 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -556,6 +556,7 @@ def schedule_qafile_subtask(qafile_subtask: Subtask):
feedback_template=DataproductFeedbackTemplate.objects.get(name="empty"),
sap=None # todo: do we need to point to a SAP here? Of which dataproduct then?
)
+ qafile_subtask_dataproduct.save()
# step 5: set state to SCHEDULED (resulting in the qaservice to pick this subtask up and run it)
qafile_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULED.value)
@@ -608,6 +609,7 @@ def schedule_qaplots_subtask(qaplots_subtask: Subtask):
feedback_template=DataproductFeedbackTemplate.objects.get(name="empty"),
sap=None # todo: do we need to point to a SAP here? Of which dataproduct then?
)
+ qaplots_subtask_dataproduct.save()
# step 5: set state to SCHEDULED (resulting in the qaservice to pick this subtask up and run it)
qaplots_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULED.value)
@@ -725,7 +727,6 @@ def schedule_observation_subtask(observation_subtask: Subtask):
antennafields += [{"station": station, "field": field, "type": antennaset.split('_')[0]} for field in fields]
sap = SAP.objects.create(specifications_doc={ "name": "%s_%s" % (observation_subtask.id, pointing['name']),
- "identifiers": {}, # todo: TMSS-324
"pointing": pointing['pointing'],
"time": {"start_time": observation_subtask.start_time.isoformat(),
"duration": (observation_subtask.stop_time - observation_subtask.start_time).total_seconds()},
@@ -735,9 +736,9 @@ def schedule_observation_subtask(observation_subtask: Subtask):
}
},
specifications_template=SAPTemplate.objects.get(name="SAP"))
-
+ sap.save()
for sb_nr in pointing['subbands']:
- Dataproduct.objects.create(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
+ dp = Dataproduct.objects.create(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
directory=directory,
dataformat=Dataformat.objects.get(value="MeasurementSet"),
datatype=Datatype.objects.get(value="visibilities"), # todo: is this correct?
@@ -749,7 +750,7 @@ def schedule_observation_subtask(observation_subtask: Subtask):
size=0 if sb_nr%10==0 else 1024*1024*1024*sb_nr,
expected_size=1024*1024*1024*sb_nr,
sap=sap)
-
+ dp.save()
# step 4: resource assigner (if possible)
_assign_resources(observation_subtask)
@@ -829,6 +830,7 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
feedback_template=dataproduct_feedback_template,
sap=input_dp.sap)
+ output_dp.save()
DataproductTransform.objects.create(input=input_dp, output=output_dp, identity=False)
output_dps.append(output_dp)
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
index 601321cf92d..54ed8f42527 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py
@@ -328,6 +328,12 @@ class SAPViewSet(LOFARViewSet):
serializer = serializers.DataproductSerializer(sap.dataproducts, many=True, context={'request': request})
return RestResponse(serializer.data)
+
class SAPTemplateViewSet(AbstractTemplateViewSet):
queryset = models.SAPTemplate.objects.all()
serializer_class = serializers.SAPTemplateSerializer
+
+
+class SIPidentifierViewSet(LOFARViewSet):
+ queryset = models.SIPidentifier.objects.all()
+ serializer_class = serializers.SIPidentifierSerializer
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index 85712c60e16..7d5e08c90cb 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -195,6 +195,7 @@ router.register(r'dataproduct_hash', viewsets.DataproductHashViewSet)
router.register(r'subtask_state_log', viewsets.SubtaskStateLogViewSet)
router.register(r'user', viewsets.UserViewSet)
router.register(r'sap', viewsets.SAPViewSet)
+router.register(r'sip_identifier', viewsets.SIPidentifierViewSet)
# ---
--
GitLab