From 0a283258689a1a9ea518ed535285095f07b7e393 Mon Sep 17 00:00:00 2001
From: goei <JsXLRu>
Date: Thu, 29 Oct 2020 19:27:18 +0100
Subject: [PATCH] TMSS-324 Use AutoField for generate unique identifier for SIP
---
SAS/TMSS/src/tmss/tmssapp/adapters/sip.py | 10 +++++-----
.../tmss/tmssapp/migrations/0001_initial.py | 13 ++++++------
.../src/tmss/tmssapp/models/scheduling.py | 20 ++++++++-----------
SAS/TMSS/test/t_adapter.py | 6 +++---
4 files changed, 22 insertions(+), 27 deletions(-)
diff --git a/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py b/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
index 43cec8fc059..28c0e733214 100644
--- a/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
+++ b/SAS/TMSS/src/tmss/tmssapp/adapters/sip.py
@@ -131,7 +131,7 @@ def create_sip_representation_for_subtask(subtask: Subtask):
:return: A siplib.Observation object or one of the various siplib pipeline object flavors
"""
# determine common properties
- subtask_sip_identifier = get_siplib_identifier(subtask.global_sip, "Subtask id=%s" % subtask.id)
+ subtask_sip_identifier = get_siplib_identifier(subtask.global_identifier, "Subtask id=%s" % subtask.id)
name = str(subtask.id)
process_map = siplib.ProcessMap(strategyname=subtask.specifications_template.name,
strategydescription=subtask.specifications_template.description,
@@ -171,7 +171,7 @@ def create_sip_representation_for_subtask(subtask: Subtask):
elif subtask.specifications_template.type.value == SubtaskType.Choices.PIPELINE.value:
sourcedata_identifiers = []
for input in subtask.inputs.all():
- sourcedata_identifiers += [get_siplib_identifier(dp.global_sip, "Dataproduct id=%s" % dp.id) for dp in input.dataproducts.all()] # todo: use correct id, lookup based on TMSS reference or so, tbd
+ sourcedata_identifiers += [get_siplib_identifier(dp.global_identifier, "Dataproduct id=%s" % dp.id) for dp in input.dataproducts.all()] # todo: use correct id, lookup based on TMSS reference or so, tbd
if not sourcedata_identifiers:
raise TMSSException("There seems to be no subtask input associated to your pipeline subtask id %s. Please define what data the pipeline processed." % subtask.id)
@@ -283,18 +283,18 @@ def create_sip_representation_for_dataproduct(dataproduct: Dataproduct):
logger.warning("Could not determine the type of dataproduct id %s (%s). Falling back to %s" % (dataproduct.id, err, dataproduct_fileformat))
dataproduct_map = siplib.DataProductMap(type=dataproduct_type,
- identifier=get_siplib_identifier(dataproduct.global_sip, "Dataproduct %s" % dataproduct.id),
+ identifier=get_siplib_identifier(dataproduct.global_identifier, "Dataproduct %s" % dataproduct.id),
size=dataproduct.size if dataproduct.size else 0,
filename=dataproduct.filename,
fileformat=dataproduct_fileformat,
storage_writer=storage_writer_map[dataproduct.feedback_doc["samples"]["writer"] if 'samples' in dataproduct.feedback_doc else 'unknown'], # todo: verify we can use the feedback_doc here and remove the old method | storage_writer_map[dataproduct.producer.subtask.task_blueprint.specifications_doc.get("storagemanager", 'unknown')],
storage_writer_version=dataproduct.feedback_doc["samples"]["writer_version"] if 'samples' in dataproduct.feedback_doc else 'unknown',
- process_identifier=get_siplib_identifier(dataproduct.producer.subtask.global_sip, "Producer Subtask %s" % dataproduct.producer.subtask.id))
+ process_identifier=get_siplib_identifier(dataproduct.producer.subtask.global_identifier, "Producer Subtask %s" % dataproduct.producer.subtask.id))
if dataproduct.dataformat.value == Dataformat.Choices.MEASUREMENTSET.value: # <- This is the only one we currently need for UC1
sip_dataproduct = siplib.CorrelatedDataProduct(
dataproduct_map,
- subarraypointing_identifier=get_siplib_identifier(dataproduct.sap.global_sip, "SAP %s" % dataproduct.sap.id),
+ subarraypointing_identifier=get_siplib_identifier(dataproduct.sap.global_identifier, "SAP %s" % dataproduct.sap.id),
subband=dataproduct.feedback_doc['frequency']['subbands'][0],
starttime=dataproduct.feedback_doc['time']['start_time'],
duration=isodate.duration_isoformat(datetime.timedelta(seconds=dataproduct.feedback_doc['time']['duration'])),
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index a9eb77f656b..7faae82d9b3 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.7 on 2020-10-22 07:24
+# Generated by Django 3.0.7 on 2020-10-29 16:37
from django.conf import settings
import django.contrib.postgres.fields
@@ -600,12 +600,11 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='SIPidentifier',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('source', models.CharField(help_text='Source name', max_length=128)),
- ('unique_identifier', models.CharField(help_text='Unique global identifier.', max_length=255, unique=True)),
+ ('unique_identifier', models.BigAutoField(help_text='Unique global identifier.', primary_key=True, serialize=False)),
],
options={
'abstract': False,
@@ -1073,7 +1072,7 @@ class Migration(migrations.Migration):
),
migrations.AddField(
model_name='subtask',
- name='global_sip',
+ name='global_identifier',
field=models.ForeignKey(editable=False, help_text='The global unique identifier for LTA SIP.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SIPidentifier'),
),
migrations.AddField(
@@ -1174,7 +1173,7 @@ class Migration(migrations.Migration):
),
migrations.AddField(
model_name='sap',
- name='global_sip',
+ name='global_identifier',
field=models.ForeignKey(editable=False, help_text='The global unique identifier for LTA SIP.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SIPidentifier'),
),
migrations.AddField(
@@ -1205,7 +1204,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='cycles',
- field=models.ManyToManyField(blank=True, help_text='Cycles to which this project belongs (NULLable).', related_name='projects', to='tmssapp.Cycle'),
+ field=models.ManyToManyField(help_text='Cycles to which this project belongs (NULLable).', null=True, related_name='projects', to='tmssapp.Cycle'),
),
migrations.AddField(
model_name='project',
@@ -1311,7 +1310,7 @@ class Migration(migrations.Migration):
),
migrations.AddField(
model_name='dataproduct',
- name='global_sip',
+ name='global_identifier',
field=models.ForeignKey(editable=False, help_text='The global unique identifier for LTA SIP.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SIPidentifier'),
),
migrations.AddField(
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index 6d55dd0e6fe..a6257840944 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
from datetime import datetime, timedelta
from django.db.models import ForeignKey, CharField, DateTimeField, BooleanField, IntegerField, BigIntegerField, \
- ManyToManyField, CASCADE, SET_NULL, PROTECT, QuerySet
+ ManyToManyField, CASCADE, SET_NULL, PROTECT, QuerySet, BigAutoField
from django.contrib.postgres.fields import ArrayField, JSONField
from django.contrib.auth.models import User
from .specification import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template
@@ -37,13 +37,10 @@ import uuid
def generate_unique_identifier_for_SIP_when_needed(model):
"""
Create an Unique Identifier for given model class if not exist (None)
- UUID is 16 bytes per definition so have to store as string.
- This string should have integer values as defined for SIP validation in SIP xsd schema (IdentifierType)
+ We just use an Auto Increment ID which is 64 bit
"""
- print("generate_unique_identifier_for_SIP_when_needed for %s %s" % (str(type(model)), model.id))
- if model.id is not None and model.global_sip is None:
- unique_id = uuid.uuid4().int
- model.global_sip = SIPidentifier.objects.create(unique_identifier=str(unique_id), source="TMSS")
+ if model.id is not None and model.global_identifier is None:
+ model.global_identifier = SIPidentifier.objects.create(source="TMSS")
class SubtaskState(AbstractChoice):
@@ -165,7 +162,7 @@ class Subtask(BasicCommon):
# resource_claim = ForeignKey("ResourceClaim", null=False, on_delete=PROTECT) # todo <-- how is this external reference supposed to work?
created_or_updated_by_user = ForeignKey(User, null=True, editable=False, on_delete=PROTECT, help_text='The user who created / updated the subtask.')
raw_feedback = CharField(null=True, max_length=1048576, help_text='The raw feedback for this Subtask')
- global_sip = ForeignKey('SIPidentifier', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
+ global_identifier = ForeignKey('SIPidentifier', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -316,7 +313,7 @@ class SubtaskOutput(BasicCommon):
class SAP(BasicCommon):
specifications_doc = JSONField(help_text='SAP properties.')
specifications_template = ForeignKey('SAPTemplate', null=False, on_delete=CASCADE, help_text='Schema used for specifications_doc.')
- global_sip = ForeignKey('SIPidentifier', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
+ global_identifier = ForeignKey('SIPidentifier', null=True, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
@@ -347,7 +344,7 @@ class Dataproduct(BasicCommon):
feedback_doc = JSONField(help_text='Dataproduct properties, as reported by the producing process.')
feedback_template = ForeignKey('DataproductFeedbackTemplate', on_delete=PROTECT, help_text='Schema used for feedback_doc.')
sap = ForeignKey('SAP', on_delete=PROTECT, null=True, related_name="dataproducts", help_text='SAP this dataproduct was generated out of (NULLable).')
- global_sip = ForeignKey('SIPidentifier', editable=False, null=True, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
+ global_identifier = ForeignKey('SIPidentifier', editable=False, null=True, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
@@ -405,5 +402,4 @@ class DataproductHash(BasicCommon):
class SIPidentifier(BasicCommon):
source = CharField(max_length=128, help_text='Source name')
- # the identifier can not be a BigInteger because its 8 bytes and UUID is by defintion 16 byte, so store as char
- unique_identifier = CharField(max_length=255, unique=True, help_text='Unique global identifier.')
+ unique_identifier = BigAutoField(primary_key=True, help_text='Unique global identifier.')
diff --git a/SAS/TMSS/test/t_adapter.py b/SAS/TMSS/test/t_adapter.py
index ca3306fc9bd..4b11c380b6f 100755
--- a/SAS/TMSS/test/t_adapter.py
+++ b/SAS/TMSS/test/t_adapter.py
@@ -103,9 +103,9 @@ class SIPadapterTest(unittest.TestCase):
self.assertIn(str(feedback_doc['time']['start_time']), sip.get_prettyxml())
self.assertIn(constants.FILTERSELECTIONTYPE_210_250_MHZ, sip.get_prettyxml()) # specifications_doc: "HBA_210_250"
- self.assertIn(subtask.global_sip.unique_identifier, sip.get_prettyxml())
- self.assertIn(dataproduct.global_sip.unique_identifier, sip.get_prettyxml())
- self.assertIn(sap.global_sip.unique_identifier, sip.get_prettyxml())
+ self.assertIn(str(subtask.global_identifier.unique_identifier), sip.get_prettyxml())
+ self.assertIn(str(dataproduct.global_identifier.unique_identifier), sip.get_prettyxml())
+ self.assertIn(str(sap.global_identifier.unique_identifier), sip.get_prettyxml())
all_sip_ids = list(models.SIPidentifier.objects.all())
self.assertEqual(nbr_expected_sip_identifiers_before_setup+3, len(all_sip_ids))
--
GitLab