LSMR-14: implemented (semi)static choices with foreignkeys and automatic...

LSMR-14: implemented (semi)static choices with foreignkeys and automatic population of the inital database.

.gitattributes

@@ -4245,12 +4245,14 @@ SAS/LSMR/src/lsmr/lsmrapp/__init__.py -text
 SAS/LSMR/src/lsmr/lsmrapp/admin.py -text
 SAS/LSMR/src/lsmr/lsmrapp/apps.py -text
 SAS/LSMR/src/lsmr/lsmrapp/migrations/0001_initial.py -text
-SAS/LSMR/src/lsmr/lsmrapp/migrations/0002_auto_20180710_0922.py -text
+SAS/LSMR/src/lsmr/lsmrapp/migrations/0002_auto_20180711_1300.py -text
+SAS/LSMR/src/lsmr/lsmrapp/migrations/0003_auto_20180711_1300.py -text
 SAS/LSMR/src/lsmr/lsmrapp/migrations/CMakeLists.txt -text
 SAS/LSMR/src/lsmr/lsmrapp/migrations/__init__.py -text
 SAS/LSMR/src/lsmr/lsmrapp/models/CMakeLists.txt -text
 SAS/LSMR/src/lsmr/lsmrapp/models/__init__.py -text
 SAS/LSMR/src/lsmr/lsmrapp/models/specification.py -text
+SAS/LSMR/src/lsmr/lsmrapp/populate.py -text
 SAS/LSMR/src/lsmr/lsmrapp/serializers/CMakeLists.txt -text
 SAS/LSMR/src/lsmr/lsmrapp/serializers/__init__.py -text
 SAS/LSMR/src/lsmr/lsmrapp/serializers/specification.py -text

SAS/LSMR/src/lsmr/lsmrapp/CMakeLists.txt

@@ -5,6 +5,7 @@ set(_py_files
     __init__.py
     admin.py
     apps.py
+    populate.py
     )
 
 python_install(${_py_files}

SAS/LSMR/src/lsmr/lsmrapp/migrations/0002_auto_20180710_0922.py → SAS/LSMR/src/lsmr/lsmrapp/migrations/0002_auto_20180711_1300.py

-# Generated by Django 2.0.6 on 2018-07-10 09:22
+# Generated by Django 2.0.7 on 2018-07-11 13:00
 
 import django.contrib.postgres.fields
 import django.contrib.postgres.fields.jsonb
@@ -15,6 +15,15 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.CreateModel(
+            name='CopyReasonChoice',
+            fields=[
+                ('value', models.CharField(editable=False, max_length=30, primary_key=True, serialize=False, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
         migrations.CreateModel(
             name='Cycle',
             fields=[
@@ -29,6 +38,24 @@ class Migration(migrations.Migration):
                 'abstract': False,
             },
         ),
+        migrations.CreateModel(
+            name='DataformatChoice',
+            fields=[
+                ('value', models.CharField(editable=False, max_length=30, primary_key=True, serialize=False, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='DatatypeChoice',
+            fields=[
+                ('value', models.CharField(editable=False, max_length=30, primary_key=True, serialize=False, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
         migrations.CreateModel(
             name='GeneratorTemplate',
             fields=[
@@ -64,6 +91,12 @@ class Migration(migrations.Migration):
                 'abstract': False,
             },
         ),
+        migrations.CreateModel(
+            name='RoleChoice',
+            fields=[
+                ('value', models.CharField(editable=False, max_length=30, primary_key=True, serialize=False, unique=True)),
+            ],
+        ),
         migrations.CreateModel(
             name='Run',
             fields=[
@@ -87,9 +120,9 @@ class Migration(migrations.Migration):
                 ('update', models.DateTimeField(auto_now=True)),
                 ('name', models.CharField(max_length=30)),
                 ('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField()),
-                ('copy_reason', models.CharField(choices=[('TEMPLATE', 'TEMPLATE'), ('REPEATED', 'REPEATED')], max_length=30)),
                 ('generator_param', models.CharField(max_length=200)),
                 ('copies', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='copied_from', to='lsmrapp.RunBlueprint')),
+                ('copy_reason', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.CopyReasonChoice')),
             ],
             options={
                 'abstract': False,
@@ -140,9 +173,8 @@ class Migration(migrations.Migration):
             name='WorkIORoles',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('role', models.CharField(choices=[('CORRELATOR', 'correlator'), ('BEAMFORMER', 'beamformer'), ('INSPECTION_PLOTS', 'inspection plots'), ('CALIBRATOR', 'calibrator'), ('TARGET', 'target'), ('INPUT_OUTPUT', 'input, output')], max_length=30)),
-                ('datatype', models.CharField(choices=[('VISIBILITIES', 'visibilities'), ('TIME_SERIES', 'time series'), ('INSTRUMENT_MODEL', 'instrument model'), ('IMAGE', 'image'), ('QUALITY', 'quality')], max_length=30)),
-                ('dataformat', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(choices=[('MEASUREMENTSET', 'MeasurementSet'), ('HDF5', 'HDF5')], max_length=30), blank=True, size=8)),
+                ('dataformat', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.DataformatChoice')),
+                ('datatype', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.DatatypeChoice')),
             ],
         ),
         migrations.CreateModel(
@@ -185,8 +217,8 @@ class Migration(migrations.Migration):
                 ('update', models.DateTimeField(auto_now=True)),
                 ('name', models.CharField(max_length=30)),
                 ('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField()),
-                ('copy_reason', models.CharField(choices=[('TEMPLATE', 'TEMPLATE'), ('REPEATED', 'REPEATED')], max_length=30)),
                 ('copies', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='copied_from', to='lsmrapp.WorkRequestBlueprint')),
+                ('copy_reason', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.CopyReasonChoice')),
             ],
             options={
                 'abstract': False,
@@ -200,7 +232,6 @@ class Migration(migrations.Migration):
                 ('creation', models.DateTimeField(auto_now_add=True)),
                 ('update', models.DateTimeField(auto_now=True)),
                 ('selection_doc', django.contrib.postgres.fields.jsonb.JSONField()),
-                ('dataformat', models.CharField(choices=[('MEASUREMENTSET', 'MeasurementSet'), ('HDF5', 'HDF5')], max_length=30)),
             ],
             options={
                 'abstract': False,
@@ -215,9 +246,9 @@ class Migration(migrations.Migration):
                 ('update', models.DateTimeField(auto_now=True)),
                 ('name', models.CharField(max_length=30)),
                 ('selection_doc', django.contrib.postgres.fields.jsonb.JSONField()),
-                ('dataformat', models.CharField(choices=[('MEASUREMENTSET', 'MeasurementSet'), ('HDF5', 'HDF5')], max_length=30)),
                 ('blueprint', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.WorkRelationSelectionTemplate')),
                 ('consumer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_request_relation_blueprint_consumer', to='lsmrapp.WorkRequestBlueprint')),
+                ('dataformat', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.DataformatChoice')),
                 ('input_role', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_request_relation_blueprint_input_role', to='lsmrapp.WorkIORoles')),
                 ('output_role', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_request_relation_blueprint_output_role', to='lsmrapp.WorkIORoles')),
                 ('producer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_request_relation_blueprint_producer', to='lsmrapp.WorkRequestBlueprint')),
@@ -254,6 +285,11 @@ class Migration(migrations.Migration):
             name='consumer',
             field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_request_relation_consumer', to='lsmrapp.WorkRequest'),
         ),
+        migrations.AddField(
+            model_name='workrequestrelation',
+            name='dataformat',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.DataformatChoice'),
+        ),
         migrations.AddField(
             model_name='workrequestrelation',
             name='input_role',
@@ -289,6 +325,11 @@ class Migration(migrations.Migration):
             name='outputs',
             field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='role_output', to='lsmrapp.WorkRequestTemplate'),
         ),
+        migrations.AddField(
+            model_name='workioroles',
+            name='role',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lsmrapp.RoleChoice'),
+        ),
         migrations.AddField(
             model_name='runblueprint',
             name='generator_source',

SAS/LSMR/src/lsmr/lsmrapp/migrations/0003_auto_20180711_1300.py

+# Generated by Django 2.0.7 on 2018-07-11 13:00
+
+from django.db import migrations
+
+from ..populate import *
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('lsmrapp', '0002_auto_20180711_1300'),
+    ]
+
+    operations = [ migrations.RunPython(populate_choices) ]

SAS/LSMR/src/lsmr/lsmrapp/models/specification.py

@@ -52,9 +52,39 @@ class Tags(Model):
 # I/O
 #
 
-# enums
+# choices
 
-class RoleChoice(Enum):
+class AbstractChoice(Model):
+    """
+    Abstract class for all derived 'choices' models.
+    We define a 'choice' as an item that you can pick from a predefined list.
+    In the derived classes, we use an enum.Enum to define such a predefined list here in code.
+    All values of the enums are then put automagically into the database in the populate module, which
+    is/can_be/should_be called in the last migration step to populate the database with inital values
+    for our 'static choices'.
+
+    Design decision: Django also provides the 'choices' property on fields which sort of limits the number of choices
+    one can make, and which sort of does some validation. In our opinion the validation is done in the wrong place, and
+    no data consistency is enforced.
+    So, we decided to follow Django's own hint, see https://docs.djangoproject.com/en/2.0/ref/models/fields/#choices
+    "you’re probably better off using a proper database table with a ForeignKey"
+
+    You can find the derived AbstractChoice classes being used as ForeignKey in other models, thus enforcing data
+    consistency at database level.
+    """
+    value = CharField(max_length=30, editable=False, null=False, blank=False, unique=True, primary_key=True)
+
+    class Meta:
+        abstract = True
+
+    def __str__(self):
+        return self.value
+
+
+class RoleChoice(AbstractChoice):
+    """Defines the model and predefined list of possible RoleChoice's for WorkIORoles.
+    The items in the Choises class below are automagically populated into the database via a data migration."""
+    class Choices(Enum):
         CORRELATOR = "correlator"
         BEAMFORMER = "beamformer"
         INSPECTION_PLOTS = "inspection plots"
@@ -63,59 +93,40 @@ class RoleChoice(Enum):
         INPUT_OUTPUT = "input, output"
 
 
-class DatatypeChoice(Enum):
+class DatatypeChoice(AbstractChoice):
+    """Defines the model and predefined list of possible DatatypeChoice's for WorkIORoles.
+    The items in the Choises class below are automagically populated into the database via a data migration."""
+    class Choices(Enum):
         VISIBILITIES = "visibilities"
         TIME_SERIES = "time series"
         INSTRUMENT_MODEL = "instrument model"
         IMAGE = "image"
         QUALITY = "quality"
 
-
-class DataformatChoice(Enum):
+class DataformatChoice(AbstractChoice):
+    """Defines the model and predefined list of possible DataformatChoice's for WorkRequestRelationBlueprint and WorkRequestRelation.
+    The items in the Choises class below are automagically populated into the database via a data migration."""
+    class Choices(Enum):
         MEASUREMENTSET = "MeasurementSet"
         HDF5 = "HDF5"
 
 
-class CopyReasonChoice(Enum):
+class CopyReasonChoice(AbstractChoice):
+    """Defines the model and predefined list of possible CopyReasonChoice's for RunBlueprint and WorkRequestBlueprint.
+    The items in the Choises class below are automagically populated into the database via a data migration."""
+    class Choices(Enum):
         TEMPLATE = "TEMPLATE"
         REPEATED = "REPEATED"
 
-# todo: fix this!
-#class EnumField(Field):  # todo: Test if CharField works better e.g. for forms
-#    """
-#    Django does not support creating db enums, so we got to do that ourselves here
-#    """
-#    def __init__(self, *args, **kwargs):
-#        super(EnumField, self).__init__(*args, **kwargs)
-#
-#    def db_type(self, connection):
-#        return "enum(%s)" % ','.join("'%s'" % key for (key, _) in self.choices)
-# # todo: unfortunately this won't work for Postgres since a CREATE TYPE is required first...
-
-
 # concrete models
 
 class WorkIORoles(Model):
-    # todo: Choices do not translate to a Postgres ENUM type currently, but are only enforced on the Django level.
-    # todo: If we are not happy with this as is, I see two options:
-    # todo: 1. Wait to see if they implement that EnumField in the meantime (https://code.djangoproject.com/ticket/24342)
-    # todo: 2. Build this ourselves. I tried to achieve that with the EnumField above, but it won't be that easy, I guess
-    role = CharField(
-        max_length=30,
-        choices=[(item.name, item.value) for item in RoleChoice]
-    )
-    datatype = CharField(
-        max_length=30,
-        choices=[(item.name, item.value) for item in DatatypeChoice]
-    )
-    dataformat = ArrayField(CharField(
-        max_length=30,
-        choices=[(item.name, item.value) for item in DataformatChoice]
-    ), size=8, blank=True)
+    role = ForeignKey('RoleChoice', null=False, on_delete=CASCADE)
+    datatype = ForeignKey('DatatypeChoice', null=False, on_delete=CASCADE)
+    dataformat = ForeignKey('DataformatChoice', null=False, on_delete=CASCADE)
     outputs = ForeignKey("WorkRequestTemplate", related_name='role_output', on_delete=CASCADE, null=True)
     inputs = ForeignKey("WorkRequestTemplate", related_name='role_input', on_delete=CASCADE, null=True)
 
-
 #
 # Templates
 #
@@ -178,9 +189,7 @@ class RunSet(UserDefinedCommon):
 class RunBlueprint(UserDefinedCommon):
     requirements_doc = JSONField()
     copies = ForeignKey('RunBlueprint', related_name="copied_from", on_delete=SET_NULL, null=True)
-    copy_reason = CharField(max_length=30,
-                            choices=
-                            [(item.name, item.value) for item in CopyReasonChoice])
+    copy_reason = ForeignKey('CopyReasonChoice', null=False, on_delete=CASCADE)
     generator_param = CharField(max_length=200)  # todo: check if the field size is good enough
     run_set = ForeignKey(RunSet, related_name='run_blueprints', on_delete=CASCADE)
     generator_source = ForeignKey(RunSet, on_delete=SET_NULL, null=True)
@@ -196,9 +205,7 @@ class Run(SystemDefinedCommon):
 class WorkRequestBlueprint(UserDefinedCommon):
     requirements_doc = JSONField()
     copies = ForeignKey('WorkRequestBlueprint', related_name="copied_from", on_delete=SET_NULL, null=True)
-    copy_reason = CharField(max_length=30,
-                            choices=
-                            [(item.name, item.value) for item in CopyReasonChoice])
+    copy_reason = ForeignKey('CopyReasonChoice', null=False, on_delete=CASCADE)
 
 
 class WorkRequest(SystemDefinedCommon):
@@ -210,8 +217,7 @@ class WorkRequest(SystemDefinedCommon):
 
 class WorkRequestRelationBlueprint(UserDefinedCommon):
     selection_doc = JSONField()
-    dataformat = CharField(max_length=30,
-                           choices=[(item.name, item.value) for item in DataformatChoice])
+    dataformat = ForeignKey('DataformatChoice', null=False, on_delete=CASCADE)
     producer = ForeignKey(WorkRequestBlueprint, related_name='work_request_relation_blueprint_producer', on_delete=CASCADE)
     consumer = ForeignKey(WorkRequestBlueprint, related_name='work_request_relation_blueprint_consumer', on_delete=CASCADE)
     input_role = ForeignKey(WorkIORoles, related_name='work_request_relation_blueprint_input_role', on_delete=CASCADE)
@@ -221,8 +227,7 @@ class WorkRequestRelationBlueprint(UserDefinedCommon):
 
 class WorkRequestRelation(SystemDefinedCommon):
     selection_doc = JSONField()
-    dataformat = CharField(max_length=30,
-                           choices=[(item.name, item.value) for item in DataformatChoice])
+    dataformat = ForeignKey('DataformatChoice', null=False, on_delete=CASCADE)
     producer = ForeignKey(WorkRequest, related_name='work_request_relation_producer', on_delete=CASCADE)
     consumer = ForeignKey(WorkRequest, related_name='work_request_relation_consumer', on_delete=CASCADE)
     input_role = ForeignKey(WorkIORoles, related_name='work_request_relation_input_role', on_delete=CASCADE)

SAS/LSMR/src/lsmr/lsmrapp/populate.py

+
+"""
+This module 'populate' defines methods to populate the database with predefined ('static') data,
+according to the proposed Django way: https://docs.djangoproject.com/en/2.0/topics/migrations/#data-migrations
+
+import this module in your empty migration step file, and add the following migration:
+
+from ..populate import *
+
+class Migration(migrations.Migration):
+
+    dependencies = [ <the dependency is automatically inserted here> ]
+
+    operations = [ migrations.RunPython(populate_choices) ]
+
+"""
+
+from .models.specification import RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice
+
+def populate_choices(apps, schema_editor):
+    '''
+    populate each 'choice' table in the database with the 'static' list of 'choice'.Choices for
+    each 'choice'type in RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice
+    :return: None
+    '''
+    for choice_class in [RoleChoice, DatatypeChoice, DataformatChoice, CopyReasonChoice]:
+        choice_class.objects.bulk_create([choice_class(value=x.value) for x in choice_class.Choices])
+

SAS/LSMR/src/lsmr/lsmrapp/serializers/specification.py

@@ -36,6 +36,30 @@ class WorkRelationSelectionTemplateSerializer(serializers.HyperlinkedModelSerial
         fields = '__all__'
 
 
+class RoleChoiceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.RoleChoice
+        fields = '__all__'
+
+
+class DatatypeChoiceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.DatatypeChoice
+        fields = '__all__'
+
+
+class DataformatChoiceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.DataformatChoice
+        fields = '__all__'
+
+
+class CopyReasonChoiceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.CopyReasonChoice
+        fields = '__all__'
+
+
 class WorkIORolesSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = models.WorkIORoles

SAS/LSMR/src/lsmr/lsmrapp/viewsets/specification.py

@@ -31,6 +31,26 @@ class WorkRelationSelectionTemplateViewSet(viewsets.ModelViewSet):
     serializer_class = serializers.WorkRelationSelectionTemplateSerializer
 
 
+class RoleChoiceViewSet(viewsets.ModelViewSet):
+    queryset = models.RoleChoice.objects.all()
+    serializer_class = serializers.RoleChoiceSerializer
+
+
+class DatatypeChoiceViewSet(viewsets.ModelViewSet):
+    queryset = models.DatatypeChoice.objects.all()
+    serializer_class = serializers.DatatypeChoiceSerializer
+
+
+class DataformatChoiceViewSet(viewsets.ModelViewSet):
+    queryset = models.DataformatChoice.objects.all()
+    serializer_class = serializers.DataformatChoiceSerializer
+
+
+class CopyReasonChoiceViewSet(viewsets.ModelViewSet):
+    queryset = models.CopyReasonChoice.objects.all()
+    serializer_class = serializers.CopyReasonChoiceSerializer
+
+
 class WorkIORolesViewSet(viewsets.ModelViewSet):
     queryset = models.WorkIORoles.objects.all()
     serializer_class = serializers.WorkIORolesSerializer

SAS/LSMR/src/lsmr/settings.py

@@ -31,7 +31,7 @@ ALLOWED_HOSTS = []
 # Application definition
 
 INSTALLED_APPS = [
-    'lofar.sas.lsmr.lsmrapp',
+    'lsmr.lsmrapp',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

SAS/LSMR/src/lsmr/urls.py

@@ -43,6 +43,10 @@ router.register(r'generator_template', viewsets.GeneratorTemplateViewSet)
 router.register(r'run_template', viewsets.RunTemplateViewSet)
 router.register(r'work_request_template', viewsets.WorkRequestTemplateViewSet)
 router.register(r'work_relation_selection_template', viewsets.WorkRelationSelectionTemplateViewSet)
+router.register(r'role_choice', viewsets.RoleChoiceViewSet)
+router.register(r'datatype_choice', viewsets.DatatypeChoiceViewSet)
+router.register(r'dataformat_choice', viewsets.DataformatChoiceViewSet)
+router.register(r'copy_reason_choice', viewsets.CopyReasonChoiceViewSet)
 router.register(r'work_io_roles', viewsets.WorkIORolesViewSet)
 
 # instances