From 7672d147b859af81e889e3ea5c44a0c10c2c8dec Mon Sep 17 00:00:00 2001
From: jkuensem <jkuensem@physik.uni-bielefeld.de>
Date: Thu, 18 Jun 2020 10:25:05 +0200
Subject: [PATCH] TMSS-187: add settings functionality and a flag that can be
used to prevent scheduling of observations through TMSS
---
SAS/TMSS/client/bin/CMakeLists.txt | 2 ++
SAS/TMSS/client/bin/tmss_get_setting | 23 +++++++++++++++
SAS/TMSS/client/bin/tmss_set_setting | 23 +++++++++++++++
SAS/TMSS/client/lib/mains.py | 28 +++++++++++++++++++
SAS/TMSS/client/lib/tmss_http_rest_client.py | 24 ++++++++++++++++
SAS/TMSS/src/remakemigrations.py | 3 +-
.../tmss/tmssapp/migrations/0001_initial.py | 28 ++++++++++++++++++-
.../tmss/tmssapp/migrations/0002_populate.py | 3 +-
.../src/tmss/tmssapp/models/specification.py | 13 ++++++++-
SAS/TMSS/src/tmss/tmssapp/populate.py | 6 +++-
.../tmss/tmssapp/serializers/specification.py | 12 ++++++++
SAS/TMSS/src/tmss/tmssapp/subtasks.py | 7 +++++
.../tmss/tmssapp/viewsets/specification.py | 10 +++++++
SAS/TMSS/src/tmss/urls.py | 2 ++
SAS/TMSS/test/t_subtasks.py | 12 +++++++-
15 files changed, 190 insertions(+), 6 deletions(-)
create mode 100755 SAS/TMSS/client/bin/tmss_get_setting
create mode 100755 SAS/TMSS/client/bin/tmss_set_setting
diff --git a/SAS/TMSS/client/bin/CMakeLists.txt b/SAS/TMSS/client/bin/CMakeLists.txt
index a7142728b75..d2bd6170e88 100644
--- a/SAS/TMSS/client/bin/CMakeLists.txt
+++ b/SAS/TMSS/client/bin/CMakeLists.txt
@@ -5,3 +5,5 @@ lofar_add_bin_scripts(tmss_get_subtasks)
lofar_add_bin_scripts(tmss_get_subtask_predecessors)
lofar_add_bin_scripts(tmss_get_subtask_successors)
lofar_add_bin_scripts(tmss_schedule_subtask)
+lofar_add_bin_scripts(tmss_get_setting)
+lofar_add_bin_scripts(tmss_set_setting)
diff --git a/SAS/TMSS/client/bin/tmss_get_setting b/SAS/TMSS/client/bin/tmss_get_setting
new file mode 100755
index 00000000000..34d79f641c2
--- /dev/null
+++ b/SAS/TMSS/client/bin/tmss_get_setting
@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+from lofar.sas.tmss.client.mains import main_get_setting
+
+if __name__ == "__main__":
+ main_get_setting()
diff --git a/SAS/TMSS/client/bin/tmss_set_setting b/SAS/TMSS/client/bin/tmss_set_setting
new file mode 100755
index 00000000000..e54dd118f9a
--- /dev/null
+++ b/SAS/TMSS/client/bin/tmss_set_setting
@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+from lofar.sas.tmss.client.mains import main_set_setting
+
+if __name__ == "__main__":
+ main_set_setting()
diff --git a/SAS/TMSS/client/lib/mains.py b/SAS/TMSS/client/lib/mains.py
index f645b9643a1..ca614b9f218 100644
--- a/SAS/TMSS/client/lib/mains.py
+++ b/SAS/TMSS/client/lib/mains.py
@@ -125,3 +125,31 @@ def main_schedule_subtask():
except Exception as e:
print(e)
exit(1)
+
+
+def main_get_setting():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("setting_name", type=str, help="The name of the TMSS setting to get")
+ args = parser.parse_args()
+
+ try:
+ with TMSSsession.create_from_dbcreds_for_ldap() as session:
+ pprint(session.get_setting(args.setting_name))
+ except Exception as e:
+ print(e)
+ exit(1)
+
+
+def main_set_setting():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("setting_name", type=str, help="The name of the TMSS setting to set")
+ parser.add_argument("setting_value", type=lambda s: s.lower() in ['true', 'True', '1'], # argparse is noot very good at speaking bool...
+ help="The value to set for the TMSS setting")
+ args = parser.parse_args()
+
+ try:
+ with TMSSsession.create_from_dbcreds_for_ldap() as session:
+ pprint(session.set_setting(args.setting_name, args.setting_value))
+ except Exception as e:
+ print(e)
+ exit(1)
diff --git a/SAS/TMSS/client/lib/tmss_http_rest_client.py b/SAS/TMSS/client/lib/tmss_http_rest_client.py
index 867a29c05bc..1f426c0b52a 100644
--- a/SAS/TMSS/client/lib/tmss_http_rest_client.py
+++ b/SAS/TMSS/client/lib/tmss_http_rest_client.py
@@ -216,3 +216,27 @@ class TMSSsession(object):
returns the scheduled subtask upon success, or raises."""
return self.get_path_as_json_object('subtask/%s/schedule' % subtask_id)
+ def get_setting(self, setting_name: str) -> {}:
+ """get the value of a TMSS setting.
+ returns the setting value upon success, or raises."""
+ response = self.session.get(url='%s/setting/%s/' % (self.base_url, setting_name),
+ params={'format': 'json'})
+
+ if response.status_code >= 200 and response.status_code < 300:
+ return json.loads(response.content.decode('utf-8'))['value']
+
+ content = response.content.decode('utf-8')
+ raise Exception("Could not get setting with name %s.\nResponse: %s" % (setting_name, content))
+
+ def set_setting(self, setting_name: str, setting_value: bool) -> {}:
+ """set a value for a TMSS setting.
+ returns the setting value upon success, or raises."""
+ response = self.session.patch(url='%s/setting/%s/' % (self.base_url, setting_name),
+ json={'value': setting_value})
+
+ if response.status_code >= 200 and response.status_code < 300:
+ return json.loads(response.content.decode('utf-8'))['value']
+
+ content = response.content.decode('utf-8')
+ raise Exception("Could not set status with url %s - %s %s - %s" % (response.request.url, response.status_code, responses.get(response.status_code), content))
+
diff --git a/SAS/TMSS/src/remakemigrations.py b/SAS/TMSS/src/remakemigrations.py
index 10c7ac1b295..03fdec4cec3 100755
--- a/SAS/TMSS/src/remakemigrations.py
+++ b/SAS/TMSS/src/remakemigrations.py
@@ -77,7 +77,8 @@ class Migration(migrations.Migration):
operations = [ migrations.RunSQL('ALTER SEQUENCE tmssapp_SubTask_id_seq RESTART WITH 2000000;'),
migrations.RunPython(populate_choices),
migrations.RunPython(populate_misc),
- migrations.RunPython(populate_lofar_json_schemas) ]
+ migrations.RunPython(populate_lofar_json_schemas),
+ migrations.RunPython(populate_settings)]
"""
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index af286d3fe40..a20fb8d0837 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.6 on 2020-06-09 17:31
+# Generated by Django 3.0.6 on 2020-06-17 15:34
from django.conf import settings
import django.contrib.postgres.fields
@@ -284,6 +284,15 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='Flag',
+ fields=[
+ ('value', models.CharField(max_length=128, primary_key=True, serialize=False, unique=True)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='GeneratorTemplate',
fields=[
@@ -628,6 +637,19 @@ class Migration(migrations.Migration):
('validation_code_js', models.CharField(help_text='JavaScript code for additional (complex) validation.', max_length=128)),
],
),
+ migrations.CreateModel(
+ name='Setting',
+ fields=[
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, primary_key=True, serialize=False, to='tmssapp.Flag', unique=True)),
+ ('value', models.BooleanField()),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.AddConstraint(
model_name='tasktemplate',
constraint=models.UniqueConstraint(fields=('name', 'version'), name='TaskTemplate_unique_name_version'),
@@ -1044,6 +1066,10 @@ class Migration(migrations.Migration):
model_name='subtask',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_sub_tags_d2fc43_gin'),
),
+ migrations.AddIndex(
+ model_name='setting',
+ index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_set_tags_41a1ba_gin'),
+ ),
migrations.AddIndex(
model_name='defaulttasktemplate',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_c88200_gin'),
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
index e33461a9295..a4fcecd5e9c 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0002_populate.py
@@ -18,4 +18,5 @@ class Migration(migrations.Migration):
operations = [ migrations.RunSQL('ALTER SEQUENCE tmssapp_SubTask_id_seq RESTART WITH 2000000;'),
migrations.RunPython(populate_choices),
migrations.RunPython(populate_misc),
- migrations.RunPython(populate_lofar_json_schemas) ]
+ migrations.RunPython(populate_lofar_json_schemas),
+ migrations.RunPython(populate_settings)]
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
index 2956c3e09c5..2d93d37047d 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
@@ -131,8 +131,20 @@ class CopyReason(AbstractChoice):
REPEATED = "repeated"
+class Flag(AbstractChoice):
+ """Defines the model and predefined list of possible Flags to be used in Setting.
+ The items in the Choises class below are automagically populated into the database via a data migration."""
+ class Choices(Enum):
+ AUTOSCHEDULE = "allow_scheduling_observations"
+
+
# concrete models
+class Setting(BasicCommon):
+ name = ForeignKey('Flag', null=False, on_delete=PROTECT, unique=True, primary_key=True)
+ value = BooleanField(null=False)
+
+
class TaskConnector(BasicCommon):
role = ForeignKey('Role', null=False, on_delete=PROTECT)
datatype = ForeignKey('Datatype', null=False, on_delete=PROTECT)
@@ -381,4 +393,3 @@ class TaskRelationBlueprint(BasicCommon):
validate_json_against_schema(self.selection_doc, self.selection_template.schema)
super().save(force_insert, force_update, using, update_fields)
-
diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index 7a16ac32591..8d0daea2ee2 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -32,9 +32,12 @@ def populate_choices(apps, schema_editor):
:return: None
'''
for choice_class in [Role, Datatype, Dataformat, CopyReason,
- SubtaskState, SubtaskType, StationType, Algorithm, ScheduleMethod]:
+ SubtaskState, SubtaskType, StationType, Algorithm, ScheduleMethod, Flag]:
choice_class.objects.bulk_create([choice_class(value=x.value) for x in choice_class.Choices])
+def populate_settings(apps, schema_editor):
+ Setting.objects.create(name=Flag.objects.get(value='allow_scheduling_observations'), value=True)
+
def populate_lofar_json_schemas(apps, schema_editor):
_populate_dataproduct_specifications_templates()
@@ -98,6 +101,7 @@ def populate_resources(apps, schema_editor):
ResourceType.objects.create(name="cep_storage", description="Amount of storage at CEP processing cluster", resource_unit=ru_bytes)
ResourceType.objects.create(name="cep_processing_hours", description="Number of processing hours for CEP processing cluster", resource_unit=ru_hours)
+
def populate_misc(apps, schema_editor):
cluster = Cluster.objects.create(name="CEP4", location="CIT")
fs = Filesystem.objects.create(name="LustreFS", cluster=cluster, capacity=3.6e15)
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
index a745c8e18e3..afbc1b2ed5e 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
@@ -148,6 +148,18 @@ class ResourceTypeSerializer(RelationalHyperlinkedModelSerializer):
extra_fields = ['name']
+class FlagSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.Flag
+ fields = '__all__'
+
+
+class SettingSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.Setting
+ fields = '__all__'
+
+
class SchedulingSetSerializer(RelationalHyperlinkedModelSerializer):
# Create a JSON editor form to replace the simple text field based on the schema in the template that this
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 781a4512802..80f76dd09e6 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -417,6 +417,13 @@ def schedule_observation_subtask(observation_subtask: Subtask):
observation_subtask.specifications_template.type,
SubtaskType.Choices.OBSERVATION.value))
+ # check if settings allow scheduling observations
+ # (not sure if this should be in check_prerequities_for_scheduling() instead....?)
+ setting = Setting.objects.get(name='allow_scheduling_observations')
+ if not setting.value:
+ raise SubtaskSchedulingException("Cannot schedule subtask id=%d because setting %s=%s does not allow that." %
+ (observation_subtask.pk, setting.name, setting.value))
+
# step 1: set state to SCHEDULING
observation_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULING.value)
observation_subtask.save()
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index 4516d434acb..b8d0b66260b 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -162,6 +162,16 @@ class SchedulingSetViewSet(LOFARViewSet):
serializer_class = serializers.SchedulingSetSerializer
+class FlagViewSet(LOFARViewSet):
+ queryset = models.Flag.objects.all()
+ serializer_class = serializers.FlagSerializer
+
+
+class SettingViewSet(LOFARViewSet):
+ queryset = models.Setting.objects.all()
+ serializer_class = serializers.SettingSerializer
+
+
class SchedulingUnitDraftViewSet(LOFARViewSet):
queryset = models.SchedulingUnitDraft.objects.all()
serializer_class = serializers.SchedulingUnitDraftSerializer
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index e6c2a1d6c0c..5edc1429295 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -71,6 +71,7 @@ router.register(r'role', viewsets.RoleViewSet)
router.register(r'datatype', viewsets.DatatypeViewSet)
router.register(r'dataformat', viewsets.DataformatViewSet)
router.register(r'copy_reason', viewsets.CopyReasonViewSet)
+router.register(r'flag', viewsets.FlagViewSet)
# templates
router.register(r'generator_template', viewsets.GeneratorTemplateViewSet)
@@ -89,6 +90,7 @@ router.register(r'project', viewsets.ProjectViewSet)
router.register(r'resource_unit', viewsets.ResourceUnitViewSet)
router.register(r'resource_type', viewsets.ResourceTypeViewSet)
router.register(r'project_quota', viewsets.ProjectQuotaViewSet)
+router.register(r'setting', viewsets.SettingViewSet)
router.register(r'scheduling_set', viewsets.SchedulingSetViewSet)
router.register(r'scheduling_unit_draft', viewsets.SchedulingUnitDraftViewSet)
diff --git a/SAS/TMSS/test/t_subtasks.py b/SAS/TMSS/test/t_subtasks.py
index 4988e783d4e..86df1080123 100755
--- a/SAS/TMSS/test/t_subtasks.py
+++ b/SAS/TMSS/test/t_subtasks.py
@@ -42,7 +42,6 @@ from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.subtasks import *
-
class SubtaskInputSelectionFilteringTest(unittest.TestCase):
# todo: merge in tests from TMSS-207 and deduplicate staticmethods
@@ -134,6 +133,17 @@ class SubtaskInputSelectionFilteringTest(unittest.TestCase):
self.assertEqual(set(pipe_in2.dataproducts.all()), {dp2_2})
+class SettingTest(unittest.TestCase):
+
+ def test_schedule_observation_subtask_raises_when_flag_is_false(self):
+ setting = Setting.objects.get(name='allow_scheduling_observations')
+ setting.value = False
+ setting.save()
+ obs_st = SubtaskInputSelectionFilteringTest.create_subtask_object('observation', 'defined')
+
+ with self.assertRaises(SubtaskSchedulingException):
+ schedule_observation_subtask(obs_st)
+
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
unittest.main()
--
GitLab