diff --git a/LCS/PyCommon/CMakeLists.txt b/LCS/PyCommon/CMakeLists.txt
index 044b7da9bfbaaeb4b0364febcb55f347e08e386a..f5040f3b74119e61788d2cf3793230496216397b 100644
--- a/LCS/PyCommon/CMakeLists.txt
+++ b/LCS/PyCommon/CMakeLists.txt
@@ -34,6 +34,7 @@ set(_py_files
json_utils.py
locking.py
test_utils.py
+ typing.py
ring_coordinates.py)
python_install(${_py_files} DESTINATION lofar/common)
diff --git a/LCS/PyCommon/test/CMakeLists.txt b/LCS/PyCommon/test/CMakeLists.txt
index bf1bfce981f17ca4553ce3fba4329c4d350298d9..624f130d2336df98ce720564ea572792c39bc894 100644
--- a/LCS/PyCommon/test/CMakeLists.txt
+++ b/LCS/PyCommon/test/CMakeLists.txt
@@ -28,6 +28,7 @@ IF(BUILD_TESTING)
lofar_add_test(t_util)
lofar_add_test(t_test_utils)
lofar_add_test(t_cep4_utils)
+ lofar_add_test(t_typing)
IF(PYTHON_JSONSCHEMA)
lofar_add_test(t_json_utils)
@@ -37,4 +38,4 @@ IF(BUILD_TESTING)
lofar_add_test(t_postgres)
ENDIF()
-ENDIF()
\ No newline at end of file
+ENDIF()
diff --git a/LCS/PyCommon/test/t_typing.py b/LCS/PyCommon/test/t_typing.py
new file mode 100755
index 0000000000000000000000000000000000000000..55eb4fc32e433106d39371da9ce59ade2b227060
--- /dev/null
+++ b/LCS/PyCommon/test/t_typing.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(format='%(asctime)s %(process)s %(threadName)s %(levelname)s %(message)s', level=logging.DEBUG)
+
+from lofar.common.typing import check_type_hints
+
+import typing
+import unittest
+
+class TestCheckTypeHints(unittest.TestCase):
+ def test_no_argument(self):
+ """ Elementary test for the type hint of the return type. """
+
+ @check_type_hints
+ def myfunc() -> str:
+ return "ok"
+
+ self.assertEqual("ok", myfunc())
+
+ def test_one_argument(self):
+ """ Elementary test for one argument with a type hint. """
+
+ @check_type_hints
+ def myfunc(i: int) -> str:
+ return str(i)
+
+ self.assertEqual("1", myfunc(1))
+
+ with self.assertRaises(TypeError):
+ myfunc("1")
+
+ with self.assertRaises(TypeError):
+ myfunc(i="1")
+
+ def test_argument_default(self):
+ """ Check whether argument defaults still function correctly. """
+
+ @check_type_hints
+ def myfunc(i: int = 1) -> str:
+ return str(i)
+
+ self.assertEqual("1", myfunc())
+
+ def test_multiple_arguments(self):
+ """ Check whether multiple arguments are handled correctly with various calling conventions. """
+
+ @check_type_hints
+ def myfunc(i: int, j:int) -> str:
+ return "%d %d" % (i,j)
+
+ self.assertEqual("1 2", myfunc(1,2))
+ self.assertEqual("1 2", myfunc(1,j=2))
+ self.assertEqual("1 2", myfunc(i=1,j=2))
+
+ with self.assertRaises(TypeError):
+ myfunc("1",2)
+
+ with self.assertRaises(TypeError):
+ myfunc(1,"2")
+
+ with self.assertRaises(TypeError):
+ myfunc(1, j="2")
+
+ with self.assertRaises(TypeError):
+ myfunc(i="1", j=2)
+
+ def test_wrong_return_value(self):
+ """ Check whether return values are validated. """
+
+ @check_type_hints
+ def myfunc(i: int) -> str:
+ return i
+
+ with self.assertRaises(TypeError):
+ myfunc(1)
+
+ def test_inheritance(self):
+ """ Provided values can also be subclasses of the types provided in the hints. """
+
+ @check_type_hints
+ def myfunc(i: int) -> int:
+ return i
+
+ class DerivedInt(int):
+ pass
+
+ myfunc(DerivedInt(1))
+
+ def test_no_hints(self):
+ """ Functions without any hints should always work. """
+
+ @check_type_hints
+ def myfunc(i):
+ return str(i)
+
+ self.assertEqual("1", myfunc(1))
+ self.assertEqual("1", myfunc("1"))
+
+ def test_some_hints(self):
+ """ Not all parameters are necessarily annotated. """
+
+ @check_type_hints
+ def myfunc(i, j: int):
+ return str(i)
+
+ self.assertEqual("1", myfunc(1, 2))
+ self.assertEqual("1", myfunc("1", 2))
+
+ with self.assertRaises(TypeError):
+ self.assertEqual("1", myfunc("1", "2"))
+
+ def test_union_hint(self):
+ """ Python allows supplying multiple types as a list, any of which is valid. """
+
+ @check_type_hints
+ def myfunc(i: [int, str]):
+ return str(i)
+
+ self.assertEqual("1", myfunc(1))
+ self.assertEqual("1", myfunc("1"))
+
+ with self.assertRaises(TypeError):
+ self.assertEqual("1", myfunc(1.0))
+
+ def test_args_kwargs(self):
+ """ Check whether args & kwargs don't break. """
+
+ @check_type_hints
+ def myfunc(*args, **kwargs):
+ return str(kwargs["i"])
+
+ self.assertEqual("1", myfunc(i=1))
+ self.assertEqual("1", myfunc(i="1"))
+
+
+ def test_asterics(self):
+ """ Check whether forced named arguments don't break. """
+
+ @check_type_hints
+ def myfunc(*, i: int):
+ return str(i)
+
+ self.assertEqual("1", myfunc(i=1))
+
+ with self.assertRaises(TypeError):
+ self.assertEqual("1", myfunc(i="1"))
+
+ def test_none(self):
+ """ Check whether None as an argument functions correctly. """
+
+ @check_type_hints
+ def myfunc(i: int) -> str:
+ return str(i)
+
+ with self.assertRaises(TypeError):
+ myfunc(None)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/LCS/PyCommon/test/t_typing.run b/LCS/PyCommon/test/t_typing.run
new file mode 100755
index 0000000000000000000000000000000000000000..6bc23fadc736235c1143d3317d88307ffeac0f67
--- /dev/null
+++ b/LCS/PyCommon/test/t_typing.run
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source python-coverage.sh
+python_coverage_test "*json_utils*" t_typing.py
+
diff --git a/LCS/PyCommon/test/t_typing.sh b/LCS/PyCommon/test/t_typing.sh
new file mode 100755
index 0000000000000000000000000000000000000000..d788f5a03bee1f34f0c524afadfee796de8e081a
--- /dev/null
+++ b/LCS/PyCommon/test/t_typing.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+./runctest.sh t_typing
diff --git a/LCS/PyCommon/typing.py b/LCS/PyCommon/typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd154ec09a2352afe744e5605a460a989b6413bc
--- /dev/null
+++ b/LCS/PyCommon/typing.py
@@ -0,0 +1,67 @@
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+from functools import wraps
+import inspect
+
+def check_type_hints(func):
+ """ Decorator that verifies the type hints of the decorated function.
+
+ Raises a TypeError if the type is not met, that is, the parameters and/or return value
+ that have a type hint are not given values that are of that type, or a subclass.
+
+ Example usage:
+
+ @check_type_hints
+ def myfunc(i: int, j) -> str:
+ return "%d %s" % (i,j)
+
+ myfunc(1, 2) # ok, type of i matches type hint
+ myfunc(1, "2") # ok, type of j is not checked, as it has no type hint
+ myfunc("1", 2) # throws TypeError, type i does not match type hint
+ """
+
+ def check_type(obj, cls):
+ if isinstance(cls, list):
+ return any((isinstance(obj, c) for c in cls))
+
+ return isinstance(obj, cls)
+
+ @wraps(func)
+ def decorator(*args, **kwargs):
+ argspec = inspect.getfullargspec(func)
+ hints = argspec.annotations
+
+ for i, (arg, argname) in enumerate(zip(args, argspec.args)):
+ if argname in hints:
+ argtype = hints[argname]
+ if not check_type(arg, argtype):
+ raise TypeError("Positional parameter %d (named %s) must have type %s (has type %s)" % (i, argname, argtype, type(arg)))
+
+ for argname, argtype in hints.items():
+ if argname in kwargs:
+ if not check_type(kwargs[argname], argtype):
+ raise TypeError("Parameter %s must have type %s (has type %s)" % (argname, argtype, type(kwargs[argname])))
+
+ return_value = func(*args, **kwargs)
+ if 'return' in hints:
+ if not check_type(return_value, hints['return']):
+ raise TypeError("Return value must have type %s (has type %s)" % (hints['return'], type(return_value)))
+
+ return return_value
+
+ return decorator
diff --git a/SAS/ResourceAssignment/TaskPrescheduler/lib/cobaltblocksize.py b/SAS/ResourceAssignment/TaskPrescheduler/lib/cobaltblocksize.py
index daa5266fc31381ea20a84d6200d696383b0608e9..ac14727d9a2c2645de608bf7454bd9bf60e30175 100644
--- a/SAS/ResourceAssignment/TaskPrescheduler/lib/cobaltblocksize.py
+++ b/SAS/ResourceAssignment/TaskPrescheduler/lib/cobaltblocksize.py
@@ -47,7 +47,7 @@ class BlockConstraints(object):
""" Provide the constraints for the block size, as derived
from the correlator and beamformer settings. """
- def __init__(self, correlatorSettings=None, coherentStokesSettings=None, incoherentStokesSettings=None, clockMHz=200):
+ def __init__(self, correlatorSettings=None, coherentStokesSettings=[], incoherentStokesSettings=[], clockMHz=200):
self.correlator = correlatorSettings
self.coherentStokes = coherentStokesSettings
self.incoherentStokes = incoherentStokesSettings
@@ -107,28 +107,28 @@ class BlockConstraints(object):
# Correlator.cu (minimum of 16 samples per channel)
factor = lcm(factor, CORRELATOR_BLOCKSIZE * self.correlator.nrChannelsPerSubband * self.nrSubblocks())
- if self.coherentStokes:
+ for coherentStokes in self.coherentStokes:
# DelayAndBandPass.cu
factor = lcm(factor, BEAMFORMER_DELAYCOMPENSATION_BLOCKSIZE * BEAMFORMER_NR_DELAYCOMPENSATION_CHANNELS)
# FIR_Filter.cu
- factor = lcm(factor, NR_PPF_TAPS * self.coherentStokes.nrChannelsPerSubband)
+ factor = lcm(factor, NR_PPF_TAPS * coherentStokes.nrChannelsPerSubband)
# CoherentStokesKernel.cc
- factor = lcm(factor, MAX_THREADS_PER_BLOCK * self.coherentStokes.timeIntegrationFactor)
+ factor = lcm(factor, MAX_THREADS_PER_BLOCK * coherentStokes.timeIntegrationFactor)
#CoherentStokes.cu (integration should fit)
- factor = lcm(factor, 1024 * self.coherentStokes.timeIntegrationFactor * self.coherentStokes.nrChannelsPerSubband)
+ factor = lcm(factor, 1024 * coherentStokes.timeIntegrationFactor * coherentStokes.nrChannelsPerSubband)
- if self.incoherentStokes:
+ for incoherentStokes in self.incoherentStokes:
# DelayAndBandPass.cu
factor = lcm(factor, BEAMFORMER_DELAYCOMPENSATION_BLOCKSIZE * BEAMFORMER_NR_DELAYCOMPENSATION_CHANNELS)
# FIR_Filter.cu
- factor = lcm(factor, NR_PPF_TAPS * self.incoherentStokes.nrChannelsPerSubband)
+ factor = lcm(factor, NR_PPF_TAPS * incoherentStokes.nrChannelsPerSubband)
# IncoherentStokes.cu (integration should fit)
- factor = lcm(factor, 1024 * self.incoherentStokes.timeIntegrationFactor * self.incoherentStokes.nrChannelsPerSubband)
+ factor = lcm(factor, 1024 * incoherentStokes.timeIntegrationFactor * incoherentStokes.nrChannelsPerSubband)
return factor
diff --git a/SAS/ResourceAssignment/TaskPrescheduler/taskprescheduler.py b/SAS/ResourceAssignment/TaskPrescheduler/taskprescheduler.py
index e3cf4e6ccc1730279de43c26cb2617b56709e09a..5cf07d6b85ab9866355c1a352df47d1a3697e1ab 100644
--- a/SAS/ResourceAssignment/TaskPrescheduler/taskprescheduler.py
+++ b/SAS/ResourceAssignment/TaskPrescheduler/taskprescheduler.py
@@ -69,7 +69,7 @@ def calculateCobaltSettings(spec):
incoherent = None
clock = parset["Observation.sampleClock"]
- constraints = BlockConstraints(corr, coherent, incoherent, clock)
+ constraints = BlockConstraints(corr, [coherent], [incoherent], clock)
calculator = BlockSize(constraints)
return {'nrSubblocks': calculator.nrSubblocks, 'blockSize': calculator.blockSize,
diff --git a/SAS/ResourceAssignment/TaskPrescheduler/test/t_cobaltblocksize.py b/SAS/ResourceAssignment/TaskPrescheduler/test/t_cobaltblocksize.py
index fe7acef4cf0ab8d1c2fb3baa6938b2eeacfa7e1b..8eaec011e3fd642723377b9ace171db8a687dfd1 100644
--- a/SAS/ResourceAssignment/TaskPrescheduler/test/t_cobaltblocksize.py
+++ b/SAS/ResourceAssignment/TaskPrescheduler/test/t_cobaltblocksize.py
@@ -56,7 +56,7 @@ class TestBlockConstraints(unittest.TestCase):
coh.nrChannelsPerSubband = 16
coh.timeIntegrationFactor = 4
- c = BlockConstraints(coherentStokesSettings=coh)
+ c = BlockConstraints(coherentStokesSettings=[coh])
self.assertEqual(c.nrSubblocks(), 1)
self.assertGreaterEqual(c.factor(), 1)
@@ -69,7 +69,7 @@ class TestBlockConstraints(unittest.TestCase):
incoh.nrChannelsPerSubband = 16
incoh.timeIntegrationFactor = 4
- c = BlockConstraints(incoherentStokesSettings=incoh)
+ c = BlockConstraints(incoherentStokesSettings=[incoh])
self.assertEqual(c.nrSubblocks(), 1)
self.assertGreaterEqual(c.factor(), 1)
@@ -94,7 +94,7 @@ class TestBlockSize(unittest.TestCase):
correlator.nrChannelsPerSubband = 64
correlator.integrationTime = integrationTime
- c = BlockConstraints( correlator, None, None )
+ c = BlockConstraints(correlator)
bs = BlockSize(c)
self.assertAlmostEquals(c._samples2time(bs.integrationSamples), integrationTime, delta = integrationTime * 0.05)
diff --git a/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py b/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py
index 3c1277412b49bbeafaa15f05f58014bbb88c3dc7..4b46cc74036b44625b0df5d88c1d03d07c6c8449 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py
@@ -41,6 +41,7 @@ from typing import NamedTuple
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.exceptions import *
+from lofar.sas.tmss.tmss.tmssapp.reservations import get_active_station_reservations_in_timewindow
################## main data struct and methods ##################
@@ -238,23 +239,7 @@ def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUni
return lower_bound
-def get_active_station_reservations_in_timewindow(lower_bound, upper_bound):
- """
- Retrieve a list of all active stations reservations, which are reserved between a timewindow
- TODO: use filter like filter(start_time__lte=upper) filter(stop_time__gte=lower)
- BUT can not use filter of property, so find another 'fast' solution (no loop), therefore stop_time has to move to
- to the model. See TMSS-668
- Also move this part to other module
- """
- lst_active_station_reservations = []
- reservations = models.Reservation.objects.all()
- for station_reservation in reservations:
- if (station_reservation.duration is not None and \
- station_reservation.start_time < upper_bound and station_reservation.stop_time > lower_bound) \
- or (station_reservation.duration is None and station_reservation.start_time < upper_bound):
- lst_active_station_reservations += station_reservation.specifications_doc["resources"]["stations"]
-
- return lst_active_station_reservations
+
def can_run_within_station_reservations(scheduling_unit: models.SchedulingUnitBlueprint) -> bool:
diff --git a/SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py b/SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py
index 2b68de96458c95d1cd2e068f7916dc9851ff3a45..910fc96e2c37ba32e21546ed87935083b3bba7a9 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py
@@ -158,14 +158,23 @@ def can_run_within_timewindow_with_time_constraints(scheduling_unit: models.Sche
constraints are met over the runtime of the observation, else False.
"""
main_observation_task_name = get_target_observation_task_name_from_requirements_doc(scheduling_unit)
- duration = timedelta(
- seconds=scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']['duration'])
- window_lower_bound = lower_bound
- while window_lower_bound + duration < upper_bound:
- window_upper_bound = window_lower_bound + duration
- if can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit, window_lower_bound, window_upper_bound):
- return True
- window_lower_bound += min(timedelta(hours=1), upper_bound - window_lower_bound)
+ constraints = scheduling_unit.draft.scheduling_constraints_doc
+
+ # Check the 'at' constraint and then only check can_run_anywhere for the single possible time window
+ if 'at' in constraints['time']:
+ at = parser.parse(constraints['time']['at'], ignoretz=True)
+ if (at >= lower_bound and at + scheduling_unit.duration <= upper_bound): # todo: suggestion: use scheduling_unit.requirements_doc['tasks']['Observation']['specifications_doc']['duration']
+ return can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit, lower_bound=at,
+ upper_bound=at + scheduling_unit.duration)
+ else:
+ duration = timedelta(
+ seconds=scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']['duration'])
+ window_lower_bound = lower_bound
+ while window_lower_bound + duration <= upper_bound:
+ window_upper_bound = window_lower_bound + duration
+ if can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit, window_lower_bound, window_upper_bound):
+ return True
+ window_lower_bound += min(timedelta(hours=1), upper_bound - window_lower_bound)
return False
@@ -176,25 +185,21 @@ def can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit: mo
i.e. the time constraints must be met over the full time window.
:return: True if all time constraints are met over the entire time window, else False.
"""
- can_run_at = True
can_run_before = True
can_run_with_after = True
can_run_between = True
can_run_not_between = True
constraints = scheduling_unit.draft.scheduling_constraints_doc
- # TODO TMSS-672 Move to can_run_within and make logic correct
- if has_manual_scheduler_constraint(scheduling_unit):
- at = parser.parse(constraints['time']['at'], ignoretz=True)
- can_run_at = (at >= lower_bound and at+scheduling_unit.duration <= upper_bound) # todo: suggestion: use scheduling_unit.requirements_doc['tasks']['Observation']['specifications_doc']['duration']
-
+ # given time window needs to end before constraint
if 'before' in constraints['time']:
before = parser.parse(constraints['time']['before'], ignoretz=True)
- can_run_before = (before <= upper_bound-scheduling_unit.duration) # todo: suggestion: use scheduling_unit.requirements_doc['tasks']['Observation']['specifications_doc']['duration']
+ can_run_before = (upper_bound < before)
+ # given time window needs to start after constraint
if 'after' in constraints['time']:
after = parser.parse(constraints['time']['after'], ignoretz=True)
- can_run_with_after = (lower_bound >= after)
+ can_run_with_after = (lower_bound > after)
# Run within one of these time windows
if 'between' in constraints['time']:
@@ -202,9 +207,9 @@ def can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit: mo
for between in constraints['time']['between']:
time_from = parser.parse(between["from"], ignoretz=True)
time_to = parser.parse(between["to"], ignoretz=True)
- if time_from >= lower_bound and time_to <= upper_bound:
+ if time_from <= lower_bound and time_to >= upper_bound:
can_run_between = True
- break # something inside the boundary so True and don't look any further
+ break # constraint window completely covering the boundary, so True and don't look any further
else:
can_run_between = False
@@ -216,11 +221,11 @@ def can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit: mo
time_to = parser.parse(not_between["to"], ignoretz=True)
if time_from <= upper_bound and time_to >= lower_bound:
can_run_not_between = False
- break # something outside the boundary so False and don't look any further
+ break # constraint window at least partially inside the boundary, so False and don't look any further
else:
can_run_not_between = True
- return can_run_at & can_run_before & can_run_with_after & can_run_between & can_run_not_between
+ return can_run_before & can_run_with_after & can_run_between & can_run_not_between
def can_run_within_timewindow_with_sky_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
@@ -233,7 +238,7 @@ def can_run_within_timewindow_with_sky_constraints(scheduling_unit: models.Sched
if 'duration' in task['specifications_doc']:
duration = timedelta(seconds=task['specifications_doc']['duration'])
window_lower_bound = lower_bound
- while window_lower_bound + duration < upper_bound:
+ while window_lower_bound + duration <= upper_bound:
window_upper_bound = window_lower_bound + duration
if can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit, window_lower_bound, window_upper_bound):
return True
@@ -283,7 +288,9 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
target_rise_and_set_times = coordinates_timestamps_and_stations_to_target_rise_and_set(angle1=angle1, angle2=angle2, direction_type=direction_type, timestamps=timestamps, stations=tuple(stations), angle_to_horizon=min_elevation)
for station, times in target_rise_and_set_times.items():
for i in range(len(timestamps)):
- if not (timestamps[i] > times[0]['rise'] and timestamps[i] < times[0]['set']):
+ if times[0]['always_above_horizon']:
+ continue
+ if times[0]['always_below_horizon'] or not (timestamps[i] > times[0]['rise'] and timestamps[i] < times[0]['set']):
if task['specifications_template'] == 'calibrator observation':
logger.info('min_calibrator_elevation=%s constraint is not met at timestamp=%s' % (min_elevation.rad, timestamps[i]))
else:
@@ -307,7 +314,7 @@ def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBluep
main_observation_task_name = get_target_observation_task_name_from_requirements_doc(scheduling_unit)
duration = timedelta(seconds=scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']['duration'])
try:
- if has_manual_scheduler_constraint(scheduling_unit) and 'at' in constraints['time']:
+ if 'at' in constraints['time']:
at = parser.parse(constraints['time']['at'], ignoretz=True)
return max(lower_bound, at)
@@ -381,10 +388,10 @@ def compute_scores(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound:
# TODO: TMSS-244 (and more?), compute score using the constraints in constraints['time']
# TODO: TMSS-245 TMSS-250 (and more?), compute score using the constraints in constraints['sky']
- # for now (as a proof of concept and sort of example), just return 1's
+ # for now (as a proof of concept and sort of example), just return 1's. Return 1000 (placeholder value, change later) if the 'at' constraint is in, so it gets prioritised.
scores = {'daily': 1.0,
- 'time': 1.0,
- 'sky': 1.0 }
+ 'time': 1000.0 if ('at' in constraints['time'] and constraints['time']['at'] is not None) else 1.0,
+ 'sky': 1.0}
# add "common" scores which do not depend on constraints, such as project rank and creation date
# TODO: should be normalized!
diff --git a/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py b/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
index 45efbf8ae32a51b02d26139010d6f5c125fc2334..bcd9f1fb6aa1d3dbbed8334c186dd3f53cb1e161 100755
--- a/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
@@ -33,6 +33,7 @@ if skip_integration_tests():
TEST_UUID = uuid.uuid1()
from datetime import datetime, timedelta
+from lofar.common.datetimeutils import round_to_second_precision
from lofar.common.json_utils import get_default_json_object_for_schema, add_defaults_to_json_object_for_schema
from lofar.messaging.messagebus import TemporaryExchange, BusListenerJanitor
@@ -136,6 +137,76 @@ class TestDynamicScheduling(TestCase): # Note: we use django.test.TestCase inst
scheduling_constraints_doc=constraints,
scheduling_constraints_template=constraints_template)
+ def test_simple_observation_with_at_constraint(self):
+ """
+ Test a simple observation with the 'at' constraint
+ """
+ scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data())
+ scheduling_unit_draft = self.create_simple_observation_scheduling_unit('scheduling_unit for at constraint', scheduling_set=scheduling_set)
+ # Clear constraints
+ scheduling_unit_draft.scheduling_constraints_doc['sky'] = {}
+ scheduling_unit_draft.scheduling_constraints_doc['time']["between"] = []
+ scheduling_unit_draft.scheduling_constraints_doc['time']["not_between"] = []
+ scheduling_unit_draft.scheduling_constraints_doc['time'].pop('at', None)
+ scheduling_unit_draft.scheduling_constraints_doc['time'].pop("before", None)
+ scheduling_unit_draft.scheduling_constraints_doc['time'].pop('after', None)
+ # Set at constraint
+ at = round_to_second_precision(datetime.utcnow() + timedelta(minutes=10))
+ scheduling_unit_draft.scheduling_constraints_doc['time']['at'] = at.isoformat()
+ scheduling_unit_draft.save()
+ scheduling_unit_blueprint = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+
+ scheduled_scheduling_unit = do_dynamic_schedule()
+
+ # Assert the scheduling_unit has been scheduled and assert is has been scheduled at "at" timestamp
+ self.assertIsNotNone(scheduled_scheduling_unit)
+ self.assertEqual(scheduled_scheduling_unit.id, scheduling_unit_blueprint.id)
+ self.assertEqual(scheduled_scheduling_unit.status, 'scheduled')
+ self.assertEqual(scheduled_scheduling_unit.start_time, at)
+
+ def test_n_simple_observations_one_at_constraint(self):
+ """
+ Test n simple observations where only one of them has an 'at' constraint
+ """
+ n = 5 # No of SU to be created
+ target = 4 # SU id to be within the 'at' constraint
+ target_scheduling_unit_blueprint = None # SU which will be our target
+
+ # Create constraints to be assigned to all of the scheduling_units
+ from_timestamp = round_to_second_precision(datetime.utcnow())
+ to_timestamp = round_to_second_precision(datetime.utcnow() + timedelta(hours=12))
+ between_constraints = [{"from": from_timestamp.isoformat(), "to": to_timestamp.isoformat()},]
+ # Create at constraint to be assigned only to one of the scheduling_units
+ at = round_to_second_precision((datetime.utcnow() + timedelta(minutes=30)))
+
+ # Create n scheduling_units and set the proper constraints
+ for su in range(1, n+1):
+ scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data())
+ scheduling_unit_draft = self.create_simple_observation_scheduling_unit('scheduling_unit %s' % su,
+ scheduling_set=scheduling_set)
+ # Clear constraints
+ scheduling_unit_draft.scheduling_constraints_doc['sky'] = {}
+ scheduling_unit_draft.scheduling_constraints_doc['time']["between"] = between_constraints
+ scheduling_unit_draft.scheduling_constraints_doc['time']["not_between"] = []
+ scheduling_unit_draft.scheduling_constraints_doc['time'].pop("before", None)
+ scheduling_unit_draft.scheduling_constraints_doc['time'].pop('after', None)
+ scheduling_unit_draft.scheduling_constraints_doc['time'].pop("at", None)
+ scheduling_unit_draft.save()
+ if su == target: # Only scheduling_unit with id 'target' is set within an 'at' constraint
+ scheduling_unit_draft.scheduling_constraints_doc['time']['at'] = at.isoformat()
+ scheduling_unit_draft.save()
+ target_scheduling_unit_blueprint = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+ else:
+ create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+
+ scheduled_scheduling_unit = do_dynamic_schedule()
+
+ # Assert the 'target' scheduling_unit has been scheduled with priority and assert it is has been scheduled at "at" timestamp
+ self.assertIsNotNone(scheduled_scheduling_unit)
+ self.assertEqual(scheduled_scheduling_unit.id, target_scheduling_unit_blueprint.id)
+ self.assertEqual(scheduled_scheduling_unit.status, 'scheduled')
+ self.assertEqual(scheduled_scheduling_unit.start_time, at)
+
@unittest.skip("FIX TEST, skipping it for now, see TODO comment in assign_start_stop_times_to_schedulable_scheduling_units")
def test_three_simple_observations_no_constraints_different_project_priority(self):
scheduling_unit_draft_low = self.create_simple_observation_scheduling_unit("scheduling unit low", scheduling_set=self.scheduling_set_low)
@@ -179,6 +250,7 @@ class TestDynamicScheduling(TestCase): # Note: we use django.test.TestCase inst
self.assertGreaterEqual(scheduling_unit_blueprint_medium.start_time - scheduling_unit_blueprint_high.stop_time, DEFAULT_INTER_OBSERVATION_GAP)
self.assertGreaterEqual(scheduling_unit_blueprint_low.start_time - scheduling_unit_blueprint_medium.stop_time, DEFAULT_INTER_OBSERVATION_GAP)
+ @unittest.skip("Skipped because the corrected 'before' constraint broke scheduler behavior. See TMSS-705")
def test_time_bound_unit_wins_even_at_lower_priority(self):
# create two schedule units, one with high one with low prio.
# first create them without any further constraints, and check if high prio wins.
@@ -198,7 +270,7 @@ class TestDynamicScheduling(TestCase): # Note: we use django.test.TestCase inst
self.assertEqual(scheduling_unit_blueprint_high.id, best_scored_scheduling_unit.scheduling_unit.id)
#now update the low prio unit with a time constraint, "forcing" it to be run in a very thight upcoming time window.
- scheduling_unit_draft_low.scheduling_constraints_doc['time'] = { 'before': (now+scheduling_unit_draft_low.duration).isoformat()+'Z' }
+ scheduling_unit_draft_low.scheduling_constraints_doc['time'] = { 'before': (now+scheduling_unit_draft_low.duration+timedelta(seconds=10)).isoformat()+'Z' }
scheduling_unit_draft_low.save()
scheduling_unit_blueprint_low.refresh_from_db()
@@ -206,22 +278,20 @@ class TestDynamicScheduling(TestCase): # Note: we use django.test.TestCase inst
best_scored_scheduling_unit = find_best_next_schedulable_unit([scheduling_unit_blueprint_low, scheduling_unit_blueprint_high], now, tomorrow)
# now we expect the scheduling_unit with the lowest project rank to be scheduled first because it can only run within this limited timewindow
- self.assertEqual(scheduling_unit_draft_low.id, best_scored_scheduling_unit.scheduling_unit.id)
+ self.assertEqual(scheduling_unit_blueprint_low.id, best_scored_scheduling_unit.scheduling_unit.id)
# update the low prio unit. enlarge the time window constraint a bit, so both low and high prio units can fit
# this should result that the high prio goes first, and the low prio (which now fits as well) goes second
- scheduling_unit_draft_low.scheduling_constraints_doc['time'] = \
- { 'before': (now+scheduling_unit_draft_low.duration+scheduling_unit_draft_high.duration).isoformat()+'Z' }
+ scheduling_unit_draft_low.scheduling_constraints_doc['time'] = { 'before': (now+scheduling_unit_draft_low.duration+scheduling_unit_draft_high.duration+timedelta(seconds=10)).isoformat()+'Z' }
scheduling_unit_draft_low.save()
scheduling_unit_blueprint_low.refresh_from_db()
# call the method-under-test.
best_scored_scheduling_unit = find_best_next_schedulable_unit([scheduling_unit_blueprint_low, scheduling_unit_blueprint_high], now, tomorrow)
- # now we expect the scheduling_unit with the lowest project rank to be scheduled first because it can only
- # run within this limited timewindow
- self.assertEqual(scheduling_unit_blueprint_low.id, best_scored_scheduling_unit.scheduling_unit.id)
+ # now we again expect the scheduling_unit with the higher project rank to be scheduled first
+ self.assertEqual(scheduling_unit_blueprint_high.id, best_scored_scheduling_unit.scheduling_unit.id)
# call the method-under-test again but search after first unit (should return low prio unit)
stop_time_of_first = best_scored_scheduling_unit.start_time + best_scored_scheduling_unit.scheduling_unit.duration
@@ -735,8 +805,11 @@ class TestSkyConstraints(unittest.TestCase):
self.distance_mock.return_value = self.distance_data
self.addCleanup(self.distance_patcher.stop)
- self.target_rise_and_set_data = {"CS002": [{"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0)},
- {"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0)}]}
+ self.target_rise_and_set_data = {"CS002": [{"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0), "always_above_horizon": False, "always_below_horizon": False},
+ {"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0), "always_above_horizon": False, "always_below_horizon": False}]}
+ self.target_rise_and_set_data_always_above = {"CS002": [{"rise": None, "set": None, "always_above_horizon": True, "always_below_horizon": False}]}
+ self.target_rise_and_set_data_always_below = {"CS002": [{"rise": None, "set": None, "always_above_horizon": False, "always_below_horizon": True}]}
+
self.target_rise_and_set_patcher = mock.patch('lofar.sas.tmss.services.scheduling.constraints.template_constraints_v1.coordinates_timestamps_and_stations_to_target_rise_and_set')
self.target_rise_and_set_mock = self.target_rise_and_set_patcher.start()
self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data
@@ -757,7 +830,7 @@ class TestSkyConstraints(unittest.TestCase):
timestamp = datetime(2020, 1, 1, 10, 0, 0)
returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
self.assertFalse(returned_value)
-
+
# min_target_elevation
def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_true_when_met(self):
@@ -779,12 +852,18 @@ class TestTimeConstraints(TestCase):
"""
Tests for the time constraint checkers used in dynamic scheduling with different boundaries
Possible time constraints are
+ - at
- after
- before
- between (one or more 'from-to')
- not between (one or more 'from-to')
"""
+ def add_time_at_constraint(self, at_timestamp):
+ lst_at_constraint = self.scheduling_unit_blueprint.draft.scheduling_constraints_doc
+ lst_at_constraint['time']['at'] = at_timestamp.isoformat()
+ self.scheduling_unit_blueprint.save()
+
def add_time_between_constraint(self, from_timestamp, to_timestamp):
lst_between_constraints = self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["between"]
time_constraint_dict = {"from": from_timestamp.isoformat(), "to": to_timestamp.isoformat()}
@@ -797,6 +876,13 @@ class TestTimeConstraints(TestCase):
lst_between_constraints.append(time_constraint_dict)
self.scheduling_unit_blueprint.save()
+ def clear_time_constraints(self):
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["between"] = []
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["not_between"] = []
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time'].pop('at', None)
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time'].pop("before", None)
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time'].pop('after', None)
+
def setUp(self) -> None:
# scheduling unit
self.obs_duration = 120 * 60
@@ -807,113 +893,256 @@ class TestTimeConstraints(TestCase):
obs_duration=self.obs_duration)
self.scheduling_unit_blueprint = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+ # 'after' constraint
+
+ def test_can_run_anywhere_after_returns_true(self):
+
+ # Set datetime constraints before lower_bound
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 11, 0, 0).isoformat()
+ self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
def test_can_run_anywhere_after_returns_false(self):
+
+ # Set datetime constraints equal to lower_bound
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 12, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
# Set datetime constraints after lower_bound
+ self.clear_time_constraints()
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 13, 0, 0).isoformat()
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
# Set datetime constraints to upper_bound
+ self.clear_time_constraints()
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 2, 12, 0, 0).isoformat()
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
- def test_can_run_anywhere_after_returns_true(self):
- # Set datetime constraints before lower_bound
+ # Set datetime constraints after upper_bound
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 2, 13, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ def test_can_run_within_after_returns_false(self):
+
+ # Set datetime constraints before lower bounds, but with too short window for obs duration
+ self.clear_time_constraints()
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 11, 0, 0).isoformat()
- self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 13, 0, 0)))
+
+ # Set datetime constraints after lower bounds, and with too little space left in window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 14, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 15, 0, 0)))
+
+ def test_can_run_within_after_returns_true(self):
+
+ # Set datetime constraints before lower bounds, and with sufficient window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 11, 0, 0).isoformat()
+ self.assertTrue(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 14, 0, 0)))
+
+ # Set datetime constraints after lower bounds, but with sufficient space left in window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 13, 0, 0).isoformat()
+ self.assertTrue(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 16, 0, 0)))
+
+ # 'before' constraint
+
+ def test_can_run_anywhere_before_returns_false(self):
+
+ # Set datetime constraints before lower_bound
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 11, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
+
# Set datetime constraints equal to lower_bound
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 12, 0, 0).isoformat()
- self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 12, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
- def test_can_run_anywhere_before_returns_false(self):
- # Set datetime constraints after upper_bound
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 13, 0, 0).isoformat()
+ # Set datetime constraints after lower_bound
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 13, 0, 0).isoformat()
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
# Set datetime constraints equal to upper_bound
+ self.clear_time_constraints()
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 12, 0, 0).isoformat()
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
- # Set datetime constraints equal to upper_bound - duration + 1 sec
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = \
- (datetime(2020, 1, 2, 12, 0, 0) - self.scheduling_unit_blueprint.duration + timedelta(seconds=1)).isoformat()
- self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
- datetime(2020, 1, 1, 12, 0, 0),
- datetime(2020, 1, 2, 12, 0, 0)))
+
def test_can_run_anywhere_before_returns_true(self):
- # Set datetime constraints far before upper_bound (lower_bound)
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 12, 0, 0).isoformat()
+
+ # Set datetime constraints after upper_bound
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 13, 0, 0).isoformat()
self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
- # Set datetime constraints equal to upper_bound - duration
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = \
- (datetime(2020, 1, 2, 12, 0, 0) - self.scheduling_unit_blueprint.duration).isoformat()
- self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+
+ def test_can_run_within_before_returns_false(self):
+
+ # Set datetime constraints after upper bound, but with too short window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 13, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 2, 11, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ # Set datetime constraints after lower bound, and with too little space left in window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 13, 0, 0).isoformat()
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0),
datetime(2020, 1, 2, 12, 0, 0)))
+ def test_can_run_within_before_returns_true(self):
+
+ # Set datetime constraints after upper bounds, and with sufficient window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 13, 0, 0).isoformat()
+ self.assertTrue(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ # Set datetime constraints after lower bounds, but with sufficient space left in window for obs duration
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 15, 0, 0).isoformat()
+ self.assertTrue(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ # 'between' constraint
+
def test_can_run_anywhere_between_returns_false(self):
"""
Test 'between' constraint with start/stop datetime constraints 'outside' upper_bound or lower_bound
"""
# Set datetime constraints start > lower_bound and stop > upper_bound
+ self.clear_time_constraints()
self.add_time_between_constraint(datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 2, 15, 0, 0))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
+
# Set datetime constraints start < lower_bound and stop < upper_bound
+ self.clear_time_constraints()
self.add_time_between_constraint(datetime(2020, 1, 1, 8, 0, 0), datetime(2020, 1, 2, 8, 0, 0))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
+
# Set datetime constraints start > lower_bound and stop > upper_bound (1 second only)
+ self.clear_time_constraints()
self.add_time_between_constraint(datetime(2020, 1, 1, 12, 0, 1), datetime(2020, 1, 2, 12, 0, 1))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
+ # Set datetime constraints start > lower_bound and stop < upper_bound
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 18, 0, 0), datetime(2020, 1, 1, 19, 0, 0))
+ self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
+
def test_can_run_anywhere_between_returns_true(self):
"""
- Test 'between' constraint with start/stop datetime constraints 'inside' upper_bound and lower_bound
+ Test 'between' constraint with start/stop datetime constraints 'outside' upper_bound and lower_bound
"""
- # Set datetime constraints start > lower_bound and stop < upper_bound -duration
- self.add_time_between_constraint(datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 1, 15, 0, 0))
+ # Set datetime constraints start < lower_bound and stop > upper_bound
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 11, 0, 0), datetime(2020, 1, 2, 13, 0, 0))
self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
- datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 20, 0, 0)))
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
- # Set datetime constraints start = lower_bound and stop = upper_bound - duration
- self.add_time_between_constraint(datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 1, 15, 0, 0))
+ # Set datetime constraints start = lower_bound and stop = upper_bound
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0))
self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
- datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 1, 17, 10, 0)))
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
+
+ def test_can_run_within_between_returns_true(self):
+ """
+ Test 'between' constraint with start/stop datetime constraints (within, not anywhere within)
+ """
+ # Set datetime constraints start > lower_bound and stop > upper_bound, large window
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 2, 12, 0, 0))
+ self.assertTrue(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 20, 0, 0)))
+
+ # Set datetime constraints start = lower_bound and stop = upper_bound, window just large enough for obs
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 14, 0, 0))
+ self.assertTrue(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 14, 10, 0)))
+
+ def test_can_run_within_between_returns_false(self):
+ """
+ Test 'between' constraint with start/stop datetime constraints (within, not anywhere within)
+ """
+ # Set datetime constraints start < lower_bound and stop < upper_bound, too little overlap for obs
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 10, 0, 0), datetime(2020, 1, 1, 13, 0, 0))
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 20, 0, 0)))
+
+ # Set datetime constraints start > lower_bound and stop < upper_bound, constraint window too small for obs
+ self.clear_time_constraints()
+ self.add_time_between_constraint(datetime(2020, 1, 1, 14, 0, 0), datetime(2020, 1, 1, 15, 0, 0))
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 20, 10, 0)))
+
+ # 'not between' contraint
def test_can_run_anywhere_not_between_returns_false(self):
"""
Test 'not_between' constraint with start/stop datetime constraints 'inside' upper_bound or lower_bound
"""
# Set datetime constraints start > lower_bound and stop > upper_bound
+ self.clear_time_constraints()
self.add_time_not_between_constraint(datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 2, 15, 0, 0))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
# Set datetime constraints start < lower_bound and stop > lower_bound and < upper_bound
+ self.clear_time_constraints()
self.add_time_not_between_constraint(datetime(2020, 1, 1, 8, 0, 0), datetime(2020, 1, 2, 8, 0, 0))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
# Set datetime constraints start > lower_bound and stop < upper_bound
+ self.clear_time_constraints()
self.add_time_not_between_constraint(datetime(2020, 1, 1, 16, 0, 0), datetime(2020, 1, 2, 8, 0, 0))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
# Set datetime constraints start < lower_bound and stop > upper_bound
+ self.clear_time_constraints()
self.add_time_not_between_constraint(datetime(2020, 1, 1, 8, 0, 0), datetime(2020, 1, 2, 14, 0, 0))
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
@@ -923,23 +1152,78 @@ class TestTimeConstraints(TestCase):
Test 'not_between' constraint with start/stop datetime constraints 'outside' upper_bound and lower_bound
"""
# Set datetime constraints start < lower_bound and stop < lower_bound
+ self.clear_time_constraints()
self.add_time_not_between_constraint(datetime(2020, 1, 1, 3, 0, 0), datetime(2020, 1, 1, 11, 0, 0))
self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 16, 0, 0)))
# Set datetime constraints start > upper_bound and stop > upper_bound
+ self.clear_time_constraints()
self.add_time_not_between_constraint(datetime(2020, 1, 1, 16, 0, 0), datetime(2020, 1, 1, 20, 0, 0))
self.assertTrue(tc1.can_run_anywhere_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 1, 15, 0, 0)))
+ # several simultaneous time ranges in 'at' / 'between' / 'not between' constraints
+
def execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary(self):
"""
- Just a simple wrapper to call 'can_run_anywhere_within_timewindow_with_time_constraints' function
+ Just a simple wrapper to call 'can_run_within_timewindow_with_time_constraints' function
with a 24 hours boundary 2020-01-01 12:00 - 2020-01-02 12:00
"""
return (tc1.can_run_within_timewindow_with_time_constraints(
self.scheduling_unit_blueprint, datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0)))
+ def test_can_run_within_at_constraint(self):
+ """
+ Test "at" constraint with both boundary and 'inside' upper_bound and lower_bound
+ """
+ # no constraints defined so should be OK
+ self.clear_time_constraints()
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # Set datetime constraint before lower_bound
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 1, 11, 0, 0))
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 14, 0, 0)))
+
+ # Set datetime constraint at lower_bound, but duration exceeds upper_bound
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 1, 12, 0, 0))
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 14, 0, 0)))
+
+ # Set datetime constraint at upper_bound
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 1, 14, 0, 0))
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 14, 0, 0)))
+
+ # Set datetime constraint after upper_bound
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 1, 15, 0, 0))
+ self.assertFalse(tc1.can_run_within_timewindow_with_time_constraints(self.scheduling_unit_blueprint,
+ datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 1, 14, 0, 0)))
+
+ # Set datetime constraint at lower_bound
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 1, 12, 0, 0))
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # Set datetime constraint that fits the time window
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 1, 18, 30, 0))
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # Set datetime constraint so that obs lasts till exactly upper_bound
+ self.clear_time_constraints()
+ self.add_time_at_constraint(datetime(2020, 1, 2, 9, 50, 0))
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
def test_can_run_within_between_constraints(self):
"""
Test multiple 'between' constraints within 24 boundary and check overall result of
@@ -949,39 +1233,41 @@ class TestTimeConstraints(TestCase):
i.e. 12-14, 13-15, 14-16,..etc.., 9-11
"""
# no constraints defined so should be OK
+ self.clear_time_constraints()
self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
- # Add constraints of 1hr, we still 'can_run'
+ # Add constraints of 1hr, we cannot run
self.add_time_between_constraint(datetime(2020, 1, 1, 13, 0, 0), datetime(2020, 1, 1, 14, 0, 0))
self.add_time_between_constraint(datetime(2020, 1, 1, 16, 0, 0), datetime(2020, 1, 1, 17, 0, 0))
- self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+ self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
- # Add constraints of 2hr, we still 'can_run'
+ # Add constraints of 2hr, but partially outside the bounds, we still cannot run
self.add_time_between_constraint(datetime(2020, 1, 2, 11, 0, 0), datetime(2020, 1, 2, 13, 0, 0))
+ self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # Add constraints of 2hr, we can run again
+ self.add_time_between_constraint(datetime(2020, 1, 1, 17, 0, 0), datetime(2020, 1, 1, 19, 0, 0))
self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
# Add constraint of 24hr constraint, we still 'can_run'
self.add_time_between_constraint(datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 2, 12, 0, 0))
self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
- # Add constraint of 2hr, to fill the 'last gap', we 'can run'
- self.add_time_between_constraint(datetime(2020, 1, 2, 10, 0, 0), datetime(2020, 1, 2, 12, 0, 0))
- self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
-
# Clear all between constraints
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["between"] = []
+ self.clear_time_constraints()
- # Add constraints 'outside' the 24hr, now we 'can not run'
- self.add_time_between_constraint(datetime(2020, 1, 2, 13, 0, 0), datetime(2020, 1, 2, 14, 0, 0))
- self.add_time_between_constraint(datetime(2020, 1, 2, 16, 0, 0), datetime(2020, 1, 2, 17, 0, 0))
+ # Add constraints after the 24hr, now we 'can not run'
+ self.add_time_between_constraint(datetime(2020, 1, 2, 13, 0, 0), datetime(2020, 1, 2, 15, 0, 0))
+ self.add_time_between_constraint(datetime(2020, 1, 2, 16, 0, 0), datetime(2020, 1, 2, 20, 0, 0))
self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
- # Add constraint 'outside' the 24hr, we 'still can not run'
+ # Add constraint before the 24hr, we 'still can not run'
self.add_time_between_constraint(datetime(2020, 1, 1, 9, 0, 0), datetime(2020, 1, 1, 12, 0, 0))
self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
- # add one 'inside' constraint, 1 hour within block of 2 hour so overall must be ok
- self.add_time_between_constraint(datetime(2020, 1, 1, 13, 30, 0), datetime(2020, 1, 1, 14, 30, 0))
+ # add one 'inside' constraint of 3 hours, so overall must be ok again.
+ # Note that 2 hrs would only be sufficient if they match the moving window exactly (here: full hour)
+ self.add_time_between_constraint(datetime(2020, 1, 1, 14, 30, 0), datetime(2020, 1, 1, 17, 30, 0))
self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
def test_can_run_within_not_between_constraints(self):
@@ -993,6 +1279,7 @@ class TestTimeConstraints(TestCase):
i.e. 12-14, 13-15, 14-16,..etc.., 9-11
"""
# no constraints defined so should be OK
+ self.clear_time_constraints()
self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
# Add constraints of 1hr, we still 'can_run'
@@ -1012,12 +1299,60 @@ class TestTimeConstraints(TestCase):
self.add_time_not_between_constraint(datetime(2020, 1, 2, 10, 0, 0), datetime(2020, 1, 2, 12, 0, 0))
self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
- # Clear all not_between constraints
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["not_between"] = []
+ self.clear_time_constraints()
+
# Add 4 hr constraints within 24 hours boundary, we can run
self.add_time_not_between_constraint(datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 16, 0, 0))
self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+ # combined time contraints tests
+
+ def test_can_run_anywhere_combined_time_constraints(self):
+ """
+ Test multiple time constraints in combination and make sure that they block the time window as expected,
+ even though each constraint individually would allow the observation to run.
+ """
+
+ # Set before and after constraint with sufficient gap to fit observation, and assert True
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 12, 59, 59).isoformat()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 15, 0, 1).isoformat()
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # set before and after constraint with slightly smaller gap for observation, and assert False
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 13, 0, 0).isoformat()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 1, 15, 0, 0).isoformat()
+ self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # set before and after constraint with large gap
+ # then and add additional between and not between constraints until window is blocked
+ # can run 13-8h
+ self.clear_time_constraints()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["after"] = datetime(2020, 1, 1, 13, 0, 0).isoformat()
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 8, 0, 0).isoformat()
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # can run 13h-20h
+ self.add_time_between_constraint(datetime(2020, 1, 1, 11, 0, 0), datetime(2020, 1, 1, 20, 0, 0))
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # can run 13h-17h
+ self.add_time_not_between_constraint(datetime(2020, 1, 1, 17, 0, 0), datetime(2020, 1, 2, 4, 0, 0))
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # can not run anymore
+ self.add_time_not_between_constraint(datetime(2020, 1, 1, 12, 0, 0), datetime(2020, 1, 1, 16, 0, 0))
+ self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # add another between window, can run 4h-8h
+ self.add_time_between_constraint(datetime(2020, 1, 1, 2, 0, 0), datetime(2020, 1, 2, 12, 0, 0))
+ self.assertTrue(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
+ # move before constraint, can not run anymore
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['time']["before"] = datetime(2020, 1, 2, 5, 0, 0).isoformat()
+ self.assertFalse(self.execute_can_run_within_timewindow_with_time_constraints_of_24hour_boundary())
+
class TestReservedStations(unittest.TestCase):
"""
@@ -1035,9 +1370,10 @@ class TestReservedStations(unittest.TestCase):
"""
@staticmethod
- def create_station_reservation(additional_name, lst_stations, start_time=datetime(2100, 1, 1, 0, 0, 0), duration=86400):
+ def create_station_reservation(additional_name, lst_stations, start_time=datetime(2100, 1, 1, 0, 0, 0),
+ stop_time=datetime(2100, 1, 2, 0, 0, 0)):
"""
- Create a station reservation with given list of stations, start_time and duration (optional)
+ Create a station reservation with given list of stations, start_time and stop_time (optional)
Default duration is 24 hours (defined in seconds)
"""
reservation_template = models.ReservationTemplate.objects.get(name="resource reservation")
@@ -1048,7 +1384,7 @@ class TestReservedStations(unittest.TestCase):
specifications_template=reservation_template,
specifications_doc=reservation_template_spec,
start_time=start_time,
- duration=duration)
+ stop_time=stop_time)
return res
def setUp(self) -> None:
@@ -1070,8 +1406,7 @@ class TestReservedStations(unittest.TestCase):
Set (1) reservation start_time > SUB start_time and reservation stop_time > SUB stop_time
"""
station_reservation.start_time = self.scheduling_unit_blueprint.start_time + timedelta(minutes=5)
- reservation_stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
- station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+ station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
station_reservation.save()
def set_2_reservation_start_time_lt_sub_start_time_and_stop_time_lt_sub_stop_time(self, station_reservation):
@@ -1079,8 +1414,7 @@ class TestReservedStations(unittest.TestCase):
Set (2) reservation start_time < SUB start_time and reservation stop_time < SUB stop_time
"""
station_reservation.start_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
- reservation_stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
- station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+ station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
station_reservation.save()
def set_3_reservation_start_time_gt_sub_start_time_and_stop_time_lt_sub_stop_time(self, station_reservation):
@@ -1088,8 +1422,7 @@ class TestReservedStations(unittest.TestCase):
Set (3) reservation start_time > SUB start_time and reservation stop_time < SUB stop_time
"""
station_reservation.start_time = self.scheduling_unit_blueprint.start_time + timedelta(minutes=5)
- reservation_stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
- station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+ station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
station_reservation.save()
def set_4_reservation_start_time_lt_sub_start_time_and_stop_time_gt_sub_stop_time(self, station_reservation):
@@ -1097,8 +1430,7 @@ class TestReservedStations(unittest.TestCase):
Set (4) reservation start_time < SUB start_time and reservation stop_time > SUB stop_time
"""
station_reservation.start_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
- reservation_stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
- station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+ station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
station_reservation.save()
def set_5_reservation_start_time_and_stop_time_lt_sub_start_time(self, station_reservation):
@@ -1106,8 +1438,7 @@ class TestReservedStations(unittest.TestCase):
Set (5) reservation start_time and reservation stop_time < SUB start_time
"""
station_reservation.start_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=60)
- reservation_stop_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
- station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+ station_reservation.stop_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
station_reservation.save()
def set_6_reservation_start_time_and_stop_time_gt_sub_stop_time(self, station_reservation):
@@ -1115,8 +1446,7 @@ class TestReservedStations(unittest.TestCase):
Set (6) reservation start_time and reservation stop_time > SUB stop_time
"""
station_reservation.start_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
- reservation_stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=65)
- station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+ station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=65)
station_reservation.save()
def update_station_groups_of_scheduling_unit_blueprint(self):
@@ -1234,7 +1564,7 @@ class TestReservedStations(unittest.TestCase):
Test with different reservation start time and NO stop_time
start_time after SUB stop_time 'can run' all others 'can NOT run'
"""
- reservation_two_no_duration = self.create_station_reservation("Two-NoDuration", ["CS001", "CS002"], duration=None)
+ reservation_two_no_duration = self.create_station_reservation("Two-NoDuration", ["CS001", "CS002"], stop_time=None)
# reservation start_time > SUB start_time and < SUB stop_time
reservation_two_no_duration.start_time = self.scheduling_unit_blueprint.start_time + timedelta(minutes=5)
reservation_two_no_duration.save()
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/CMakeLists.txt b/SAS/TMSS/backend/src/tmss/tmssapp/CMakeLists.txt
index 58c545f7ed434d8c05064e1fad48ebf0c93d821a..456c9935792dbfd31873e09098211a46c046828d 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/CMakeLists.txt
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/CMakeLists.txt
@@ -10,6 +10,7 @@ set(_py_files
subtasks.py
tasks.py
conversions.py
+ reservations.py
)
python_install(${_py_files}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py
index a4fd63788ffff44af3696a8b2c3e4be9999e4d49..00ed6e2a27944488a317e540fa46c972b8b0f13e 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py
@@ -107,15 +107,20 @@ def _convert_correlator_settings_to_parset_dict(subtask: models.Subtask, spec: d
parset[beam_prefix+"Correlator.angle2"] = phase_center['pointing']['angle2']
- # TODO: do not use SubtaskOutput.objects.filter but make subtask.subtask_outputs work
- subtask_outputs = list(models.SubtaskOutput.objects.filter(subtask_id=subtask.id))
- subtask_output_ids = [o.id for o in subtask_outputs]
+ dataproducts = list(subtask.output_dataproducts.filter(dataformat__value=Dataformat.Choices.MEASUREMENTSET.value).filter(datatype__value=Datatype.Choices.VISIBILITIES.value).order_by('filename'))
- # TODO: don't assume ordering by filename is sufficient: we need to inspect the dataproduct properties to make sure saps and subbands are in the correct order
- dataproducts = list(models.Dataproduct.objects.filter(producer_id__in=subtask_output_ids).filter(dataformat=Dataformat.Choices.MEASUREMENTSET.value).filter(datatype=Datatype.Choices.VISIBILITIES).order_by('filename'))
+ # marshall dataproducts, but only if they're supplied. in some use cases, we want a parset before the subtask is scheduled.
+ correlator_dataproducts = []
+ for digi_beam in digi_beams:
+ for subband in digi_beam["subbands"]:
+ dataproduct = [dp for dp in dataproducts
+ if dp.specifications_doc.get("sap") == digi_beam['name']
+ and dp.specifications_doc.get("subband") == subband]
- parset["Observation.DataProducts.Output_Correlated.filenames"] = [dp.filename for dp in dataproducts]
- parset["Observation.DataProducts.Output_Correlated.locations"] = ["%s:%s" % (subtask.cluster.name, dp.directory) for dp in dataproducts]
+ correlator_dataproducts.append(dataproduct[0] if dataproduct else null_dataproduct)
+
+ parset["Observation.DataProducts.Output_Correlated.filenames"] = [dp.filename for dp in correlator_dataproducts]
+ parset["Observation.DataProducts.Output_Correlated.locations"] = ["%s:%s" % (subtask.cluster.name, dp.directory) for dp in correlator_dataproducts]
# mimic MoM placeholder thingy (the resource estimator parses this)
parset["Observation.DataProducts.Output_Correlated.identifications"] = ["TMSS_subtask_%s.SAP%03d" % (subtask.id, sap_nr) for sap_nr in range(len(digi_beams))]
@@ -129,12 +134,8 @@ def _convert_beamformer_settings_to_parset_dict(subtask: models.Subtask, spec: d
parset = {}
- # TODO: do not use SubtaskOutput.objects.filter but make subtask.subtask_outputs work
- subtask_outputs = list(models.SubtaskOutput.objects.filter(subtask_id=subtask.id))
- subtask_output_ids = [o.id for o in subtask_outputs]
-
# TODO: don't assume ordering by filename is sufficient: we need to inspect the dataproduct properties to make sure saps and subbands are in the correct order
- dataproducts = list(models.Dataproduct.objects.filter(producer_id__in=subtask_output_ids).filter(dataformat=Dataformat.Choices.MEASUREMENTSET.value).filter(datatype=Datatype.Choices.TIME_SERIES.value).order_by('filename'))
+ dataproducts = list(subtask.output_dataproducts.filter(dataformat__value=Dataformat.Choices.BEAMFORMED.value).filter(datatype__value=Datatype.Choices.TIME_SERIES.value).order_by('filename'))
# Lists of coherent and incoherent dataproducts that will be produced, in the order COBALT wants them
coherent_dataproducts = []
@@ -144,7 +145,7 @@ def _convert_beamformer_settings_to_parset_dict(subtask: models.Subtask, spec: d
beamformer_pipeline_parsets = []
# Process beamformer pipelines
- for pipeline in spec['COBALT']['beamformer']['tab_pipelines']:
+ for pipeline_idx, pipeline in enumerate(spec['COBALT']['beamformer']['tab_pipelines']):
pipeline_parset = {}
pipeline_parset.update(_add_prefix(_stokes_settings_parset_subkeys(pipeline['coherent']), "CoherentStokes."))
pipeline_parset.update(_add_prefix(_stokes_settings_parset_subkeys(pipeline['incoherent']), "IncoherentStokes."))
@@ -174,11 +175,18 @@ def _convert_beamformer_settings_to_parset_dict(subtask: models.Subtask, spec: d
# marshall dataproducts, but only if they're supplied. in some use cases, we want a parset before the subtask is scheduled.
for s in range(nr_stokes):
for p in range(nr_parts):
- # TODO: don't assume ordering by filename is sufficient: we need to inspect the dataproduct properties to make sure saps and subbands are in the correct order
+ dataproduct = [dp for dp in dataproducts
+ if dp.specifications_doc.get("sap") == sap['name']
+ and "identifiers" in dp.specifications_doc
+ and dp.specifications_doc["identifiers"]["pipeline_index"] == pipeline_idx
+ and dp.specifications_doc["identifiers"]["tab_index"] == tab_idx
+ and dp.specifications_doc["identifiers"]["stokes_index"] == s
+ and dp.specifications_doc["identifiers"]["part_index"] == p
+ and dp.specifications_doc.get("coherent") == tab['coherent']]
if tab['coherent']:
- coherent_dataproducts.append(dataproducts.pop(0) if dataproducts else null_dataproduct)
+ coherent_dataproducts.append(dataproduct[0] if dataproduct else null_dataproduct)
else:
- incoherent_dataproducts.append(dataproducts.pop(0) if dataproducts else null_dataproduct)
+ incoherent_dataproducts.append(dataproduct[0] if dataproduct else null_dataproduct)
if cobalt_version >= 2:
pipeline_parset['Beam[%s].subbandList' % sap_idx] = sap['subbands']
@@ -192,7 +200,8 @@ def _convert_beamformer_settings_to_parset_dict(subtask: models.Subtask, spec: d
beamformer_pipeline_parsets.append(pipeline_parset)
# Process fly's eye pipelines
- for pipeline in spec['COBALT']['beamformer']['flyseye_pipelines']:
+ pipeline_idx_offset = len(beamformer_pipeline_parsets)
+ for pipeline_idx, pipeline in enumerate(spec['COBALT']['beamformer']['flyseye_pipelines'], start=pipeline_idx_offset):
pipeline_parset = {}
pipeline_parset.update(_add_prefix(_stokes_settings_parset_subkeys(pipeline['coherent']), "CoherentStokes."))
pipeline_parset['flysEye'] = True
@@ -206,7 +215,7 @@ def _convert_beamformer_settings_to_parset_dict(subtask: models.Subtask, spec: d
antennaset = spec['stations']['antenna_set']
fields = sum([list(antenna_fields(station, antennaset)) for station in stations], [])
- for field in fields:
+ for field_idx, field in enumerate(fields):
stokes_settings = pipeline['coherent']
nr_subbands = len(sap['subbands'])
@@ -216,8 +225,14 @@ def _convert_beamformer_settings_to_parset_dict(subtask: models.Subtask, spec: d
# marshall dataproducts, but only if they're supplied. in some use cases, we want a parset before the subtask is scheduled.
for s in range(nr_stokes):
for p in range(nr_parts):
- # TODO: don't assume ordering by filename is sufficient: we need to inspect the dataproduct properties to make sure saps and subbands are in the correct order
- coherent_dataproducts.append(dataproducts.pop(0) if dataproducts else null_dataproduct)
+ dataproduct = [dp for dp in dataproducts
+ if dp.specifications_doc["sap"] == sap["name"]
+ and dp.specifications_doc["identifiers"]["pipeline_index"] == pipeline_idx
+ and dp.specifications_doc["identifiers"]["tab_index"] == field_idx
+ and dp.specifications_doc["identifiers"]["stokes_index"] == s
+ and dp.specifications_doc["identifiers"]["part_index"] == p
+ and dp.specifications_doc["coherent"] == True]
+ coherent_dataproducts.append(dataproduct[0] if dataproduct else null_dataproduct)
if cobalt_version >= 2:
pipeline_parset['Beam[%s].stationList' % sap_idx] = pipeline['stations']
@@ -519,9 +534,15 @@ def _convert_to_parset_dict_for_pipelinecontrol_schema(subtask: models.Subtask)
# TODO: do not use SubtaskOutput.objects.filter but make subtask.subtask_outputs work
subtask_outputs = list(models.SubtaskOutput.objects.filter(subtask_id=subtask.id))
- out_dataproducts = []
- for subtask_output in subtask_outputs:
- out_dataproducts = list(models.Dataproduct.objects.filter(producer_id=subtask_output.id))
+ unsorted_out_dataproducts = sum([list(models.Dataproduct.objects.filter(producer_id=subtask_output.id)) for subtask_output in subtask_outputs],[])
+
+ def find_dataproduct(dataproducts: list, specification_doc: dict):
+ hits = [dp for dp in dataproducts if dp.specifications_doc['sap'] == specification_doc['sap']
+ and dp.specifications_doc['subband'] == specification_doc['subband']]
+ return hits[0] if hits else null_dataproduct
+
+ # list output dataproducts in the same order as input dataproducts, matched by the identifiers
+ out_dataproducts = [find_dataproduct(unsorted_out_dataproducts, in_dp.specifications_doc) for in_dp in in_dataproducts]
parset["Observation.DataProducts.Output_Correlated.enabled"] = "true"
parset["Observation.DataProducts.Output_Correlated.filenames"] = "[%s]" % ",".join([dp.filename for dp in out_dataproducts])
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py b/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
index ae926e172f4a39a4ff77a442346fbf25d4505e35..3c0e184ce79ac8e697043dcf8ced5dceba3bf1eb 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
@@ -126,9 +126,10 @@ def coordinates_timestamps_and_stations_to_target_rise_and_set(angle1: float, an
:param stations: tuple of station names, e.g. ("CS002",)
:param angle_to_horizon: the angle between horizon and given coordinates for which rise and set times are returned
:return A dict that maps station names to a list of dicts with rise and set times for each requested date.
+ If rise and set are None, the target is always above or below horizon, and the respective boolean is True.
E.g.
- {"CS002": [{"rise": datetime(2020, 1, 1, 4, 0, 0), "set": datetime(2020, 1, 1, 11, 0, 0)},
- {"rise": datetime(2020, 1, 2, 4, 0, 0), "set": datetime(2020, 1, 2, 11, 0, 0)}]
+ {"CS002": [{"rise": datetime(2020, 1, 1, 4, 0, 0), "set": datetime(2020, 1, 1, 11, 0, 0), "always_above_horizon": False, "always_below_horizon": False},
+ {"rise": datetime(2020, 1, 2, 4, 0, 0), "set": datetime(2020, 1, 2, 11, 0, 0), "always_above_horizon": False, "always_below_horizon": False}]
}
"""
if direction_type == "J2000":
@@ -140,10 +141,29 @@ def coordinates_timestamps_and_stations_to_target_rise_and_set(angle1: float, an
for timestamp in timestamps:
# todo: this can probably be made faster by moving the following logic to an own function with single station/timestamp as input and putting the lru_cache on there.
observer = create_astroplan_observer_for_station(station)
- target_set = observer.target_set_time(target=coord, time=Time(timestamp), horizon=angle_to_horizon, which='next', n_grid_points=TARGET_SET_RISE_PRECISION)
- target_rise = observer.target_rise_time(target=coord, time=Time(target_set), horizon=angle_to_horizon, which='previous', n_grid_points=TARGET_SET_RISE_PRECISION)
+ try:
+ target_set = observer.target_set_time(target=coord, time=Time(timestamp), horizon=angle_to_horizon, which='next', n_grid_points=TARGET_SET_RISE_PRECISION)
+ target_rise = observer.target_rise_time(target=coord, time=Time(target_set), horizon=angle_to_horizon, which='previous', n_grid_points=TARGET_SET_RISE_PRECISION)
+ return_dict.setdefault(station, []).append(
+ {"rise": target_rise.to_datetime(),
+ "set": target_set.to_datetime(),
+ "always_above_horizon": False,
+ "always_below_horizon": False})
+ except TypeError as e:
+ if "numpy.float64" in str(e):
+ # Note: when the target is always above or below horizon, astroplan excepts with the not very
+ # meaningful error: 'numpy.float64' object does not support item assignment
+ # Determine whether the target is always above or below horizon so that we can return some useful
+ # additional info, e.g. for scheduling purposes.
+ is_up = observer.target_is_up(target=coord, time=Time(timestamp), horizon=angle_to_horizon)
+ return_dict.setdefault(station, []).append(
+ {"rise": None,
+ "set": None,
+ "always_above_horizon": is_up,
+ "always_below_horizon": not is_up})
+ else:
+ raise
- return_dict.setdefault(station, []).append({"rise": target_rise.to_datetime(), "set": target_set.to_datetime()})
return return_dict
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
index 2df67c9b37f610d3abe48ac11aeaa440843794f5..d4095c88c950c3c628f52c4a477d6389e9dd2699 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
@@ -453,13 +453,29 @@ class Migration(migrations.Migration):
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='Short description for this reservation, used in overviews', max_length=255)),
('start_time', models.DateTimeField(help_text='Start of this reservation.')),
- ('duration', models.IntegerField(help_text='Duration of this reservation (in seconds). If null, then this reservation is indefinitely.', null=True)),
+ ('stop_time', models.DateTimeField(help_text='Stop time of this reservation. If null, then this reservation is indefinitely.', null=True)),
('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Properties of this reservation')),
],
options={
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='ReservationStrategyTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags',django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
+ ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('template', django.contrib.postgres.fields.jsonb.JSONField(help_text='JSON-data compliant with the JSON-schema in the scheduling_unit_template. This observation strategy template like a predefined recipe with all the correct settings, and defines which parameters the user can alter.')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='ReservationTemplate',
fields=[
@@ -1200,6 +1216,11 @@ class Migration(migrations.Migration):
model_name='reservationtemplate',
constraint=models.UniqueConstraint(fields=('name', 'version'), name='reservationtemplate_unique_name_version'),
),
+ migrations.AddField(
+ model_name='reservationstrategytemplate',
+ name='reservation_template',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.ReservationTemplate'),
+ ),
migrations.AddField(
model_name='reservation',
name='project',
@@ -1218,12 +1239,12 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='projectquotaarchivelocation',
name='project_quota',
- field=models.ForeignKey(help_text='Project to wich this quota belongs.', on_delete=django.db.models.deletion.PROTECT, related_name='project_quota', to='tmssapp.ProjectQuota'),
+ field=models.ForeignKey(help_text='Project to which this quota belongs.', on_delete=django.db.models.deletion.PROTECT, related_name='project_quota', to='tmssapp.ProjectQuota'),
),
migrations.AddField(
model_name='projectquota',
name='project',
- field=models.ForeignKey(help_text='Project to wich this quota belongs.', on_delete=django.db.models.deletion.PROTECT, related_name='quota', to='tmssapp.Project'),
+ field=models.ForeignKey(help_text='Project to which this quota belongs.', on_delete=django.db.models.deletion.PROTECT, related_name='quota', to='tmssapp.Project'),
),
migrations.AddField(
model_name='projectquota',
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py
index 9631cfc2fc3d8051ae1c586b673a8c4d3b553065..80a9fb61594cbe8996f45fe0b0b35a1c842fe319 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py
@@ -14,6 +14,18 @@ from django.urls import reverse as reverse_url
import json
import jsonschema
+class RefreshFromDbInvalidatesCachedPropertiesMixin():
+ """Helper Mixin class which invalidates all 'cached_property' attributes on a model upon refreshing from the db"""
+ def refresh_from_db(self, *args, **kwargs):
+ self.invalidate_cached_properties()
+ return super().refresh_from_db(*args, **kwargs)
+
+ def invalidate_cached_properties(self):
+ from django.utils.functional import cached_property
+ for key, value in self.__class__.__dict__.items():
+ if isinstance(value, cached_property):
+ self.__dict__.pop(key, None)
+
# abstract models
class BasicCommon(Model):
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
index 4c00aa3da2312e0849ca93f5deb0d3a67bee7a9a..976d84842c0464d1950d381f35910f841c9bed34 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
@@ -10,7 +10,7 @@ from django.contrib.postgres.fields import JSONField
from enum import Enum
from django.db.models.expressions import RawSQL
from django.db.models.deletion import ProtectedError
-from .common import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template, NamedCommonPK
+from .common import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template, NamedCommonPK, RefreshFromDbInvalidatesCachedPropertiesMixin
from lofar.common.json_utils import validate_json_against_schema, validate_json_against_its_schema, add_defaults_to_json_object_for_schema
from lofar.sas.tmss.tmss.exceptions import *
from django.core.exceptions import ValidationError
@@ -23,7 +23,7 @@ from django.utils.functional import cached_property
# Mixins
#
-class ProjectPropertyMixin:
+class ProjectPropertyMixin(RefreshFromDbInvalidatesCachedPropertiesMixin):
@cached_property
def project(self): # -> Project:
'''return the related project of this task
@@ -235,6 +235,26 @@ class DefaultTaskRelationSelectionTemplate(BasicCommon):
template = ForeignKey("TaskRelationSelectionTemplate", on_delete=PROTECT)
+class ReservationStrategyTemplate(NamedCommon):
+ '''
+ A ReservationStrategyTemplate is a template in the sense that it serves as a template to fill in json data objects
+ conform its referred reservation_template.
+ It is however not derived from the (abstract) Template super-class, because the Template super class is for
+ JSON schemas, not JSON data objects.
+ '''
+ version = CharField(max_length=128, help_text='Version of this template (with respect to other templates of the same name).')
+ template = JSONField(null=False, help_text='JSON-data compliant with the JSON-schema in the reservation_template. '
+ 'This reservation strategy template like a predefined recipe with all '
+ 'the correct settings, and defines which parameters the user can alter.')
+ reservation_template = ForeignKey("ReservationTemplate", on_delete=PROTECT, null=False, help_text="")
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ if self.template and self.reservation_template_id and self.reservation_template.schema:
+ validate_json_against_schema(self.template, self.reservation_template.schema)
+
+ super().save(force_insert, force_update, using, update_fields)
+
+
class ReservationTemplate(Template):
pass
@@ -248,7 +268,7 @@ class DefaultReservationTemplate(BasicCommon):
# Instance Objects
#
-class Cycle(NamedCommonPK):
+class Cycle(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommonPK):
start = DateTimeField(help_text='Moment at which the cycle starts, that is, when its projects can run.')
stop = DateTimeField(help_text='Moment at which the cycle officially ends.')
@@ -275,7 +295,7 @@ class CycleQuota(Model):
resource_type = ForeignKey('ResourceType', on_delete=PROTECT, help_text='Resource type.')
-class Project(NamedCommonPK):
+class Project(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommonPK):
# todo: cycles should be protected since we have to manually decide to clean up projects with a cycle or keep them without cycle, however, ManyToManyField does not allow for that
cycles = ManyToManyField('Cycle', related_name='projects', blank=True, help_text='Cycles to which this project belongs (NULLable).')
priority_rank = FloatField(null=False, help_text='Priority of this project w.r.t. other projects. Projects can interrupt observations of lower-priority projects.') # todo: add if needed: validators=[MinValueValidator(0.0), MaxValueValidator(1.0)]
@@ -307,8 +327,8 @@ class ProjectQuota(Model):
resource_type = ForeignKey('ResourceType', on_delete=PROTECT, help_text='Resource type.') # protected to avoid accidents
-class ProjectQuotaArchiveLocation(Model):
- project_quota = ForeignKey('ProjectQuota', null=False, related_name="project_quota", on_delete=PROTECT, help_text='Project to wich this quota belongs.')
+class ProjectQuotaArchiveLocation(RefreshFromDbInvalidatesCachedPropertiesMixin, Model):
+ project_quota = ForeignKey('ProjectQuota', null=False, related_name="project_quota_archive_location", on_delete=PROTECT, help_text='The ProjectQuota for this archive location')
archive_location = ForeignKey('Filesystem', null=False, on_delete=PROTECT, help_text='Location of an archive LTA cluster.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
@@ -344,7 +364,7 @@ class SchedulingSet(NamedCommon):
super().save(force_insert, force_update, using, update_fields)
-class SchedulingUnitDraft(NamedCommon):
+class SchedulingUnitDraft(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
requirements_doc = JSONField(help_text='Scheduling and/or quality requirements for this run.')
copies = ForeignKey('SchedulingUnitDraft', related_name="copied_from", on_delete=SET_NULL, null=True, help_text='Source reference, if we are a copy (NULLable).')
copy_reason = ForeignKey('CopyReason', null=True, on_delete=PROTECT, help_text='Reason why source was copied (NULLable).')
@@ -408,7 +428,7 @@ class SchedulingUnitDraft(NamedCommon):
return self.scheduling_set.project
-class SchedulingUnitBlueprint(NamedCommon):
+class SchedulingUnitBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
class Status(Enum):
DEFINED = "defined"
FINISHED = "finished"
@@ -807,7 +827,7 @@ class TaskDraft(NamedCommon, ProjectPropertyMixin):
# return None
-class TaskBlueprint(NamedCommon):
+class TaskBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
specifications_doc = JSONField(help_text='Schedulings for this task (IMMUTABLE).')
do_cancel = BooleanField(help_text='Cancel this task.')
@@ -1040,19 +1060,20 @@ class Reservation(NamedCommon):
project = ForeignKey('Project', null=True, related_name='reservations', on_delete=CASCADE, help_text='Reservation will be accounted for this project.')
description = CharField(max_length=255, help_text='Short description for this reservation, used in overviews')
start_time = DateTimeField(help_text='Start of this reservation.')
- duration = IntegerField(null=True, help_text='Duration of this reservation (in seconds). If null, then this reservation is indefinitely.')
+ stop_time = DateTimeField(null=True, help_text='Stop of this reservation. If null, then this reservation is indefinitely.')
specifications_doc = JSONField(help_text='Properties of this reservation')
specifications_template = ForeignKey('ReservationTemplate', on_delete=CASCADE, help_text='Schema used for specifications_doc.')
- # TODO add stop_time to the model and calculate either duration or stop_time (in serializer)
- # See TMSS-668
@property
- def stop_time(self) -> datetime.datetime:
- '''The stop_time based on start_time+duration if duration is known, else None'''
- if self.duration:
- return self.start_time + datetime.timedelta(seconds=self.duration)
- return None
+ def duration(self) -> int:
+ '''return the overall duration (in seconds) of this task, if stop_time in None than duration ia also None
+ '''
+ if self.stop_time:
+ return (self.stop_time - self.start_time).total_seconds()
+ else:
+ return None
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
super().save(force_insert, force_update, using, update_fields)
+
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
index d9352294552550890cf404aaf40074803f83c6fb..72c334977eb4d4369ab8f78e35d31ae46341aa1a 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
@@ -79,15 +79,18 @@ def populate_test_data():
if 'Commissioning' not in tmss_project.tags:
continue
- # for test purposes also add a reservation object
- reservation_template = models.ReservationTemplate.objects.get(name="resource reservation")
- reservation_template_spec = get_default_json_object_for_schema(reservation_template.schema)
- Reservation.objects.create(name="DummyReservation",
- description="Just A non-scheduled reservation as example",
- project=tmss_project,
- specifications_template=reservation_template,
- specifications_doc=reservation_template_spec,
- start_time=datetime.now())
+ # for test purposes also create reservation objects from all reservation strategies
+ for strategy_template in ReservationStrategyTemplate.objects.all():
+ reservation_spec = add_defaults_to_json_object_for_schema(strategy_template.template,
+ strategy_template.reservation_template.schema)
+ reservation = Reservation.objects.create(name=strategy_template.name,
+ description=" %s created from reservation strategy" % strategy_template.description,
+ project=None,
+ specifications_template=strategy_template.reservation_template,
+ specifications_doc=reservation_spec,
+ start_time=datetime.now()+timedelta(days=1),
+ stop_time=None)
+ logger.info('created test reservation: %s', reservation.name)
for scheduling_set in tmss_project.scheduling_sets.all():
for unit_nr in range(2):
@@ -133,36 +136,158 @@ def populate_test_data():
def populate_cycles(apps, schema_editor):
- for nr in range(0, 18):
+ # Cycle 0 deviates from any patterns
+ cycle = models.Cycle.objects.create(name="Cycle 00",
+ description="Lofar Cycle 0",
+ start=datetime(2013, 2, 11, 0, 0, 0, 0, tzinfo=timezone.utc),
+ stop=datetime(2013, 11, 14, 0, 0, 0, 0, tzinfo=timezone.utc))
+
+ models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time"),
+ value=0.8 * cycle.duration.total_seconds()),
+ # rough guess. 80% of total time available for observing
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="CEP Processing Time"),
+ value=0.8 * cycle.duration.total_seconds()),
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(name="LTA Storage"),
+ value=0), # needs to be filled in by user (SOS)
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Support Time"),
+ value=0), # needs to be filled in by user (SOS)
+
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time Commissioning"),
+ value=0.05 * cycle.duration.total_seconds()),
+ # rough guess. 5% of total time available for observing
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time prio A"),
+ value=0), # needs to be filled in by user (SOS)
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time prio B"),
+ value=0) # needs to be filled in by user (SOS)
+ ])
+
+ # Cycles 1-10 follow the same pattern
+ for nr in range(1, 11):
cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
description="Lofar Cycle %s" % nr,
- start=datetime(2013+nr//2, 6 if nr%2==0 else 11, 1, 0, 0, 0, 0, tzinfo=timezone.utc),
- stop=datetime(2013+(nr+1)//2, 6 if nr%2==1 else 11, 1, 0, 0, 0, 0, tzinfo=timezone.utc))
+ start=datetime(2013+nr//2, 5 if nr%2==0 else 11, 15, 0, 0, 0, 0, tzinfo=timezone.utc),
+ stop=datetime(2013+(nr+1)//2, 5 if nr%2==1 else 11, 14, 0, 0, 0, 0, tzinfo=timezone.utc))
models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="observing_time"),
+ resource_type=ResourceType.objects.get(name="LOFAR Observing Time"),
value=0.8*cycle.duration.total_seconds()), # rough guess. 80% of total time available for observing
models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="cep_processing_time"),
+ resource_type=ResourceType.objects.get(name="CEP Processing Time"),
value=0.8*cycle.duration.total_seconds()),
models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="lta_storage"),
+ resource_type=ResourceType.objects.get(name="LTA Storage"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="support_time"),
+ resource_type=ResourceType.objects.get(name="LOFAR Support Time"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="observing_time_commissioning"),
+ resource_type=ResourceType.objects.get(name="LOFAR Observing Time Commissioning"),
value=0.05*cycle.duration.total_seconds()), # rough guess. 5% of total time available for observing
models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="observing_time_prio_a"),
+ resource_type=ResourceType.objects.get(name="LOFAR Observing Time prio A"),
value=0), # needs to be filled in by user (SOS)
models.CycleQuota(cycle=cycle,
- resource_type=ResourceType.objects.get(name="observing_time_prio_b"),
+ resource_type=ResourceType.objects.get(name="LOFAR Observing Time prio B"),
value=0) # needs to be filled in by user (SOS)
])
+ # Cycle 11 deviates from any patterns
+ cycle = models.Cycle.objects.create(name="Cycle 11",
+ description="Lofar Cycle 11",
+ start=datetime(2018, 11, 15, 0, 0, 0, 0, tzinfo=timezone.utc),
+ stop=datetime(2019, 5, 31, 0, 0, 0, 0, tzinfo=timezone.utc))
+
+ models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time"),
+ value=0.8 * cycle.duration.total_seconds()),
+ # rough guess. 80% of total time available for observing
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="CEP Processing Time"),
+ value=0.8 * cycle.duration.total_seconds()),
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LTA Storage"),
+ value=0), # needs to be filled in by user (SOS)
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Support Time"),
+ value=0), # needs to be filled in by user (SOS)
+
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time Commissioning"),
+ value=0.05 * cycle.duration.total_seconds()),
+ # rough guess. 5% of total time available for observing
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time prio A"),
+ value=0), # needs to be filled in by user (SOS)
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time prio B"),
+ value=0) # needs to be filled in by user (SOS)
+ ])
+
+ # Cycles 12-19 follow the same pattern
+ for nr in range(12, 20):
+ cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
+ description="Lofar Cycle %s" % nr,
+ start=datetime(2013 + nr // 2, 6 if nr % 2 == 0 else 12, 1, 0, 0, 0, 0,
+ tzinfo=timezone.utc),
+ stop=datetime(2013 + (nr + 1) // 2, 5 if nr % 2 == 1 else 11,
+ 30 if nr % 2 == 0 else 31, 0, 0,
+ 0, 0, tzinfo=timezone.utc))
+
+ models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time"),
+ value=0.8 * cycle.duration.total_seconds()),
+ # rough guess. 80% of total time available for observing
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="CEP Processing Time"),
+ value=0.8 * cycle.duration.total_seconds()),
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LTA Storage"),
+ value=0), # needs to be filled in by user (SOS)
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Support Time"),
+ value=0), # needs to be filled in by user (SOS)
+
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time Commissioning"),
+ value=0.05 * cycle.duration.total_seconds()),
+ # rough guess. 5% of total time available for observing
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time prio A"),
+ value=0), # needs to be filled in by user (SOS)
+ models.CycleQuota(cycle=cycle,
+ resource_type=ResourceType.objects.get(
+ name="LOFAR Observing Time prio B"),
+ value=0) # needs to be filled in by user (SOS)
+ ])
+
+
def populate_projects(apps, schema_editor):
from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingSet_test_data
@@ -181,7 +306,7 @@ def populate_projects(apps, schema_editor):
# for convenience, create a schedulingset for each project
models.SchedulingSet.objects.create(**SchedulingSet_test_data(name="Test Scheduling Set", project=tmss_project))
- project_quota = ProjectQuota.objects.create(project=tmss_project, value=1e12, resource_type=ResourceType.objects.get(name="lta_storage"))
+ project_quota = ProjectQuota.objects.create(project=tmss_project, value=1e12, resource_type=ResourceType.objects.get(name="LTA Storage"))
sara_fs = Filesystem.objects.get(name="Lofar Storage (SARA)")
models.ProjectQuotaArchiveLocation.objects.create(project_quota=project_quota, archive_location=sara_fs)
@@ -191,18 +316,7 @@ def populate_resources(apps, schema_editor):
time_q = Quantity.objects.get(value=Quantity.Choices.TIME.value)
number_q = Quantity.objects.get(value=Quantity.Choices.NUMBER.value)
- ResourceType.objects.bulk_create([ResourceType(name="lta_storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
- ResourceType(name="cep_storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=bytes_q),
- ResourceType(name="cep_processing_time", description="Processing time on the CEP processing cluster (in seconds)", quantity=time_q),
- ResourceType(name="observing_time", description="Observing time (in seconds)", quantity=time_q),
- ResourceType(name="observing_time_prio_a", description="Observing time with priority A (in seconds)", quantity=time_q),
- ResourceType(name="observing_time_prio_b", description="Observing time with priority B (in seconds)", quantity=time_q),
- ResourceType(name="observing_time_commissioning", description="Observing time for Commissioning/DDT (in seconds)", quantity=time_q),
- ResourceType(name="support_time", description="Support time by human (in seconds)", quantity=time_q),
- ResourceType(name="number_of_triggers", description="Number of trigger events (as integer)", quantity=number_q),
- # TODO these duplicates have names that front-end expects.
- # TODO We should not have doubles.
- ResourceType(name="LTA Storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
+ ResourceType.objects.bulk_create([ResourceType(name="LTA Storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
ResourceType(name="CEP Storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=bytes_q),
ResourceType(name="CEP Processing Time", description="Processing time on the CEP processing cluster (in seconds)", quantity=time_q),
ResourceType(name="LOFAR Observing Time", description="Observing time (in seconds)", quantity=time_q),
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/reservations.py b/SAS/TMSS/backend/src/tmss/tmssapp/reservations.py
new file mode 100644
index 0000000000000000000000000000000000000000..3cc5cd8794191a8e2fc9ddd064e54dc120b97f42
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/reservations.py
@@ -0,0 +1,13 @@
+from lofar.sas.tmss.tmss.tmssapp import models
+
+
+def get_active_station_reservations_in_timewindow(lower_bound, upper_bound):
+ """
+ Retrieve a list of all active stations reservations, which are reserved between a timewindow
+ """
+ lst_active_station_reservations = []
+ for res in models.Reservation.objects.filter(start_time__lt=upper_bound, stop_time__gt=lower_bound).values('specifications_doc'):
+ lst_active_station_reservations += res["specifications_doc"]["resources"]["stations"]
+ for res in models.Reservation.objects.filter(start_time__lt=upper_bound, stop_time=None).values('specifications_doc'):
+ lst_active_station_reservations += res["specifications_doc"]["resources"]["stations"]
+ return list(set(lst_active_station_reservations))
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
index 9763e45af29ef026a061b6b801f3931a0f5b8302..07081c0e3098153f07f55d8078608ece8776bec7 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
@@ -9,8 +9,7 @@
"pointing": {
"direction_type": "J2000",
"angle1": 0,
- "angle2": 0,
- "angle3": 0
+ "angle2": 0
},
"name": "calibrator1"
},
@@ -80,8 +79,7 @@
"tile_beam": {
"direction_type": "J2000",
"angle1": 0.42,
- "angle2": 0.43,
- "angle3": 0.44
+ "angle2": 0.43
},
"SAPs": [
{
@@ -89,8 +87,7 @@
"digital_pointing": {
"direction_type": "J2000",
"angle1": 0.24,
- "angle2": 0.25,
- "angle3": 0.26
+ "angle2": 0.25
},
"subbands": [
349,
@@ -102,8 +99,7 @@
"digital_pointing": {
"direction_type": "J2000",
"angle1": 0.27,
- "angle2": 0.28,
- "angle3": 0.29
+ "angle2": 0.28
},
"subbands": [
349,
@@ -169,8 +165,7 @@
"pointing": {
"direction_type": "J2000",
"angle1": 0,
- "angle2": 0,
- "angle3": 0
+ "angle2": 0
},
"name": "calibrator2"
},
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json
index 88668838c82f03c889baee2825b7f8bf9823d3a4..75e850155bd192c799fc8e659516ac23c9ee2f2d 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json
@@ -42,12 +42,6 @@
"title": "Angle 2",
"description": "Second angle (e.g. DEC)",
"default": 0
- },
- "angle3": {
- "type": "number",
- "title": "Angle 3",
- "description": "Third angle (e.g. N in LMN)",
- "default": 0
}
},
"required": [
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/dataproduct_specifications_template-timeseries-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/dataproduct_specifications_template-timeseries-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d11ec11cc085263e455984410ad0f4e3dcc8e5ca
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/dataproduct_specifications_template-timeseries-1.json
@@ -0,0 +1,71 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/dataproductspecificationtemplate/timeseries/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "timeseries",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "sap": {
+ "type": "string",
+ "title": "SAP",
+ "default": ""
+ },
+ "identifiers": {
+ "title": "Identifiers",
+ "description": "Identification of this dataproduct within the producing subtask.",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "sap_index": {
+ "title": "SAP index",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0
+ },
+ "pipeline_index": {
+ "title": "TAB index",
+ "description": "Index of beamformer pipeline within COBALT",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0
+ },
+ "tab_index": {
+ "title": "TAB index",
+ "description": "TAB index within the SAP",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0
+ },
+ "part_index": {
+ "title": "Part index",
+ "description": "Part index within the TAB",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0
+ },
+ "stokes_index": {
+ "title": "Stokes index",
+ "description": "Stokes index within the TAB",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0,
+ "maximum": 3
+ },
+ "coherent": {
+ "title": "Coherent",
+ "description": "TAB is a coherent addition",
+ "type": "boolean",
+ "default": true
+ }
+ },
+ "required": [
+ "sap_index",
+ "tab_index",
+ "part_index",
+ "stokes_index",
+ "coherent"
+ ]
+ }
+ },
+ "required": [ "identifiers" ]
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/dataproduct_specifications_template-visibilities-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/dataproduct_specifications_template-visibilities-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..161f96803940afef59c4ceaf35787ad6012f5e66
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/dataproduct_specifications_template-visibilities-1.json
@@ -0,0 +1,22 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/dataproductspecificationstemplate/visibilities/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "visibilities",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "sap": {
+ "type": "string",
+ "title": "SAP",
+ "default": ""
+ },
+ "subband": {
+ "type": "integer",
+ "title": "subband number",
+ "default": 0,
+ "minimum": 0,
+ "maximum": 511
+ }
+ },
+ "required": [ "sap", "subband" ]
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-ILTswitch.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-ILTswitch.json
new file mode 100644
index 0000000000000000000000000000000000000000..73e493db102862eafe7a179489f7bac0631f605f
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-ILTswitch.json
@@ -0,0 +1,38 @@
+{
+ "activity": {
+ "type": "stand-alone mode",
+ "name": "ILT stations in local mode",
+ "description": "Planned switch of international stations for local use by station owners",
+ "contact": "Operator",
+ "subject": "system",
+ "planned": true
+ },
+ "resources": {
+ "stations": [
+ "DE601",
+ "DE602",
+ "DE603",
+ "DE604",
+ "DE605",
+ "DE609",
+ "FR606",
+ "SE607",
+ "UK608",
+ "PL610",
+ "PL611",
+ "PL612",
+ "IE613",
+ "LV614"
+ ]
+ },
+ "effects": {
+ "lba_rfi": false,
+ "hba_rfi": false,
+ "expert": false
+ },
+ "schedulability": {
+ "manual": false,
+ "dynamic": false,
+ "project_exclusive": false
+ }
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-VLBIsession.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-VLBIsession.json
new file mode 100644
index 0000000000000000000000000000000000000000..7c25f0f83ed1efb86bedcbf5803e0dd7b56eb59b
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-VLBIsession.json
@@ -0,0 +1,38 @@
+{
+ "activity": {
+ "type": "stand-alone mode",
+ "name": "VLBI session",
+ "description": "VLBI session ongoing. International station network not available.",
+ "contact": "Operator",
+ "subject": "network",
+ "planned": true
+ },
+ "resources": {
+ "stations": [
+ "DE601",
+ "DE602",
+ "DE603",
+ "DE604",
+ "DE605",
+ "DE609",
+ "FR606",
+ "SE607",
+ "UK608",
+ "PL610",
+ "PL611",
+ "PL612",
+ "IE613",
+ "LV614"
+ ]
+ },
+ "effects": {
+ "lba_rfi": false,
+ "hba_rfi": false,
+ "expert": false
+ },
+ "schedulability": {
+ "manual": false,
+ "dynamic": false,
+ "project_exclusive": false
+ }
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-core-stations.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-core-stations.json
new file mode 100644
index 0000000000000000000000000000000000000000..334ab09f6fdf28f42793add9565d0d38c2010fb7
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-core-stations.json
@@ -0,0 +1,47 @@
+{
+ "activity": {
+ "type": "maintenance",
+ "description": "Maintenance of all core stations",
+ "contact": "Operator",
+ "subject": "system",
+ "planned": true
+ },
+ "resources": {
+ "stations": [
+ "CS001",
+ "CS002",
+ "CS003",
+ "CS004",
+ "CS005",
+ "CS006",
+ "CS007",
+ "CS011",
+ "CS013",
+ "CS017",
+ "CS021",
+ "CS024",
+ "CS026",
+ "CS028",
+ "CS030",
+ "CS031",
+ "CS032",
+ "CS101",
+ "CS103",
+ "CS201",
+ "CS301",
+ "CS302",
+ "CS401",
+ "CS501"
+ ]
+ },
+ "effects": {
+ "lba_rfi": false,
+ "hba_rfi": false,
+ "expert": false
+ },
+ "schedulability": {
+ "manual": false,
+ "dynamic": false,
+ "project_exclusive": false
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-maintenance.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-maintenance.json
new file mode 100644
index 0000000000000000000000000000000000000000..cd938b2737ac725fc13c1d7db31f8e2aca1fd26c
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-maintenance.json
@@ -0,0 +1,24 @@
+{
+ "activity": {
+ "type": "maintenance",
+ "name": "Regular station maintenance",
+ "description": "Planned station maintenance",
+ "contact": "Operator",
+ "subject": "system",
+ "planned": true
+ },
+ "resources": {
+ "stations": [
+ ]
+ },
+ "effects": {
+ "lba_rfi": false,
+ "hba_rfi": false,
+ "expert": false
+ },
+ "schedulability": {
+ "manual": false,
+ "dynamic": false,
+ "project_exclusive": false
+ }
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-overheating.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-overheating.json
new file mode 100644
index 0000000000000000000000000000000000000000..c559225a8e5df256191f080bd8c7f3de3455c11c
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/reservation-strategy-overheating.json
@@ -0,0 +1,57 @@
+{
+ "activity": {
+ "type": "outage",
+ "name": "Station cool down",
+ "description": "Stations unavailable because of too high temperature",
+ "contact": "Operator",
+ "subject": "system",
+ "planned": true
+ },
+ "resources": {
+ "stations": [
+ "CS001",
+ "CS002",
+ "CS003",
+ "CS004",
+ "CS005",
+ "CS006",
+ "CS007",
+ "CS011",
+ "CS013",
+ "CS017",
+ "CS021",
+ "CS024",
+ "CS026",
+ "CS030",
+ "CS032",
+ "CS301",
+ "CS302",
+ "CS401",
+ "CS501",
+ "RS106",
+ "RS205",
+ "RS208",
+ "RS210",
+ "RS305",
+ "RS306",
+ "RS307",
+ "RS310",
+ "RS406",
+ "RS407",
+ "RS409",
+ "RS503",
+ "RS508",
+ "RS509"
+ ]
+ },
+ "effects": {
+ "lba_rfi": false,
+ "hba_rfi": false,
+ "expert": false
+ },
+ "schedulability": {
+ "manual": false,
+ "dynamic": false,
+ "project_exclusive": false
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json
index 9caf086d923d583720925e44d47dfbc255f95885..732e7c01dc4b52dab7e4bf0b55c0972de92ea8d4 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json
@@ -111,13 +111,13 @@
"properties": {
"from": {
"type": "number",
- "minimum": -0.20943951,
- "maximum": 0.20943951
+ "minimum": -86400,
+ "maximum": 86400
},
"to": {
"type": "number",
- "minimum": -0.20943951,
- "maximum": 0.20943951
+ "minimum": -86400,
+ "maximum": 86400
}
},
"additionalProperties": false
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json
index 23756bfb22a1d883279f592b8c01d7b453847e7b..ac3277566c7e385713036301a3c2a6af7bd3c911 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json
@@ -30,8 +30,7 @@
"tile_beam": {
"direction_type": "J2000",
"angle1": 5.233660650313663,
- "angle2": 0.7109404782526458,
- "angle3": 0
+ "angle2": 0.7109404782526458
},
"SAPs": [
{
@@ -40,8 +39,7 @@
"digital_pointing": {
"direction_type": "J2000",
"angle1": 5.233660650313663,
- "angle2": 0.7109404782526458,
- "angle3": 0
+ "angle2": 0.7109404782526458
},
"subbands": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243]
}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-beamforming-observation-scheduling-unit-observation-strategy.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-beamforming-observation-scheduling-unit-observation-strategy.json
index a711c97f18d6bb9f3facbe17f4f5f1a15e41f423..f74ee652b3c73ffbedb2451edce6531cf93f8990 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-beamforming-observation-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-beamforming-observation-scheduling-unit-observation-strategy.json
@@ -14,8 +14,7 @@
"digital_pointing": {
"direction_type": "J2000",
"angle1": 5.233660650313663,
- "angle2": 0.7109404782526458,
- "angle3": 0
+ "angle2": 0.7109404782526458
},
"subbands": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243]
}
@@ -26,8 +25,7 @@
"tile_beam": {
"direction_type": "J2000",
"angle1": 5.233660650313663,
- "angle2": 0.7109404782526458,
- "angle3": 0
+ "angle2": 0.7109404782526458
},
"beamformers": [ {} ]
},
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json
index 06e34636cc16cd68ccfc10c68b3b7180634d9070..4ea17e719fad83f17b9746f474f1761f9682a48f 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json
@@ -30,8 +30,7 @@
"tile_beam": {
"direction_type": "J2000",
"angle1": 5.233660650313663,
- "angle2": 0.7109404782526458,
- "angle3": 0
+ "angle2": 0.7109404782526458
},
"SAPs": [
{
@@ -40,8 +39,7 @@
"digital_pointing": {
"direction_type": "J2000",
"angle1": 5.233660650313663,
- "angle2": 0.7109404782526458,
- "angle3": 0
+ "angle2": 0.7109404782526458
},
"subbands": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243]
}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-observation-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
index 985274ec00ccab31533717ae489dee21ad4a6b14..3555487e83beaf29a2c66bab6f7327c4cf6cee99 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
@@ -98,7 +98,7 @@
"type":"integer",
"title":"Specification version",
"description":"Version of the COBALT specification to emit",
- "default":2,
+ "default":1,
"minimum":1,
"maximum":2
},
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json
index 480d7a4abb715673befa1742ef8fedb6ac04a00f..33140a263020d32e0b1d705713bc7368d7844183 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json
@@ -35,6 +35,14 @@
"file_name": "dataproduct_specifications_template-empty-1.json",
"template": "dataproduct_specifications_template"
},
+ {
+ "file_name": "dataproduct_specifications_template-timeseries-1.json",
+ "template": "dataproduct_specifications_template"
+ },
+ {
+ "file_name": "dataproduct_specifications_template-visibilities-1.json",
+ "template": "dataproduct_specifications_template"
+ },
{
"file_name": "dataproduct_feedback_template-empty-1.json",
"template": "dataproduct_feedback_template"
@@ -167,5 +175,51 @@
{
"file_name": "reservation_template-reservation-1.json",
"template": "reservation_template"
+ },
+ {
+ "file_name": "reservation-strategy-core-stations.json",
+ "template": "reservation_strategy_template",
+ "reservation_template_name": "reservation",
+ "reservation_template_version": "1",
+ "name": "Simple Core Reservation",
+ "description": "This reservation strategy template defines a reservation of all core station for system maintenance.",
+ "version": 1
+ },
+ {
+ "file_name": "reservation-strategy-ILTswitch.json",
+ "template": "reservation_strategy_template",
+ "reservation_template_name": "reservation",
+ "reservation_template_version": "1",
+ "name": "ILT stations in local mode",
+ "description": "Planned switch of international stations for local use by station owners",
+ "version": 1
+ },
+ {
+ "file_name": "reservation-strategy-maintenance.json",
+ "template": "reservation_strategy_template",
+ "reservation_template_name": "reservation",
+ "reservation_template_version": "1",
+ "name": "Regular station maintenance",
+ "description": "Planned station maintenance",
+ "version": 1
+ },
+ {
+ "file_name": "reservation-strategy-overheating.json",
+ "template": "reservation_strategy_template",
+ "reservation_template_name": "reservation",
+ "reservation_template_version": "1",
+ "name": "Station cool down",
+ "description": "Stations unavailable because of too high temperature",
+ "version": 1
+ },
+ {
+ "file_name": "reservation-strategy-VLBIsession.json",
+ "template": "reservation_strategy_template",
+ "reservation_template_name": "reservation",
+ "reservation_template_version": "1",
+ "name": "VLBI session",
+ "description": "VLBI session ongoing. International station network not available.",
+ "version": 1
}
+
]
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py
index 650d1f6816667256d074ac1994ab3c37a8727d37..47086104958108a4cc364a1c07c84c200d909d64 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py
@@ -171,10 +171,12 @@ class ProjectSerializer(DynamicRelationalHyperlinkedModelSerializer):
class ProjectQuotaSerializer(DynamicRelationalHyperlinkedModelSerializer):
+ project_quota_archive_location = serializers.HyperlinkedRelatedField('projectquotaarchivelocation-detail', source='*', read_only=True)
+
class Meta:
model = models.ProjectQuota
fields = '__all__'
- extra_fields = ['resource_type']
+ extra_fields = ['resource_type', 'project_quota_archive_location']
class ProjectQuotaArchiveLocationSerializer(DynamicRelationalHyperlinkedModelSerializer):
@@ -362,6 +364,14 @@ class TaskTypeSerializer(DynamicRelationalHyperlinkedModelSerializer):
fields = '__all__'
+class ReservationStrategyTemplateSerializer(DynamicRelationalHyperlinkedModelSerializer):
+ template = JSONEditorField(schema_source="reservation_template.schema")
+
+ class Meta:
+ model = models.ReservationStrategyTemplate
+ fields = '__all__'
+
+
class ReservationTemplateSerializer(AbstractTemplateSerializer):
class Meta:
model = models.ReservationTemplate
@@ -380,7 +390,7 @@ class ReservationSerializer(DynamicRelationalHyperlinkedModelSerializer):
class Meta:
model = models.Reservation
fields = '__all__'
- extra_fields = ['stop_time']
+ extra_fields = ['duration']
class TaskBlueprintExtendedSerializer(TaskBlueprintSerializer):
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
index 243cb8b3ddbc8729a94b61606a3a7a4c93b5be42..856c523be56c5a471099ab484f6eb04412b678a8 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
@@ -4,12 +4,14 @@ logger = logging.getLogger(__name__)
from copy import deepcopy
from functools import cmp_to_key
from collections.abc import Iterable
+from math import ceil
from lofar.common.ring_coordinates import RingCoordinates
from lofar.common.datetimeutils import formatDatetime, round_to_second_precision
from lofar.common import isProductionEnvironment
from lofar.common.json_utils import add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
from lofar.common.lcu_utils import get_current_stations
+from lofar.stationmodel.antennafields import antenna_fields
from lofar.sas.tmss.tmss.exceptions import SubtaskCreationException, SubtaskSchedulingException, SubtaskException
@@ -20,7 +22,7 @@ from lofar.sas.tmss.tmss.tmssapp.models import *
from lofar.sas.resourceassignment.resourceassigner.rarpc import RARPC
from lofar.sas.resourceassignment.resourceassignmentservice.rpc import RADBRPC
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset_dict
-from lofar.sas.resourceassignment.taskprescheduler.cobaltblocksize import CorrelatorSettings, BlockConstraints, BlockSize
+from lofar.sas.resourceassignment.taskprescheduler.cobaltblocksize import CorrelatorSettings, StokesSettings, BlockConstraints, BlockSize
from lofar.sas.resourceassignment.resourceassigner.schedulers import ScheduleException
from lofar.sas.tmss.tmss.tmssapp.conversions import antennafields_for_antennaset_and_station
@@ -73,8 +75,9 @@ def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subta
task_blueprint.id, task_blueprint.name, task_blueprint.specifications_template.type.value,
task_blueprint.scheduling_unit_blueprint.id)
subtasks.append(subtask)
- except SubtaskCreationException as e:
- logger.error(e)
+ except Exception as e:
+ logger.exception(e)
+ raise SubtaskCreationException('Cannot create subtasks for task id=%s for its schema name=%s in generator %s' % (task_blueprint.pk, template_name, generator)) from e
return subtasks
else:
logger.error('Cannot create subtasks for task id=%s because no generator exists for its schema name=%s' % (task_blueprint.pk, template_name))
@@ -99,7 +102,7 @@ def _add_pointings(pointing_a, pointing_b):
raise SubtaskCreationException(
"Cannot add pointings because direction types differ pointing_a=%s; pointing_b=%s" % (pointing_a, pointing_b))
pointing = {"direction_type": pointing_a['direction_type']}
- for angle in ['angle1', 'angle2', 'angle3']:
+ for angle in ['angle1', 'angle2']:
pointing[angle] = pointing_a.get(angle, 0.0) + pointing_b.get(angle, 0.0)
return pointing
@@ -152,6 +155,9 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
# now go over the settings in the task_spec and 'copy'/'convert' them to the subtask_spec
task_spec = task_blueprint.specifications_doc
+ # block size calculator will need to be fed all the relevant specs
+ cobalt_calculator_constraints = BlockConstraints(None, [], [])
+
# The calibrator has a minimal calibration-specific specification subset.
# The rest of it's specs are 'shared' with the target observation.
# So... copy the calibrator specs first, then loop over the shared target/calibrator specs...
@@ -195,6 +201,17 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
logger.info("Using station and correlator settings for calibrator observation task_blueprint id=%s from target observation task_blueprint id=%s",
task_blueprint.id, target_task_blueprint.id)
+ # correlator
+ subtask_spec["COBALT"]["correlator"] = { "enabled": False }
+
+ if "correlator" in task_spec:
+ subtask_spec["COBALT"]["correlator"]["enabled"] = True
+ subtask_spec["COBALT"]["correlator"]["channels_per_subband"] = task_spec["correlator"]["channels_per_subband"]
+
+ corr = CorrelatorSettings()
+ corr.nrChannelsPerSubband = task_spec["correlator"]["channels_per_subband"]
+ corr.integrationTime = task_spec["correlator"]["integration_time"]
+ cobalt_calculator_constraints.correlator = corr
# At this moment of subtask creation we known which stations we *want* from the task_spec
# But we do not know yet which stations are available at the moment of observing.
@@ -215,70 +232,89 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
# The beamformer obs has a beamformer-specific specification block.
# The rest of it's specs is the same as in a target observation.
# So... copy the beamformer specs first, then loop over the shared specs...
- if 'beamforming' in task_blueprint.specifications_template.name.lower():
+ if 'beamformers' in task_spec:
subtask_spec['COBALT']['beamformer']['tab_pipelines'] = []
subtask_spec['COBALT']['beamformer']['flyseye_pipelines'] = []
- if 'beamformers' in task_spec:
- for task_beamformer_spec in task_spec['beamformers']:
- task_beamformer_spec = deepcopy(task_beamformer_spec)
-
- # the wanted/specified beamformer station list is the intersecion of the observation station list with the requested beamformer stations.
- # at the moment of scheduling this list is re-evaluated for available stations, and the max_nr_missing is evaluated as well.
- # this intersection is not needed per se, because COBALT plays nicely and does similar filtering for stations that are actually available,
- # but hey, if cobalt can play nice, then so can we! :)
- # So, let's come up with the correct complete beamforming-stations-list, and ask cobalt to explicitely uses these.
- beamformer_station_list = []
- if "station_groups" in task_beamformer_spec:
- # combine all stations in the groups...
- for station_group in task_beamformer_spec["station_groups"]:
- beamformer_station_list.extend(station_group["stations"])
-
- # make intersection with observing-stations...
- beamformer_station_set = set(beamformer_station_list).intersection(set(subtask_spec['stations']['station_list']))
-
- # make it a nice readable sorted list.
- beamformer_station_list = sorted(list(beamformer_station_list))
- # use the beamformer_station_list below for the tab pipeline and/or flys eye
-
- for stokes_type in ["coherent", "incoherent"]:
- if stokes_type in task_beamformer_spec:
- # SAPs
- saps = task_beamformer_spec[stokes_type]["SAPs"]
- for sap in saps:
- # determine absolute tab pointing for subtask by adding relative tab pointing from task to target sap pointing
- target_sap = _get_related_target_sap_by_name(task_blueprint, sap['name'])
- if "tabs" in sap:
- for tab in sap["tabs"]:
- tab['coherent'] = (stokes_type == "coherent")
- if "relative" in tab:
- if tab.pop("relative"):
- tab['pointing'] = _add_pointings(tab['pointing'], target_sap['digital_pointing'])
- elif stokes_type == 'incoherent':
- sap.setdefault('tabs', [])
- sap["tabs"] += [{'coherent': False}] # todo: according to confluence. Is that needed?
- if "tab_rings" in sap:
- ring_pointings = _generate_tab_ring_pointings(target_sap["digital_pointing"], sap.pop("tab_rings"))
- sap['tabs'] += [{'coherent': (stokes_type == "coherent"), 'pointing': pointing} for pointing in ring_pointings]
- if "subbands" in sap:
- sap['subbands'] = _filter_subbands(target_sap['subbands'], sap['subbands'])
-
- # create a pipeline item and add it to the list
- beamformer_pipeline = {stokes_type: task_beamformer_spec[stokes_type]["settings"],
- "stations": beamformer_station_list,
- "SAPs": saps}
- subtask_spec['COBALT']['beamformer']['tab_pipelines'].append(beamformer_pipeline)
- if task_beamformer_spec['flys eye'].get("enabled", False):
- flyseye_pipeline = {"coherent": task_beamformer_spec["flys eye"]["settings"],
- "stations": beamformer_station_list}
- subtask_spec['COBALT']['beamformer']['flyseye_pipelines'].append(flyseye_pipeline)
- # todo: Clarify if we can add a subbands_selection on the flys eye task spec, to filter down for sap['subbands']
- # If I got that correctly, specifying subbands is not really supported later down the chain, so whatever we do here gets ignored anyway?
- # for sap in task_spec["SAPs"]:
- # target_sap = _get_related_target_sap_by_name(task_blueprint, sap['name'])
- # sap['subbands'] = filter_subbands(...)
- # if sap['subbands'] == target_sap['subbands']: # todo: is this really required? pseudo-code in confluence suggests so, but what harm does the list do?
- # sap['subbands'] = []
+ for task_beamformer_spec in task_spec['beamformers']:
+ # the wanted/specified beamformer station list is the intersecion of the observation station list with the requested beamformer stations.
+ # at the moment of scheduling this list is re-evaluated for available stations, and the max_nr_missing is evaluated as well.
+ # this intersection is not needed per se, because COBALT plays nicely and does similar filtering for stations that are actually available,
+ # but hey, if cobalt can play nice, then so can we! :)
+ # So, let's come up with the correct complete beamforming-stations-list, and ask cobalt to explicitely uses these.
+
+ # combine all stations in the groups...
+ beamformer_station_list = sum([station_group["stations"] for station_group in task_beamformer_spec["station_groups"]], [])
+
+ # make intersection with observing-stations...
+ beamformer_station_set = set(beamformer_station_list).intersection(set(subtask_spec['stations']['station_list']))
+
+ # make it a nice readable sorted list.
+ beamformer_station_list = sorted(list(beamformer_station_list))
+ # use the beamformer_station_list below for the tab pipeline and/or flys eye
+
+ for stokes_type in ["coherent", "incoherent"]:
+ if not task_beamformer_spec[stokes_type]["SAPs"]:
+ # nothing specified for this stokes type
+ continue
+
+ # SAPs
+ subtask_saps = []
+ for sap in task_beamformer_spec[stokes_type]["SAPs"]:
+ subtask_sap = { "name": sap["name"], "tabs": [] }
+
+ target_sap = _get_related_target_sap_by_name(task_blueprint, sap['name'])
+ if stokes_type == "coherent":
+ for tab in sap["tabs"]:
+ subtask_sap["tabs"].append({
+ "coherent": True,
+ # determine absolute tab pointing for subtask by adding relative tab pointing from task to target sap pointing
+ "pointing": tab["pointing"] if not tab.get("relative", False) else _add_pointings(tab['pointing'], target_sap['digital_pointing'])
+ })
+
+ if "tab_rings" in sap:
+ ring_pointings = _generate_tab_ring_pointings(target_sap["digital_pointing"], sap.pop("tab_rings"))
+ subtask_sap['tabs'] += [{'coherent': True, 'pointing': pointing} for pointing in ring_pointings]
+ else:
+ subtask_sap["tabs"] = [{"coherent": False}]
+
+ if "subbands" in sap:
+ sap['subbands'] = _filter_subbands(target_sap['subbands'], sap['subbands'])
+
+ subtask_saps.append(subtask_sap)
+
+ # create a pipeline item and add it to the list
+ beamformer_pipeline = {stokes_type: task_beamformer_spec[stokes_type]["settings"],
+ "stations": beamformer_station_list,
+ "SAPs": subtask_saps}
+ subtask_spec['COBALT']['beamformer']['tab_pipelines'].append(beamformer_pipeline)
+
+ # add constraints for calculator
+ ss = StokesSettings()
+ ss.nrChannelsPerSubband = task_beamformer_spec[stokes_type]["settings"]["channels_per_subband"]
+ ss.timeIntegrationFactor = task_beamformer_spec[stokes_type]["settings"]["time_integration_factor"]
+ if stokes_type == "coherent":
+ cobalt_calculator_constraints.coherentStokes.append(ss)
+ else:
+ cobalt_calculator_constraints.incoherentStokes.append(ss)
+
+ if task_beamformer_spec['flys eye']['enabled']:
+ # add constraints for calculator
+ ss = StokesSettings()
+ ss.nrChannelsPerSubband = task_beamformer_spec["flys eye"]["settings"]["channels_per_subband"]
+ ss.timeIntegrationFactor = task_beamformer_spec["flys eye"]["settings"]["time_integration_factor"]
+ cobalt_calculator_constraints.coherentStokes.append(ss)
+
+ flyseye_pipeline = {"coherent": task_beamformer_spec["flys eye"]["settings"],
+ "stations": beamformer_station_list}
+ subtask_spec['COBALT']['beamformer']['flyseye_pipelines'].append(flyseye_pipeline)
+ # todo: Clarify if we can add a subbands_selection on the flys eye task spec, to filter down for sap['subbands']
+ # If I got that correctly, specifying subbands is not really supported later down the chain, so whatever we do here gets ignored anyway?
+ # for sap in task_spec["SAPs"]:
+ # target_sap = _get_related_target_sap_by_name(task_blueprint, sap['name'])
+ # sap['subbands'] = filter_subbands(...)
+ # if sap['subbands'] == target_sap['subbands']: # todo: is this really required? pseudo-code in confluence suggests so, but what harm does the list do?
+ # sap['subbands'] = []
subtask_spec['stations']["antenna_set"] = task_spec["antenna_set"]
subtask_spec['stations']["filter"] = task_spec["filter"]
@@ -301,15 +337,15 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
"angle1": task_spec["tile_beam"]["angle1"],
"angle2": task_spec["tile_beam"]["angle2"] }
+
+
+ # Calculate block sizes and feed those to the spec
+ cobalt_calculator = BlockSize(constraints=cobalt_calculator_constraints)
+ subtask_spec["COBALT"]["blocksize"] = cobalt_calculator.blockSize
+
if "correlator" in task_spec:
- corr = CorrelatorSettings()
- corr.nrChannelsPerSubband = task_spec["correlator"]["channels_per_subband"]
- corr.integrationTime = task_spec["correlator"]["integration_time"]
- calculator = BlockSize(constraints=BlockConstraints(correlatorSettings=corr))
- subtask_spec["COBALT"]["correlator"] = {}
- subtask_spec["COBALT"]["correlator"]["enabled"] = True
- subtask_spec["COBALT"]["correlator"]["blocks_per_integration"] = calculator.nrBlocks
- subtask_spec["COBALT"]["correlator"]["integrations_per_block"] = calculator.nrSubblocks
+ subtask_spec["COBALT"]["correlator"]["blocks_per_integration"] = cobalt_calculator.nrBlocks
+ subtask_spec["COBALT"]["correlator"]["integrations_per_block"] = cobalt_calculator.nrSubblocks
# make sure that the subtask_spec is valid conform the schema
validate_json_against_schema(subtask_spec, subtask_template.schema)
@@ -472,6 +508,18 @@ def create_qafile_subtask_from_observation_subtask(observation_subtask: Subtask)
def create_qaplots_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) -> Subtask:
+ if 'calibrator' in task_blueprint.specifications_template.name.lower():
+ # Calibrator requires related Target Task Observation for some specifications
+ target_task_blueprint = get_related_target_observation_task_blueprint(task_blueprint)
+ if target_task_blueprint is None:
+ raise SubtaskCreationException("Cannot retrieve specifications for task id=%d because no related target observation is found " % task.pk)
+ else:
+ target_task_blueprint = task_blueprint
+
+ if not target_task_blueprint.specifications_doc.get("QA", {}).get("file_conversion", {}).get("enabled", False):
+ logger.debug("Skipping creation of qaplots_subtask because QA.file_conversion is not enabled")
+ return None
+
qafile_subtasks = [st for st in task_blueprint.subtasks.all() if st.specifications_template.type.value == SubtaskType.Choices.QA_FILES.value]
if qafile_subtasks:
qafile_subtask = qafile_subtasks[0] # TODO: decide what to do when there are multiple qafile subtasks?
@@ -673,7 +721,7 @@ def schedule_subtask(subtask: Subtask) -> Subtask:
logger.error(e2)
finally:
# ... and re-raise the original exception (wrapped)
- raise SubtaskSchedulingException("Error while scheduling subtask id=%d: %s" % (subtask.pk, str(e)))
+ raise SubtaskSchedulingException("Error while scheduling subtask id=%d" % (subtask.pk,)) from e
def unschedule_subtask(subtask: Subtask) -> Subtask:
@@ -1100,48 +1148,117 @@ def schedule_observation_subtask(observation_subtask: Subtask):
# TODO: are there any observations that take input dataproducts?
# step 3: create output dataproducts, and link these to the output
+ dataproducts = []
specifications_doc = observation_subtask.specifications_doc
- dataproduct_specifications_template = DataproductSpecificationsTemplate.objects.get(name="SAP") # todo: should this be derived from the task relation specification template?
dataproduct_feedback_template = DataproductFeedbackTemplate.objects.get(name="empty")
subtask_output = observation_subtask.outputs.first() # TODO: make proper selection, not default first()
- directory = "/data/%s/%s/L%s/uv" % ("projects" if isProductionEnvironment() else "test-projects",
- observation_subtask.task_blueprint.scheduling_unit_blueprint.draft.scheduling_set.project.name,
- observation_subtask.id)
-
- for sap_nr, pointing in enumerate(specifications_doc['stations']['digital_pointings']):
- antennaset = specifications_doc['stations']['antenna_set']
- antennafields = []
- for station in specifications_doc['stations']['station_list']:
- fields = antennafields_for_antennaset_and_station(antennaset, station)
- antennafields += [{"station": station, "field": field, "type": antennaset.split('_')[0]} for field in fields]
-
- sap = SAP.objects.create(specifications_doc={ "name": "%s_%s" % (observation_subtask.id, pointing['name']),
- "pointing": pointing['pointing'],
- "time": {"start_time": observation_subtask.start_time.isoformat(),
- "duration": (observation_subtask.stop_time - observation_subtask.start_time).total_seconds()},
- "antennas": {
+
+ # create SAP objects, as observations create new beams
+ antennaset = specifications_doc['stations']['antenna_set']
+ antennafields = []
+ for station in specifications_doc['stations']['station_list']:
+ fields = antennafields_for_antennaset_and_station(antennaset, station)
+ antennafields += [{"station": station, "field": field, "type": antennaset.split('_')[0]} for field in fields]
+
+ saps = [SAP.objects.create(specifications_doc={ "name": "%s_%s" % (observation_subtask.id, pointing['name']),
+ "pointing": pointing['pointing'],
+ "time": {"start_time": observation_subtask.start_time.isoformat(),
+ "duration": (observation_subtask.stop_time - observation_subtask.start_time).total_seconds()},
+ "antennas": {
"antenna_set": antennaset,
"fields": antennafields
- }
- },
- specifications_template=SAPTemplate.objects.get(name="SAP"))
-
- # create dataproducts in bulk, and assign each dp its own unique global identifier
- dp_global_identifiers = SIPidentifier.objects.bulk_create([SIPidentifier(source="TMSS") for _ in pointing['subbands']])
- Dataproduct.objects.bulk_create([Dataproduct(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
- directory=directory,
- dataformat=Dataformat.objects.get(value="MeasurementSet"),
- datatype=Datatype.objects.get(value="visibilities"),
- producer=subtask_output,
- specifications_doc={"sap": [str(sap_nr)]},
- specifications_template=dataproduct_specifications_template,
- feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
- feedback_template=dataproduct_feedback_template,
- size=0 if sb_nr%10==0 else 1024*1024*1024*sb_nr,
- expected_size=1024*1024*1024*sb_nr,
- sap=sap,
- global_identifier=dp_global_identifier)
- for sb_nr, dp_global_identifier in zip(pointing['subbands'], dp_global_identifiers)])
+ }
+ },
+ specifications_template=SAPTemplate.objects.get(name="SAP")) for sap_nr, pointing in enumerate(specifications_doc['stations']['digital_pointings'])]
+
+ # store everything below this directory
+ directory = "/data/%s/%s/L%s" % ("projects" if isProductionEnvironment() else "test-projects",
+ observation_subtask.task_blueprint.scheduling_unit_blueprint.draft.scheduling_set.project.name,
+ observation_subtask.id)
+
+ # create correlated dataproducts
+ if specifications_doc['COBALT']['correlator']['enabled']:
+ dataproduct_specifications_template_visibilities = DataproductSpecificationsTemplate.objects.get(name="visibilities")
+ sb_nr_offset = 0 # subband numbers run from 0 to (nr_subbands-1), increasing across SAPs
+
+ for sap_nr, pointing in enumerate(specifications_doc['stations']['digital_pointings']):
+ for sb_nr, subband in enumerate(pointing['subbands'], start=sb_nr_offset):
+ dataproducts.append(Dataproduct(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
+ directory=directory+"/uv",
+ dataformat=Dataformat.objects.get(value="MeasurementSet"),
+ datatype=Datatype.objects.get(value="visibilities"),
+ producer=subtask_output,
+ specifications_doc={"sap": pointing["name"], "subband": subband},
+ specifications_template=dataproduct_specifications_template_visibilities,
+ feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
+ feedback_template=dataproduct_feedback_template,
+ size=0,
+ expected_size=1024*1024*1024*sb_nr,
+ sap=saps[sap_nr],
+ global_identifier=None))
+
+ sb_nr_offset += len(pointing['subbands'])
+
+
+ # create beamformer dataproducts
+ dataproduct_specifications_template_timeseries = DataproductSpecificationsTemplate.objects.get(name="timeseries")
+
+ def _sap_index(saps: dict, sap_name: str) -> int:
+ """ Return the SAP index in the observation given a certain SAP name. """
+
+ sap_indices = [idx for idx,sap in enumerate(saps) if sap['name'] == sap_name]
+
+ # needs to be exactly one hit
+ if len(sap_indices) != 1:
+ raise SubtaskSchedulingException("SAP name %s must appear exactly once in the specification. It appeared %d times. Available names: %s" % (sap_name, len(sap_indices), [sap['name'] for sap in saps]))
+
+ return sap_indices[0]
+
+ def tab_dataproducts(sap_nr, pipeline_nr, tab_nr, stokes_settings, coherent):
+ nr_subbands = len(sap['subbands']) or len(specifications_doc['stations']['digital_pointings'][sap_nr]['subbands'])
+ nr_stokes = len(stokes_settings['stokes'])
+ nr_parts = ceil(1.0 * nr_subbands / stokes_settings['subbands_per_file'])
+
+ return [Dataproduct(filename="L%d_SAP%03d_N%03d_B%03d_S%03d_P%03d_bf.h5" % (observation_subtask.id, sap_nr, pipeline_nr, tab_nr, stokes_nr, part_nr),
+ directory=directory+("/cs" if coherent else "/is"),
+ dataformat=Dataformat.objects.get(value="Beamformed"),
+ datatype=Datatype.objects.get(value="time series"),
+ producer=subtask_output,
+ specifications_doc={"sap": specifications_doc['stations']['digital_pointings'][sap_nr]["name"], "coherent": coherent, "identifiers": {"pipeline_index": pipeline_nr, "tab_index": tab_nr, "stokes_index": stokes_nr, "part_index": part_nr}},
+ specifications_template=dataproduct_specifications_template_timeseries,
+ feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
+ feedback_template=dataproduct_feedback_template,
+ size=0,
+ expected_size=1024*1024*1024*tab_nr,
+ sap=saps[sap_nr],
+ global_identifier=None)
+ for part_nr in range(nr_parts) for stokes_nr in range(nr_stokes)]
+
+
+ # beamformer pipelines: one set of dataproducts per TAB.
+ pipeline_nr_offset = 0
+ for pipeline_nr, pipeline in enumerate(specifications_doc['COBALT']['beamformer']['tab_pipelines'], start=pipeline_nr_offset):
+ for sap in pipeline['SAPs']:
+ sap_idx = _sap_index(specifications_doc['stations']['digital_pointings'], sap['name'])
+
+ for tab_idx, tab in enumerate(sap['tabs']):
+ dataproducts += tab_dataproducts(sap_idx, pipeline_nr, tab_idx, pipeline['coherent'] if tab['coherent'] else pipeline['incoherent'], tab['coherent'])
+
+ # fly's eye pipelines: one set of dataproducts per antenna field.
+ pipeline_nr_offset += len(specifications_doc['COBALT']['beamformer']['tab_pipelines'])
+ for pipeline_nr, pipeline in enumerate(specifications_doc['COBALT']['beamformer']['flyseye_pipelines'], start=pipeline_nr_offset):
+ for sap_idx, sap in enumerate(specifications_doc['stations']['digital_pointings']):
+ stations = pipeline['stations'] or specifications_doc['stations']['station_list']
+ fields = sum([list(antenna_fields(station, antennaset)) for station in stations], [])
+ for tab_idx, tab in enumerate(fields):
+ dataproducts += tab_dataproducts(sap_idx, pipeline_nr, tab_idx, pipeline['coherent'], True)
+
+ # Bulk create identifiers, and then update the dataproducts with a link to the actual created objects.
+ # This is needed as bulk_create needs to have any relations resolved.
+ dp_global_identifiers = SIPidentifier.objects.bulk_create([SIPidentifier(source="TMSS") for _ in dataproducts])
+ for dp, global_identifier in zip(dataproducts, dp_global_identifiers):
+ dp.global_identifier = global_identifier
+ Dataproduct.objects.bulk_create(dataproducts)
# step 4: resource assigner (if possible)
assign_or_unassign_resources(observation_subtask)
@@ -1194,7 +1311,7 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
pipeline_subtask.specifications_template.type))
# TODO: use existing and reasonable selection and specification templates for output when we have those, for now, use "empty"
- dataproduct_specifications_template = DataproductSpecificationsTemplate.objects.get(name="empty")
+ dataproduct_specifications_template = DataproductSpecificationsTemplate.objects.get(name="visibilities")
dataproduct_feedback_template = DataproductFeedbackTemplate.objects.get(name="empty")
# iterate over all inputs
@@ -1225,7 +1342,7 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
dataformat=dataformat,
datatype=Datatype.objects.get(value="visibilities"), # todo: is this correct?
producer=pipeline_subtask_output,
- specifications_doc=get_default_json_object_for_schema(dataproduct_specifications_template.schema),
+ specifications_doc=input_dp.specifications_doc,
specifications_template=dataproduct_specifications_template,
feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
feedback_template=dataproduct_feedback_template,
@@ -1485,10 +1602,13 @@ def specifications_doc_meets_selection_doc(specifications_doc, selection_doc):
meets_criteria = False
else:
spec = specifications_doc[k]
- if isinstance(spec, Iterable) and isinstance(v, Iterable):
+ if isinstance(spec, list) and isinstance(v, list):
for spec_v in spec:
if spec_v not in v:
meets_criteria = False
+ elif isinstance(v, list):
+ if spec not in v:
+ meets_criteria = False
else:
if spec != v:
meets_criteria = False
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
index 9605ead221a9ae4a18596d0c6d887b4ad2791bc2..d53ace784b028f01ba199a80e067090526a66a41 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
@@ -37,6 +37,7 @@ from rest_framework.filters import OrderingFilter
import json
import logging
+import dateutil
from django.core.exceptions import ObjectDoesNotExist
@@ -199,6 +200,66 @@ class DefaultTaskRelationSelectionTemplateViewSet(LOFARViewSet):
serializer_class = serializers.DefaultTaskRelationSelectionTemplateSerializer
+class ReservationStrategyTemplateViewSet(LOFARViewSet):
+ queryset = models.ReservationStrategyTemplate.objects.all()
+ serializer_class = serializers.ReservationStrategyTemplateSerializer
+
+ @swagger_auto_schema(responses={status.HTTP_201_CREATED: 'The newly created reservation',
+ status.HTTP_403_FORBIDDEN: 'forbidden'},
+ operation_description="Create a new Reservation based on this ReservationStrategyTemplate, "
+ "with the given <name>, <description>, <start_time> and <stop_time>",
+ manual_parameters=[Parameter(name='start_time', required=True, type='string', in_='query',
+ description="The start time as a timestamp string in isoformat"),
+ Parameter(name='stop_time', required=True, type='string', in_='query',
+ description="The stop time as a timestamp string in isoformat"),
+ Parameter(name='name', required=False, type='string', in_='query',
+ description="The name for the newly created reservation"),
+ Parameter(name='description', required=False, type='string', in_='query',
+ description="The description for the newly created reservation"),
+ Parameter(name='project_id', required=False, type='integer', in_='query',
+ description="the id of the project which will be the parent of the newly created reservation"),
+ ])
+ @action(methods=['get'], detail=True)
+ def create_reservation(self, request, pk=None):
+ strategy_template = get_object_or_404(models.ReservationStrategyTemplate, pk=pk)
+ reservation_template_spec = add_defaults_to_json_object_for_schema(strategy_template.template,
+ strategy_template.reservation_template.schema)
+
+ start_time = request.query_params.get('start_time', None)
+ stop_time = request.query_params.get('stop_time', None)
+ if start_time:
+ start_time = dateutil.parser.parse(start_time) # string to datetime
+ else:
+ start_time = datetime.now()
+ if stop_time:
+ stop_time = dateutil.parser.parse(stop_time) # string to datetime
+ else:
+ stop_time = None
+
+ project_id = request.query_params.get('project_id', None)
+ if project_id:
+ project = get_object_or_404(models.Project, pk=request.query_params['project_id'])
+ else:
+ project = None
+
+ reservation = Reservation.objects.create(name=request.query_params.get('name', "reservation"),
+ description=request.query_params.get('description', ""),
+ project=project,
+ specifications_template=strategy_template.reservation_template,
+ specifications_doc=reservation_template_spec,
+ start_time=start_time,
+ stop_time=stop_time)
+
+ reservation_strategy_template_path = request._request.path
+ base_path = reservation_strategy_template_path[:reservation_strategy_template_path.find('/reservation_strategy_template')]
+ reservation_path = '%s/reservation/%s/' % (base_path, reservation.id,)
+
+ # return a response with the new serialized Reservation, and a Location to the new instance in the header
+ return Response(serializers.ReservationSerializer(reservation, context={'request':request}).data,
+ status=status.HTTP_201_CREATED,
+ headers={'Location': reservation_path})
+
+
class DefaultReservationTemplateViewSet(LOFARViewSet):
queryset = models.DefaultReservationTemplate.objects.all()
serializer_class = serializers.DefaultReservationTemplateSerializer
diff --git a/SAS/TMSS/backend/src/tmss/urls.py b/SAS/TMSS/backend/src/tmss/urls.py
index 039b531a658e3bed589f131860f3d1193bfc3b39..66e58162725f917a20c5020e6492ad6f39bed7d0 100644
--- a/SAS/TMSS/backend/src/tmss/urls.py
+++ b/SAS/TMSS/backend/src/tmss/urls.py
@@ -142,6 +142,7 @@ router.register(r'default_scheduling_constraints_template', viewsets.DefaultSche
router.register(r'default_task_template', viewsets.DefaultTaskTemplateViewSet)
router.register(r'default_task_relation_selection_template', viewsets.DefaultTaskRelationSelectionTemplateViewSet)
router.register(r'default_reservation_template', viewsets.DefaultReservationTemplateViewSet)
+router.register(r'reservation_strategy_template', viewsets.ReservationStrategyTemplateViewSet)
# instances
router.register(r'cycle', viewsets.CycleViewSet)
diff --git a/SAS/TMSS/backend/test/CMakeLists.txt b/SAS/TMSS/backend/test/CMakeLists.txt
index 18d00552af28dc70f428041fd2c028057a64b021..91dc978b752ed05cf2ebe07732a07c760808ae53 100644
--- a/SAS/TMSS/backend/test/CMakeLists.txt
+++ b/SAS/TMSS/backend/test/CMakeLists.txt
@@ -36,6 +36,7 @@ if(BUILD_TESTING)
lofar_add_test(t_permissions)
lofar_add_test(t_permissions_system_roles)
lofar_add_test(t_complex_serializers)
+ lofar_add_test(t_reservations)
set_tests_properties(t_scheduling PROPERTIES TIMEOUT 300)
set_tests_properties(t_tmssapp_scheduling_REST_API PROPERTIES TIMEOUT 300)
diff --git a/SAS/TMSS/backend/test/t_adapter.py b/SAS/TMSS/backend/test/t_adapter.py
index f67d013c90e33f656334d3416444ec70006f7ffb..772a2d43ed706e328371dc2cdb048f38f65db9ed 100755
--- a/SAS/TMSS/backend/test/t_adapter.py
+++ b/SAS/TMSS/backend/test/t_adapter.py
@@ -214,7 +214,7 @@ class SIPadapterTest(unittest.TestCase):
specifications_doc['stations']['filter'] = "HBA_210_250"
feedback_template = models.DataproductFeedbackTemplate.objects.get(name='feedback')
# feedback_doc = get_default_json_object_for_schema(feedback_template.schema) # todo <- fix the default generator, for some reason it does not produce valid json here...
- feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'angle3': 0, 'direction_type': 'J2000'}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
+ feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'direction_type': 'J2000'}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
for dp in specifications_doc['stations']['digital_pointings']:
dp['subbands'] = list(range(8))
# Create SubTask(output)
diff --git a/SAS/TMSS/backend/test/t_conversions.py b/SAS/TMSS/backend/test/t_conversions.py
index 7f8d66d6e4b8758b3cf13bf04bf3d8488deb89ad..1773168c7b1ded14c41aee27f0fddd6683d9f9f7 100755
--- a/SAS/TMSS/backend/test/t_conversions.py
+++ b/SAS/TMSS/backend/test/t_conversions.py
@@ -362,6 +362,35 @@ class UtilREST(unittest.TestCase):
self.assertNotEqual(rise, rise_last)
rise_last = rise
+ def test_util_target_rise_and_set_detects_when_target_above_horizon(self):
+
+ # assert always below and always above are usually false
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.8×tamps=2020-01-01&horizon=0.2', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+ self.assertIsNotNone(r_dict['CS002'][0]['rise'])
+ self.assertIsNotNone(r_dict['CS002'][0]['set'])
+ self.assertFalse(r_dict['CS002'][0]['always_below_horizon'])
+ self.assertFalse(r_dict['CS002'][0]['always_above_horizon'])
+
+ # assert rise and set are None and flag is true when target is always above horizon
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.8×tamps=2020-01-01&horizon=0.1', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+ self.assertIsNone(r_dict['CS002'][0]['rise'])
+ self.assertIsNone(r_dict['CS002'][0]['set'])
+ self.assertTrue(r_dict['CS002'][0]['always_above_horizon'])
+ self.assertFalse(r_dict['CS002'][0]['always_below_horizon'])
+
+ # assert rise and set are None and flag is true when target is always below horizon
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=-0.5×tamps=2020-01-01&horizon=0.2', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+ self.assertIsNone(r_dict['CS002'][0]['rise'])
+ self.assertIsNone(r_dict['CS002'][0]['set'])
+ self.assertFalse(r_dict['CS002'][0]['always_above_horizon'])
+ self.assertTrue(r_dict['CS002'][0]['always_below_horizon'])
+
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
diff --git a/SAS/TMSS/backend/test/t_reservations.py b/SAS/TMSS/backend/test/t_reservations.py
new file mode 100755
index 0000000000000000000000000000000000000000..9cc99f3a7da802c98d3e39f3dd608068351fbff1
--- /dev/null
+++ b/SAS/TMSS/backend/test/t_reservations.py
@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import unittest
+import requests
+
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+from lofar.common.test_utils import exit_with_skipped_code_if_skip_integration_tests
+exit_with_skipped_code_if_skip_integration_tests()
+
+from lofar.common.json_utils import get_default_json_object_for_schema, add_defaults_to_json_object_for_schema
+
+
+# Do Mandatory setup step:
+# use setup/teardown magic for tmss test database, ldap server and django server
+# (ignore pycharm unused import statement, python unittests does use at RunTime the tmss_test_environment_unittest_setup module)
+from lofar.sas.tmss.test.tmss_test_environment_unittest_setup import *
+tmss_test_env.populate_schemas()
+
+from lofar.sas.tmss.test.tmss_test_data_django_models import *
+
+# import and setup rest test data creator
+from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
+rest_data_creator = TMSSRESTTestDataCreator(BASE_URL, AUTH)
+
+from lofar.sas.tmss.tmss.tmssapp import models
+
+
+from lofar.sas.tmss.tmss.tmssapp.reservations import get_active_station_reservations_in_timewindow
+
+
+class TestStationReservations(unittest.TestCase):
+ """
+ Tests for the active station reservations
+ """
+
+ def setUp(self) -> None:
+ # wipe all reservations in between tests, so the tests don't influence each other
+ for reservation in models.Reservation.objects.all():
+ reservation.delete()
+
+ @staticmethod
+ def create_station_reservation(additional_name, lst_stations, start_time, stop_time=None):
+ """
+ Create a station reservation with given list of stations, start_time and stop_time
+ """
+ reservation_template = models.ReservationTemplate.objects.get(name="resource reservation")
+ reservation_template_spec = get_default_json_object_for_schema(reservation_template.schema)
+ reservation_template_spec["resources"] = {"stations": lst_stations }
+ res = models.Reservation.objects.create(name="Station Reservation %s" % additional_name,
+ description="Station reservation for testing",
+ specifications_template=reservation_template,
+ specifications_doc=reservation_template_spec,
+ start_time=start_time,
+ stop_time=stop_time)
+
+ def test_no_stations_reservation(self):
+ """
+ Check that creating 'default' reservation with no additional station reservation added, we still can
+ call 'get_active_station_reservations_in_timewindow' and it will return an empty list
+ """
+ reservation_template = models.ReservationTemplate.objects.get(name="resource reservation")
+ reservation_template_spec = get_default_json_object_for_schema(reservation_template.schema)
+ res = models.Reservation.objects.create(name="AnyReservation",
+ description="Reservation of something else",
+ specifications_template=reservation_template,
+ specifications_doc=reservation_template_spec,
+ start_time=datetime.now(),
+ stop_time=None)
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(datetime.now(), datetime.now()+timedelta(weeks=53)))
+
+ def test_active_station_reservation(self):
+ """
+ Test station reservation when 2 stations are reserved for 24hr with both same start and stop time
+ Check 'get_active_station_reservations_in_timewindow' with different time ranges
+ """
+ reservation_start_time = datetime(2020, 1, 1, 0, 0, 0)
+ reservation_stop_time = datetime(2020, 1, 2, 0, 0, 0)
+ reservation_stations = ["CS001", "CS002"]
+ self.create_station_reservation("two_stations", reservation_stations, reservation_start_time, reservation_stop_time)
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(datetime.now(), datetime.now()))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_stop_time))
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_start_time))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_start_time+timedelta(seconds=1)))
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(reservation_stop_time,
+ reservation_stop_time))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_stop_time-timedelta(seconds=1),
+ reservation_stop_time))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time-timedelta(weeks=53),
+ reservation_stop_time+timedelta(weeks=53)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ datetime.now()+timedelta(weeks=53)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ def test_active_station_reservation_with_same_station_overlap(self):
+ """
+ Test station reservation when 2 stations are reserved for 24hr with both same start and stop time
+ Same stations are reserved separately as well, which should result NOT result in having the station twice in
+ the resulting active station list
+ Check 'get_active_station_reservations_in_timewindow' with different time ranges
+ """
+ reservation_start_time = datetime(2020, 1, 1, 0, 0, 0)
+ reservation_stop_time = datetime(2020, 1, 2, 0, 0, 0)
+ reservation_stations = ["CS001", "CS002"]
+ self.create_station_reservation("two_stations", reservation_stations, reservation_start_time, reservation_stop_time)
+ self.create_station_reservation("cs1", ["CS001"], reservation_start_time, reservation_stop_time)
+ self.create_station_reservation("cs2", ["CS002"], reservation_start_time, reservation_stop_time)
+
+ # same lower_bound as upper_bound, empty list
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(datetime.now(), datetime.now()))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_stop_time))
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_start_time))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_start_time + timedelta(seconds=1)))
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(reservation_stop_time,
+ reservation_stop_time + timedelta(seconds=1)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_stop_time - timedelta(seconds=1),
+ reservation_stop_time))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time - timedelta(weeks=53),
+ reservation_stop_time + timedelta(weeks=53)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ datetime.now() + timedelta(weeks=53)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ def test_active_station_reservation_with_station_no_stop_time(self):
+ """
+ Test station reservation when 2 stations are reserved forever both same start and stop time
+ Check 'get_active_station_reservations_in_timewindow' with different time ranges
+ """
+ reservation_start_time = datetime(2020, 1, 1, 0, 0, 0)
+ reservation_stations = ["CS001", "CS002"]
+ self.create_station_reservation("two_stations_no_end_time", reservation_stations, reservation_start_time)
+
+ # we are now still in reservation started in 2020
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(datetime.now(), datetime.now()))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ reservation_start_time + timedelta(seconds=1)))
+ # before start time, always empty
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(reservation_start_time - timedelta(seconds=1),
+ reservation_start_time))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time - timedelta(weeks=53),
+ reservation_start_time + timedelta(weeks=53)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(reservation_start_time,
+ datetime.now() + timedelta(weeks=53)))
+ self.assertCountEqual(reservation_stations,
+ get_active_station_reservations_in_timewindow(datetime(2020, 1, 1, 12, 0, 0),
+ datetime(2020, 1, 2, 12, 0, 0)))
+
+ def test_active_station_reservation_every_hour_one_station(self):
+ """
+ Test station reservation when 1 station is reserved for 1 hour
+ Check 'get_active_station_reservations_in_timewindow' with different time ranges
+ """
+ first_day = 1
+ last_day = 7
+ reservation_start_time = datetime(2020, 1, first_day, 0, 0, 0)
+ reservation_stop_time = datetime(2020, 1, last_day+1, 0, 0, 0)
+ for day_nbr in range(first_day, last_day+1):
+ self.create_station_reservation("cs%s" % day_nbr, ["CS00%d" % day_nbr],
+ datetime(2020, 1, day_nbr, 12, 0, 0), datetime(2020, 1, day_nbr+1, 0, 0, 0))
+
+ self.assertCountEqual([],
+ get_active_station_reservations_in_timewindow(datetime.now(), datetime.now()))
+ self.assertCountEqual(["CS001","CS002","CS003","CS004","CS005","CS006","CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time))
+ self.assertCountEqual(["CS002","CS003","CS004","CS005","CS006","CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time+timedelta(days=1), reservation_stop_time))
+ self.assertCountEqual(["CS003","CS004","CS005","CS006","CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time+timedelta(days=2), reservation_stop_time))
+ self.assertCountEqual(["CS004","CS005","CS006","CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time+timedelta(days=3), reservation_stop_time))
+ self.assertCountEqual(["CS005","CS006","CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time+timedelta(days=4), reservation_stop_time))
+ self.assertCountEqual(["CS006","CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time+timedelta(days=5), reservation_stop_time))
+ self.assertCountEqual(["CS007"],
+ get_active_station_reservations_in_timewindow(reservation_start_time+timedelta(days=6), reservation_stop_time))
+
+ self.assertCountEqual(["CS001","CS002","CS003","CS004","CS005","CS006"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time-timedelta(days=1)))
+ self.assertCountEqual(["CS001","CS002","CS003","CS004","CS005"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time-timedelta(days=2)))
+ self.assertCountEqual(["CS001","CS002","CS003","CS004"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time-timedelta(days=3)))
+ self.assertCountEqual(["CS001","CS002","CS003"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time-timedelta(days=4)))
+ self.assertCountEqual(["CS001","CS002"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time-timedelta(days=5)))
+ self.assertCountEqual(["CS001"],
+ get_active_station_reservations_in_timewindow(reservation_start_time, reservation_stop_time-timedelta(days=6)))
+from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
+from django.core.exceptions import ValidationError
+
+
+class CreationFromReservationStrategyTemplate(unittest.TestCase):
+ """
+ Test that reservations can be created from strategy template
+ """
+
+ def test_create_reservation_ok(self):
+ """
+ Check that reservations from the reservation strategy can be created with api
+ """
+ strategy_template = models.ReservationStrategyTemplate.objects.get(name="Regular station maintenance")
+
+ reservation_spec = add_defaults_to_json_object_for_schema(strategy_template.template,
+ strategy_template.reservation_template.schema)
+ reservation = models.Reservation.objects.create(name=strategy_template.name,
+ description="Unittest with %s" % strategy_template.description,
+ project=None,
+ specifications_template=strategy_template.reservation_template,
+ specifications_doc=reservation_spec,
+ start_time=datetime.now(),
+ stop_time=None)
+
+ # Check URL of the reservation that is created
+ response = GET_and_assert_equal_expected_code(self, BASE_URL + '/reservation/%d' % reservation.pk, 200)
+ self.assertEqual(response['id'], reservation.pk) # should be different id then previous one created
+
+ # Check that action call 'create_reservation' (no parameters) of strategy template creates a
+ # new reservation (with http result code 201)
+ response = GET_and_assert_equal_expected_code(self, BASE_URL + '/reservation_strategy_template/%d/create_reservation' % strategy_template.pk, 201)
+ self.assertNotEqual(response['id'], reservation.pk) # should be different id then previous one created
+ self.assertLess(response['start_time'], datetime.utcnow().isoformat()) # start_time created with now so that was some micro seconds ago
+ self.assertEqual(response['stop_time'], None)
+ self.assertEqual(response['duration'], None)
+ self.assertEqual(response['name'], "reservation")
+ self.assertEqual(response['specifications_doc'], reservation_spec)
+
+ def test_create_reservation_exception(self):
+ """
+ Check that reservations from the reservation strategy results in an Exception due to wrong
+ station assignment
+ """
+ strategy_template = models.ReservationStrategyTemplate.objects.get(name="Regular station maintenance")
+ strategy_template.template['resources']['stations'] = ['CS999']
+ # using ValidationError seem not to work?
+ with self.assertRaises(Exception) as context:
+ strategy_template.save()
+ self.assertIn('is not one of', str(context.exception))
+ self.assertIn('Failed validating', str(context.exception))
+
+
+class ReservationTest(unittest.TestCase):
+ """
+ Check the Reservation model
+ TODO: more testcases to be added
+ """
+
+ def test_create_reservation_validation_error(self):
+ """
+ Check that creating reservation with results in SchemaValidationException due to wrong station assignment
+ """
+ reservation_template = models.ReservationTemplate.objects.get(pk=1)
+ reservation_spec = get_default_json_object_for_schema(reservation_template.schema)
+ reservation_spec['resources']['stations'] = ['CS999']
+ with self.assertRaises(SchemaValidationException) as context:
+ models.Reservation.objects.create(name="Test Reservation",
+ description="Unittest",
+ project=None,
+ specifications_template=reservation_template,
+ specifications_doc=reservation_spec,
+ start_time=datetime.now(),
+ stop_time=None)
+ self.assertIn('is not one of', str(context.exception))
+
diff --git a/SAS/TMSS/backend/test/t_reservations.run b/SAS/TMSS/backend/test/t_reservations.run
new file mode 100755
index 0000000000000000000000000000000000000000..ba642dab7566ba76dd28753265ba8cfa8e02fe28
--- /dev/null
+++ b/SAS/TMSS/backend/test/t_reservations.run
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "*tmss*" t_reservations.py
+
diff --git a/SAS/TMSS/backend/test/t_reservations.sh b/SAS/TMSS/backend/test/t_reservations.sh
new file mode 100755
index 0000000000000000000000000000000000000000..87d473569831af900660644c91bbaf5f7ad6d292
--- /dev/null
+++ b/SAS/TMSS/backend/test/t_reservations.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_reservations
\ No newline at end of file
diff --git a/SAS/TMSS/backend/test/t_scheduling.py b/SAS/TMSS/backend/test/t_scheduling.py
index 6dda9cf61de9fa857d009bec6204fad744de1e75..5bcfa16e9e29e9e82b75a3c5f13dff663a89289d 100755
--- a/SAS/TMSS/backend/test/t_scheduling.py
+++ b/SAS/TMSS/backend/test/t_scheduling.py
@@ -97,6 +97,18 @@ def create_reserved_stations_for_testing(station_list):
assigned = rarpc.do_assignment(ra_spec)
return assigned
+def duplicates(l: list) -> list:
+ # O(n^2), but that's good enough.
+ uniques = []
+ dupes = []
+
+ for e in l:
+ if e not in uniques:
+ uniques.append(e)
+ elif e not in dupes:
+ dupes.append(e)
+
+ return dupes
class SchedulingTest(unittest.TestCase):
def setUp(self):
@@ -113,13 +125,12 @@ class SchedulingTest(unittest.TestCase):
test_data_creator.wipe_cache()
- def test_schedule_observation_subtask_with_enough_resources_available(self):
+ def _test_schedule_observation_subtask_with_enough_resources_available(self, observation_specification_doc):
with tmss_test_env.create_tmss_client() as client:
task_blueprint_data = test_data_creator.TaskBlueprint(template_url=client.get_task_template(name="target observation")['url'])
task_blueprint = test_data_creator.post_data_and_get_response_as_json_object(task_blueprint_data, '/task_blueprint/')
subtask_template = client.get_subtask_template("observation control")
- spec = get_default_json_object_for_schema(subtask_template['schema'])
- spec['stations']['digital_pointings'][0]['subbands'] = [0]
+ spec = add_defaults_to_json_object_for_schema(observation_specification_doc, subtask_template['schema'])
cluster_url = client.get_path_as_json_object('/cluster/1')['url']
subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
@@ -137,6 +148,34 @@ class SchedulingTest(unittest.TestCase):
self.assertEqual('scheduled', subtask['state_value'])
self.assertEqual('scheduled', tmss_test_env.ra_test_environment.radb.getTask(tmss_id=subtask_id)['status'])
+ # test whether all dataproduct specifications are unique
+ outputs = Subtask.objects.get(pk=subtask_id).outputs.all()
+ dataproduct_specifications_docs = [dp.specifications_doc for output in outputs for dp in output.dataproducts.all()]
+ duplicate_dataproduct_specification_docs = duplicates(dataproduct_specifications_docs)
+
+ self.assertEqual([], duplicate_dataproduct_specification_docs)
+
+ def test_schedule_observation_subtask_with_enough_resources_available(self):
+ spec = { "stations": { "digital_pointings": [ { "subbands": [0] } ] } }
+ self._test_schedule_observation_subtask_with_enough_resources_available(spec)
+
+ def test_schedule_beamformer_observation_subtask_with_enough_resources_available(self):
+ spec = {
+ "stations": { "digital_pointings": [ { "name": "target0", "subbands": [0] } ] },
+ "COBALT": {
+ "version": 1,
+ "correlator": { "enabled": False },
+ "beamformer": {
+ "tab_pipelines": [
+ {
+ "SAPs": [ { "name": "target0", "tabs": [ { "coherent": False }, { "coherent": True } ] } ]
+ }
+ ]
+ }
+ }
+ }
+ self._test_schedule_observation_subtask_with_enough_resources_available(spec)
+
def test_schedule_observation_subtask_with_one_blocking_reservation_failed(self):
"""
Set (Resource Assigner) station CS001 to reserved
@@ -260,6 +299,7 @@ class SchedulingTest(unittest.TestCase):
obs_subtask = test_data_creator.post_data_and_get_response_as_json_object(obs_subtask_data, '/subtask/')
obs_subtask_output_url = test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=obs_subtask['url']), '/subtask_output/')
test_data_creator.post_data_and_get_url(test_data_creator.Dataproduct(filename="L%s_SB000.MS"%obs_subtask['id'],
+ specifications_doc={"sap": "target0", "subband": 0 },
subtask_output_url=obs_subtask_output_url), '/dataproduct/')
# now create the pipeline...
@@ -304,6 +344,7 @@ class SchedulingTest(unittest.TestCase):
obs_subtask = test_data_creator.post_data_and_get_response_as_json_object(obs_subtask_data, '/subtask/')
obs_subtask_output_url = test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=obs_subtask['url']), '/subtask_output/')
test_data_creator.post_data_and_get_url(test_data_creator.Dataproduct(filename="L%s_SB000.MS"%obs_subtask['id'],
+ specifications_doc={"sap": "target0", "subband": 0},
subtask_output_url=obs_subtask_output_url), '/dataproduct/')
# now create the ingest...
@@ -416,6 +457,23 @@ class SubtaskInputOutputTest(unittest.TestCase):
setting.value = True
setting.save()
+
+ def test_specifications_doc_meets_selection_doc(self):
+ # empty selection matches all
+ self.assertTrue(specifications_doc_meets_selection_doc({'something else': 'target0'}, {}))
+
+ # specification is a list? specification must be a subset of the selection
+ self.assertTrue(specifications_doc_meets_selection_doc({'sap': ['target0']}, {'sap': ['target0']}))
+ self.assertFalse(specifications_doc_meets_selection_doc({'sap': ['target0','target1','target2']}, {'sap': ['target0','target1']}))
+
+ # specification is a value? it must appear in the selection
+ self.assertTrue(specifications_doc_meets_selection_doc({'sap': 'target0'}, {'sap': ['target0']}))
+ self.assertTrue(specifications_doc_meets_selection_doc({'sap': 'target0'}, {'sap': ['target0','target1']}))
+ self.assertTrue(specifications_doc_meets_selection_doc({'sap': 'target0'}, {'sap': 'target0'}))
+
+ # specification must contain the selection key
+ self.assertFalse(specifications_doc_meets_selection_doc({'something else': 'target0'}, {'sap': 'target0'}))
+
@mock.patch("lofar.sas.tmss.tmss.tmssapp.subtasks.assign_or_unassign_resources")
def test_schedule_pipeline_subtask_filters_predecessor_output_dataproducts_for_input(self, assign_resources_mock):
# setup:
@@ -431,12 +489,12 @@ class SubtaskInputOutputTest(unittest.TestCase):
pipe_in2 = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=pipe_st, producer=obs_out2, selection_doc={'sap': ['target1']}))
# create obs output dataproducts with specs we can filter on
- dp1_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': ['target0']}))
- dp1_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': ['target1']}))
- dp1_3 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': ['target0']}))
+ dp1_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': 'target0', 'subband': 0}))
+ dp1_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': 'target1', 'subband': 0}))
+ dp1_3 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': 'target0', 'subband': 1}))
- dp2_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': ['target0']}))
- dp2_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': ['target1']}))
+ dp2_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': 'target0', 'subband': 0}))
+ dp2_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': 'target1', 'subband': 0}))
# trigger:
# schedule pipeline, which should attach the correct subset of dataproducts to the pipeline inputs
@@ -488,6 +546,7 @@ class SAPTest(unittest.TestCase):
client.set_subtask_status(subtask_id, 'defined')
subtask = client.schedule_subtask(subtask_id)
+ self.assertEqual(1, subtask_model.output_dataproducts.count())
self.assertEqual(1, subtask_model.output_dataproducts.values('sap').count())
self.assertEqual(subtask_model.output_dataproducts.first().sap.specifications_doc['pointing']['angle1'], pointing['angle1'])
self.assertEqual(subtask_model.output_dataproducts.first().sap.specifications_doc['pointing']['angle2'], pointing['angle2'])
@@ -505,8 +564,8 @@ class SAPTest(unittest.TestCase):
pipe_in = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=pipe_st, producer=obs_out))
# create obs output dataproducts
- dp1_in = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out))
- dp2_in = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out))
+ dp1_in = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out, specifications_doc={"identifiers": { "sap_index": 0, "subband_index": 0 }}))
+ dp2_in = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out, specifications_doc={"identifiers": { "sap_index": 0, "subband_index": 1 }}))
# schedule pipeline, which should copy the SAP
schedule_pipeline_subtask(pipe_st)
diff --git a/SAS/TMSS/backend/test/t_schemas.py b/SAS/TMSS/backend/test/t_schemas.py
index 0cf0157e39e2917d8baaa06384836c4795c41ab4..e9b25c35efca7a967bf7bf541c027cb15b836f7b 100755
--- a/SAS/TMSS/backend/test/t_schemas.py
+++ b/SAS/TMSS/backend/test/t_schemas.py
@@ -43,13 +43,17 @@ class TestSchemas(unittest.TestCase):
""" Check whether the given schema is valid. """
# Can all $refs be actually resolved?
- logger.info("Resolving references for schema %s", name)
- resolved_refs(schema)
+ try:
+ resolved_refs(schema)
+ except Exception as e:
+ raise Exception("Failed to resolve references in schema %s" % name) from e
# Does this schema provide actually valid defaults?
- logger.info("Validating defaults of schema %s", name)
- defaults = get_default_json_object_for_schema(schema)
- validate_json_against_schema(defaults, schema)
+ try:
+ defaults = get_default_json_object_for_schema(schema)
+ validate_json_against_schema(defaults, schema)
+ except Exception as e:
+ raise Exception("Failure in defaults in schema %s" % name) from e
def check_schema_table(self, model):
""" Check all schemas present in the database for a given model. """
diff --git a/SAS/TMSS/backend/test/t_subtasks.py b/SAS/TMSS/backend/test/t_subtasks.py
index c3a8c261bd74c019f4728aa4d525e376aad875cf..8086f231da703fba4bcdf574bed9940f0ee6d3d2 100755
--- a/SAS/TMSS/backend/test/t_subtasks.py
+++ b/SAS/TMSS/backend/test/t_subtasks.py
@@ -181,9 +181,9 @@ class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
self.assertEqual(None, subtask)
# Next call will fail due to no qa_files object
- # ValueError: Cannot create qa_plots subtask for task_blueprint id=1 because it has no qafile subtask(s)
- with self.assertRaises(SubtaskCreationException):
- subtask = create_qaplots_subtask_from_task_blueprint(task_blueprint)
+ subtask = create_qaplots_subtask_from_task_blueprint(task_blueprint)
+ # subtask object is None because QA file conversion is by default not enabled!!!!
+ self.assertEqual(None, subtask)
def test_create_sequence_of_subtask_from_task_blueprint_with_QA_enabled(self):
@@ -230,10 +230,10 @@ class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
def test_create_subtasks_from_task_blueprint_translates_SAP_names(self):
task_blueprint = create_task_blueprint_object_for_testing('target observation')
task_blueprint.specifications_doc['SAPs'] = [{'name': 'target1', 'target': '', 'subbands': [],
- 'digital_pointing': {'angle1': 0.1, 'angle2': 0.1, 'angle3': 0.1,
+ 'digital_pointing': {'angle1': 0.1, 'angle2': 0.1,
'direction_type': 'J2000'}},
{'name': 'target2', 'target': '', 'subbands': [],
- 'digital_pointing': {'angle1': 0.2, 'angle2': 0.2, 'angle3': 0.2,
+ 'digital_pointing': {'angle1': 0.2, 'angle2': 0.2,
'direction_type': 'J2000'}}]
subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
i = 0
@@ -386,9 +386,9 @@ class SettingTest(unittest.TestCase):
class SubTaskCreationFromTaskBlueprintBeamformer(unittest.TestCase):
saps = [{"name": "target1", "target": "", "subbands": [349, 372],
- "digital_pointing": {"angle1": 0.24, "angle2": 0.25, "angle3": 0.26, "direction_type": "J2000"}},
+ "digital_pointing": {"angle1": 0.24, "angle2": 0.25, "direction_type": "J2000"}},
{"name": "target2", "target": "", "subbands": [309, 302],
- "digital_pointing": {"angle1": 0.42, "angle2": 0.52, "angle3": 0.62, "direction_type": "J2000"}}
+ "digital_pointing": {"angle1": 0.42, "angle2": 0.52, "direction_type": "J2000"}}
]
beamformers = [{"name": "beamformer1",
"coherent": {"settings": {"stokes": "I", "time_integration_factor": 8, "subbands_per_file": 244,
@@ -447,7 +447,7 @@ class SubTaskCreationFromTaskBlueprintBeamformer(unittest.TestCase):
def test_generate_tab_ring_pointings_returns_correct_pointings(self):
- pointing = {"angle1": 0.11, "angle2": 0.22, "angle3": 0.33, "direction_type": "J2000"}
+ pointing = {"angle1": 0.11, "angle2": 0.22, "direction_type": "J2000"}
tab_rings = {"width": 1, "count": 1}
# assert center pointing is returned
@@ -469,9 +469,9 @@ class SubTaskCreationFromTaskBlueprintBeamformer(unittest.TestCase):
def test_add_pointings_adds_correctly(self):
pointing_a = {"angle1": 0.11, "angle2": 0.22, "direction_type": "J2000"}
- pointing_b = {"angle1": 0.88, "angle2": 0.66, "angle3": 0.77, "direction_type": "J2000"}
+ pointing_b = {"angle1": 0.88, "angle2": 0.66, "direction_type": "J2000"}
pointing_sum = _add_pointings(pointing_a, pointing_b)
- self.assertEqual(pointing_sum, {"angle1": 0.99, "angle2": 0.88, "angle3": 0.77, "direction_type": "J2000"})
+ self.assertEqual(pointing_sum, {"angle1": 0.99, "angle2": 0.88, "direction_type": "J2000"})
def test_filter_subbands_filters_correctly(self):
subbands = [1,3,4,5,10,11,12,13,19,20]
diff --git a/SAS/TMSS/backend/test/t_tasks.py b/SAS/TMSS/backend/test/t_tasks.py
index 2652a8ff989b584ae69834b1b50beaf5dc51a2f2..88e4791390c6e46ff365372fe86cc79be91f24b3 100755
--- a/SAS/TMSS/backend/test/t_tasks.py
+++ b/SAS/TMSS/backend/test/t_tasks.py
@@ -44,6 +44,8 @@ rest_data_creator = TMSSRESTTestDataCreator(tmss_test_env.django_server.url, (tm
from lofar.sas.tmss.tmss.tmssapp.tasks import *
+from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
+
class CreationFromSchedulingUnitDraft(unittest.TestCase):
"""
@@ -397,7 +399,6 @@ class TaskBlueprintStateTest(unittest.TestCase):
self.assertEqual(expected_task_state, task_blueprint.status)
-
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
unittest.main()
diff --git a/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py b/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py
index 9fb0b24d6fdf1f97882ec4dc2a5157646dcd5247..d3da150deaa98063eb7c714b99090c37447c8597 100755
--- a/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py
+++ b/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py
@@ -335,6 +335,69 @@ class ReservationTemplateTestCase(unittest.TestCase):
DELETE_and_assert_gone(self, url)
+class ReservationStrategyTemplateTestCase(unittest.TestCase):
+ def test_reservation_strategy_template_list_apiformat(self):
+ r = requests.get(BASE_URL + '/reservation_strategy_template/?format=api', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ self.assertTrue("Reservation Strategy Template List" in r.content.decode('utf8'))
+
+ def test_reservation_strategy_template_GET_nonexistant_raises_error(self):
+ GET_and_assert_equal_expected_code(self, BASE_URL + '/reservation_strategy_template/1234321/', 404)
+
+ def test_reservation_strategy_template_POST_and_GET(self):
+ # POST and GET a new item and assert correctness
+ test_data = test_data_creator.ReservationStrategyTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_strategy_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ def test_reservation_strategy_template_PUT_invalid_raises_error(self):
+ test_data = test_data_creator.ReservationStrategyTemplate()
+ PUT_and_assert_expected_response(self, BASE_URL + '/reservation_strategy_template/9876789876/', test_data, 404, {})
+
+ def test_reservation_strategy_template_PUT(self):
+ # POST new item, verify
+ test_data = test_data_creator.ReservationStrategyTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_strategy_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ # PUT new values, verify
+ test_data2 = test_data_creator.ReservationStrategyTemplate("reservationtemplate2")
+ expected_data2 = test_data_creator.update_schema_from_template("reservationtemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
+
+ def test_reservation_strategy_template_PATCH(self):
+ # POST new item, verify
+ test_data = test_data_creator.ReservationStrategyTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_strategy_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ test_patch = {"name": "new_name",
+ "description": "better description"}
+
+ # PATCH item and verify
+ expected_patch_data = test_data_creator.update_schema_from_template("reservationtemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ def test_reservation_strategy_template_DELETE(self):
+ # POST new item, verify
+ test_data = test_data_creator.ReservationStrategyTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_strategy_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ # DELETE and check it's gone
+ DELETE_and_assert_gone(self, url)
+
+
class TaskTemplateTestCase(unittest.TestCase):
def test_task_template_list_apiformat(self):
@@ -2835,7 +2898,7 @@ class ReservationTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, url, reservation_test_data)
test_patch = {"description": "This is a new and improved description",
- "duration": 90}
+ "stop_time": None}
# PATCH item and verify
expected_patch_data = test_data_creator.update_schema_from_template("reservationtemplate", test_patch)
diff --git a/SAS/TMSS/backend/test/tmss_test_data_django_models.py b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
index 6b1089baf00d6989aff0ad87ad132c21000f4b1c..03bd63e347821654485556b3e2c146e9aea2d92b 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
@@ -562,6 +562,11 @@ def Reservation_test_data(name="MyReservation", duration=None, start_time=None,
if start_time is None:
start_time = datetime.utcnow() + timedelta(hours=12)
+ if duration is None:
+ stop_time = None
+ else:
+ stop_time = start_time + timedelta(seconds=duration)
+
specifications_template = models.ReservationTemplate.objects.create(**ReservationTemplate_test_data())
specifications_doc = get_default_json_object_for_schema(specifications_template.schema)
@@ -570,11 +575,27 @@ def Reservation_test_data(name="MyReservation", duration=None, start_time=None,
"description": "Test Reservation",
"tags": ["TMSS", "TESTING"],
"start_time": start_time,
- "duration": duration, # can be None
+ "stop_time": stop_time, # can be None
"specifications_doc": specifications_doc,
"specifications_template": specifications_template}
+def ReservationStrategyTemplate_test_data(name="my_ReservationStrategyTemplate",
+ reservation_template:models.ReservationTemplate=None,
+ template:dict=None) -> dict:
+ if reservation_template is None:
+ reservation_template = models.ReservationTemplate.objects.create(**ReservationTemplate_test_data())
+
+ if template is None:
+ template = get_default_json_object_for_schema(reservation_template.schema)
+
+ return {"name": name,
+ "description": 'My Reservation Template description',
+ "template": template,
+ "reservation_template": reservation_template,
+ "tags": ["TMSS", "TESTING"]}
+
+
def ProjectPermission_test_data(name=None, GET=None, PUT=None, POST=None, PATCH=None, DELETE=None) -> dict:
if name is None:
name = 'MyProjectPermission_%s' % uuid.uuid4()
diff --git a/SAS/TMSS/backend/test/tmss_test_data_rest.py b/SAS/TMSS/backend/test/tmss_test_data_rest.py
index 8d59aa434be32e177cffc564ec6b1f6edb44e938..551a251a68857807ceb4e4bf63699d6bd575c44d 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_rest.py
@@ -147,6 +147,29 @@ class TMSSRESTTestDataCreator():
"schema": schema,
"tags": ["TMSS", "TESTING"]}
+ @property
+ def cached_reservation_template_url(self):
+ try:
+ return self._reservation_template_url
+ except AttributeError:
+ self._reservation_template_url = self.post_data_and_get_url(self.ReservationTemplate(), '/reservation_template/')
+ return self._reservation_template_url
+
+ def ReservationStrategyTemplate(self, name="my_ReservationStrategyTemplate",
+ reservation_template_url=None,
+ template:dict=None) -> dict:
+ if reservation_template_url is None:
+ reservation_template_url = self.cached_reservation_template_url
+
+ if template is None:
+ template = self.get_response_as_json_object(reservation_template_url+'/default')
+
+ return {"name": name,
+ "description": 'My ReservationTemplate description',
+ "template": template,
+ "reservation_template": reservation_template_url,
+ "version": "1",
+ "tags": ["TMSS", "TESTING"]}
def SchedulingUnitObservingStrategyTemplate(self, name="my_SchedulingUnitObservingStrategyTemplate",
scheduling_unit_template_url=None,
@@ -828,6 +851,11 @@ class TMSSRESTTestDataCreator():
if start_time is None:
start_time = datetime.utcnow() + timedelta(hours=12)
+ if duration is None:
+ stop_time = None
+ else:
+ stop_time = start_time + timedelta(seconds=duration)
+
if specifications_template_url is None:
specifications_template_url = self.post_data_and_get_url(self.ReservationTemplate(), '/reservation_template/')
@@ -837,12 +865,15 @@ class TMSSRESTTestDataCreator():
if isinstance(start_time, datetime):
start_time = start_time.isoformat()
+ if isinstance(stop_time, datetime):
+ stop_time = stop_time.isoformat()
+
return {"name": name,
"project": project_url,
"description": "Test Reservation",
"tags": ["TMSS", "TESTING"],
"start_time": start_time,
- "duration": duration, # can be None
+ "stop_time": stop_time, # can be None
"specifications_doc": specifications_doc,
"specifications_template": specifications_template_url}
diff --git a/SAS/TMSS/client/lib/populate.py b/SAS/TMSS/client/lib/populate.py
index 6d3420403a6490f9b74c7117f4fb845bce66e9e5..ccadba3d1274599f1d78b56c40c2be74405085fd 100644
--- a/SAS/TMSS/client/lib/populate.py
+++ b/SAS/TMSS/client/lib/populate.py
@@ -38,14 +38,18 @@ def populate_schemas(schema_dir: str=None, templates_filename: str=None):
# keep track of the templates, json schemas and references
templates_dict = {}
observing_strategy_templates = []
+ reservation_strategy_templates = []
schema_references = {}
all_references = set()
# load all templates and schemas and prepare them for upload.
# determine the dependencies, and upload the depenends first, and the rest in parallel later.
for template in templates:
- with open(os.path.join(schema_dir, template['file_name'])) as schema_file:
- json_schema = json.loads(schema_file.read())
+ try:
+ with open(os.path.join(schema_dir, template['file_name'])) as schema_file:
+ json_schema = json.loads(schema_file.read())
+ except Exception as e:
+ raise Exception("Could not decode JSON schema %s" % template['file_name']) from e
# add template name/description/version from schema if not already in template
template['name'] = template.get('name', json_schema.get('title', '<no name>'))
@@ -71,7 +75,7 @@ def populate_schemas(schema_dir: str=None, templates_filename: str=None):
# get the id without trailing # and/or /
json_schema_id = json_schema.get('$id', "").rstrip("#").rstrip("/")
- if template_name == 'scheduling_unit_observing_strategy_template':
+ if 'strategy_template' in template_name:
template['template'] = json_schema
else:
template['schema'] = json_schema
@@ -83,10 +87,17 @@ def populate_schemas(schema_dir: str=None, templates_filename: str=None):
# store the prepared template for upload
if template_name == 'scheduling_unit_observing_strategy_template':
+ template["strategy_template_name"] = template_name # so the 'strategy_template' name
+ template["template_name"] = "scheduling_unit_template"
observing_strategy_templates.append(template)
+ elif template_name == 'reservation_strategy_template':
+ template["strategy_template_name"] = template_name
+ template["template_name"] = "reservation_template"
+ reservation_strategy_templates.append(template)
else:
templates_dict[json_schema_id] = template
+
# helper functions for uploading
def upload_template(template: dict):
logger.info("Uploading template with name='%s' version='%s'", template['name'], template['version'])
@@ -103,13 +114,18 @@ def populate_schemas(schema_dir: str=None, templates_filename: str=None):
template = templates_dict.pop(id)
upload_template(template)
- # helper functions for uploading observing_strategy_templates
- def upload_observing_strategy_templates(template: dict):
- scheduling_unit_templates = client.get_path_as_json_object('scheduling_unit_template?name=' + template.get('scheduling_unit_template_name') + '&version=' + template.get('scheduling_unit_template_version'))
- scheduling_unit_template = scheduling_unit_templates[0]
- template['scheduling_unit_template'] = scheduling_unit_template['url']
- logger.info("Uploading observation strategy with name='%s' version='%s'", template['name'], template['version'])
- client.post_template(template_path='scheduling_unit_observing_strategy_template', **template)
+ def upload_strategy_templates(template: dict):
+ """
+ Helper function for uploading strategy_templates
+ Use template["strategy_template_name"] for the name of the 'strategy_template' to be uploaded
+ Use template["template_name"] for the name of the template (used for validation)
+ """
+ tn = template.get('template_name')
+ response_templates = client.get_path_as_json_object(tn+'?name=' + template.get(tn+'_name') + '&version=' + template.get(tn+'_version'))
+ template[tn] = response_templates[0]['url']
+ logger.info("Uploading strategy with name='%s' version='%s'", template['name'], template['version'])
+ client.post_template(template_path=template.get('strategy_template_name'), **template)
+
# first, upload all dependent templates
for ref in all_references:
@@ -118,11 +134,15 @@ def populate_schemas(schema_dir: str=None, templates_filename: str=None):
# then, upload the remaining templates in parallel
rest_templates = [template for template in templates_dict.values()]
with ThreadPoolExecutor() as executor:
- executor.map(upload_template, rest_templates)
+ executor.map(upload_template, rest_templates)
+
+ # the reservation_strategy_templates
+ with ThreadPoolExecutor() as executor:
+ executor.map(upload_strategy_templates, reservation_strategy_templates)
# and finally, the observing_strategy_templates
with ThreadPoolExecutor() as executor:
- executor.map(upload_observing_strategy_templates, observing_strategy_templates)
+ executor.map(upload_strategy_templates, observing_strategy_templates)
scheduling_constraints_templates = client.get_path_as_json_object('scheduling_constraints_template')
if scheduling_constraints_templates:
diff --git a/SAS/TMSS/frontend/tmss_webapp/package.json b/SAS/TMSS/frontend/tmss_webapp/package.json
index e9cc1d244a28ffcb034292897693fb7875c7a0f9..5de0cf8841f3e116bcd8cf264c26613650b2467f 100644
--- a/SAS/TMSS/frontend/tmss_webapp/package.json
+++ b/SAS/TMSS/frontend/tmss_webapp/package.json
@@ -38,6 +38,7 @@
"react-bootstrap-datetimepicker": "0.0.22",
"react-calendar-timeline": "^0.27.0",
"react-dom": "^16.13.1",
+ "react-flatpickr": "^3.10.7",
"react-frame-component": "^4.1.2",
"react-json-to-table": "^0.1.7",
"react-json-view": "^1.19.1",
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/App.js b/SAS/TMSS/frontend/tmss_webapp/src/App.js
index 7713677141facc1f7a1263fa145af6e1ee440cef..083a3978ba07503fcc752df9f49b0e393bff6fcd 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/App.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/App.js
@@ -8,6 +8,9 @@ import {AppFooter } from './layout/components/AppFooter';
import {RoutedContent} from './routes';
import {AppBreadcrumb } from "./layout/components/AppBreadcrumb";
import {withRouter } from 'react-router';
+import handleResponse from "./response.handler"
+import { setAppGrowl } from './layout/components/AppGrowl';
+import { Growl } from 'primereact/components/growl/Growl';
import 'primeicons/primeicons.css';
import 'primereact/resources/themes/nova-light/theme.css';
@@ -154,6 +157,7 @@ class App extends Component {
//console.log(this.props);
return (
<React.Fragment>
+ <Growl ref={(el) => setAppGrowl(el)} />
<div className="App">
{/* <div className={wrapperClass} onClick={this.onWrapperClick}> */}
<div className={wrapperClass}>
@@ -197,4 +201,4 @@ class App extends Component {
}
}
-export default App;
+export default handleResponse(App);
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js
index 98d84429009f475b44411113fe6d7d6d319dcf88..50623578335782048c11ba4ff25bb4f182370119 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js
@@ -1,7 +1,11 @@
import React, { Component } from 'react';
+import Flatpickr from "react-flatpickr";
import {Calendar} from 'primereact/calendar';
import moment from 'moment';
import UIConstants from '../../utils/ui.constants';
+import UtilService from '../../services/util.service';
+
+import "flatpickr/dist/flatpickr.css";
//const DATE_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
@@ -16,9 +20,13 @@ export default class CustomDateComp extends Component {
componentDidMount(){
let parentRows = this.props.agGridReact.props.rowData[this.props.node.rowIndex];
let parentCellData = parentRows[this.props.colDef.field];
- this.setState({
- date:parentCellData
- })
+ UtilService.getUTC()
+ .then(systemTime => {
+ this.setState({
+ date:parentCellData,
+ systemTime: moment.utc(systemTime)
+ })
+ });
}
isPopup() {
@@ -33,30 +41,30 @@ export default class CustomDateComp extends Component {
}
render() {
- return (
- <Calendar
- d dateFormat = {UIConstants.CALENDAR_DATE_FORMAT}
- value= {this.state.date}
- onChange= {e => {this.updateDateChanges(e)}}
- // onBlur= {e => {this.updateDateChanges(e)}}
- //data-testid="start"
- todayButtonClassName="today-calendar-btn"
- showButtonBar
- showTime= {true}
- showSeconds= {true}
- hourFormat= "24"
- showIcon= {false} inline
- />
- );
+ return this.state.systemTime?(
+ <Flatpickr
+ data-enable-time
+ options={{
+ "inline": true,
+ "enableSeconds": true,
+ "time_24hr": true,
+ "defaultDate": this.state.systemTime?this.state.systemTime.format(UIConstants.CALENDAR_DEFAULTDATE_FORMAT):"",
+ "defaultHour": this.state.systemTime?this.state.systemTime.hours():12,
+ "defaultMinute": this.state.systemTime?this.state.systemTime.minutes():0
+ }}
+ value={this.state.date}
+ onChange= {value => {this.updateDateChanges(value[0]?value[0]:this.state.date)}}
+ />
+ ):"";
}
updateDateChanges(e){
- this.setState({date : e.value || ''});
+ this.setState({date : e || ''});
}
ondatechange(e){
- this.setState({date : e.value});
+ this.setState({date : e});
}
getDate() {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/response.handler.js b/SAS/TMSS/frontend/tmss_webapp/src/response.handler.js
new file mode 100644
index 0000000000000000000000000000000000000000..7c4da4c87de73f67983fb60f36e2c6aff269ab8d
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/response.handler.js
@@ -0,0 +1,45 @@
+import React, {useEffect} from "react";
+import axios from "axios";
+import { appGrowl } from './layout/components/AppGrowl';
+import UIConstants from './utils/ui.constants';
+import Auth from './authenticate/auth';
+/**
+ * Trigger and validate the response for https status code
+ * @param {*} Wrapped
+ * @returns
+ */
+const handleResponse= Wrapped => {
+ function HandleResponse(props) {
+ useEffect(()=>{
+ axios.interceptors.response.use(function (response) {
+ return response;
+ }, function (error) {
+ showMessage(error.response);
+ return Promise.reject(error);
+ });
+ })
+ return (
+ <Wrapped {...props} />
+ );
+ }
+
+ /**
+ * Catch relavent http status code details to show in growl
+ * @param {*} response
+ */
+ function showMessage(response) {
+ const httpStatusMsg = UIConstants.httpStatusMessages[response.status];
+ if(httpStatusMsg) {
+ appGrowl.show({severity: httpStatusMsg.severity, summary: httpStatusMsg.summary, sticky: httpStatusMsg.sticky, detail: '['+response.status+'] '+JSON.stringify(response.statusText)+ ' ['+httpStatusMsg.detail+']'});
+ } else {
+ appGrowl.show({severity: 'error', summary: 'Error', sticky: 'true', detail: '['+response.status+'] '+JSON.stringify(response.statusText)+ ' '+JSON.stringify(response.data)});
+ }
+ if (response.status === 401) {
+ Auth.logout();
+ window.location.href = "/login";
+ }
+ }
+ return HandleResponse;
+}
+
+export default handleResponse;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js
index 7ef3b05cc8bf50d554ae4f4fcd64e409b7e853c0..b64a1d66d444ac9001fe31e5ce2486d70f8c54b8 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Cycle/edit.js
@@ -318,7 +318,7 @@ export class CycleEdit extends Component {
this.saveCycleQuota(cycle);
} else {
this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to update Cycle'});
- this.setState({errors: cycle});
+ //this.setState({errors: cycle});
}
});
}
@@ -373,7 +373,7 @@ export class CycleEdit extends Component {
if (_.keys(quotaError).length === 0) {
dialog = {header: 'Success', detail: 'Cycle updated successfully.'};
} else {
- dialog = {header: 'Error', detail: 'Cycle updated successfully but resource allocation not updated properly. Try again!'};
+ dialog = {header: 'Error', detail: 'Cycle updated successfully but resource allocation not updated properly.'};
}
this.setState({dialogVisible: true, dialog: dialog});
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js
index c21f5afcd834388b99fe3836cd7d598be970a608..caf0c0b6e487bff2e139cdab5f857a9cc0bca1c5 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/create.js
@@ -325,12 +325,17 @@ export class ProjectCreate extends Component {
}
ProjectService.saveProject(this.state.project, this.defaultResourcesEnabled?projectQuota:[])
.then(project => {
+
if (project.url) {
let dialog = {};
- if (this.defaultResourcesEnabled) {
- dialog = {header: 'Success', detail: 'Project saved successfully. Do you want to create another project?'};
+ if (project.isQuotaCreated) {
+ if (this.defaultResourcesEnabled) {
+ dialog = {header: 'Success', detail: 'Project saved successfully. Do you want to create another project?'};
+ } else {
+ dialog = {header: 'Success', detail: 'Project saved successfully with default Resource allocations. Do you want to view and edit them?'};
+ }
} else {
- dialog = {header: 'Success', detail: 'Project saved successfully with default Resource allocations. Do you want to view and edit them?'};
+ dialog = {header: 'Warning', detail: 'Project saved successfully, but resource allocation not saved.'};
}
this.setState({project:project, dialogVisible: true, dialog: dialog, isDirty: false});
} else {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js
index b8bd0f3e2f9833bf6290e035240e88bad4d9695d..ac275f366da7624c2c9a2149b18690d2b9db297a 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Project/edit.js
@@ -335,7 +335,7 @@ export class ProjectEdit extends Component {
// project['archive_subdirectory'] = (project['archive_subdirectory'].substr(-1) === '/' ? project['archive_subdirectory'] : `${project['archive_subdirectory']}/`).toLowerCase();
ProjectService.updateProject(this.props.match.params.id, project)
.then(async (project) => {
- if (project && this.state.project.updated_at !== project.updated_at) {
+ if (project && project.isUpdated && this.state.project.updated_at !== project.updated_at) {
this.saveProjectQuota(project);
} else {
this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to update Project'});
@@ -381,20 +381,20 @@ export class ProjectEdit extends Component {
}
for (const projectQuota of updatingProjectQuota) {
const updatedProjectQuota = await ProjectService.updateProjectQuota(projectQuota);
- if (!updatedProjectQuota) {
+ if (!updatedProjectQuota || (updatedProjectQuota.status && updatedProjectQuota.status > 299)) {
quotaError[projectQuota.resource_type_id] = true;
}
}
for (const projectQuota of newProjectQuota) {
const createdProjectQuota = await ProjectService.saveProjectQuota(projectQuota);
- if (!createdProjectQuota) {
+ if (!createdProjectQuota || (createdProjectQuota.status && createdProjectQuota.status > 299)) {
quotaError[projectQuota.resource_type_id] = true;
}
}
if (_.keys(quotaError).length === 0) {
dialog = {header: 'Success', detail: 'Project updated successfully.'};
} else {
- dialog = {header: 'Error', detail: 'Project updated successfully but resource allocation not updated properly. Try again!'};
+ dialog = {header: 'Error', detail: 'Project updated successfully but resource allocation not updated properly.'};
}
this.setState({dialogVisible: true, dialog: dialog, isDirty: false});
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
index 35725047eaeceb182457875029cbba90b4cd7320..396b74fd9c413e4ca93798b76ba3462889ac7dd0 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
@@ -4,6 +4,7 @@ import _ from 'lodash';
import Jeditor from '../../components/JSONEditor/JEditor';
import UnitConversion from '../../utils/unit.converter';
import UIConstants from '../../utils/ui.constants';
+import UtilService from '../../services/util.service';
/* eslint-disable react-hooks/exhaustive-deps */
export default (props) => {
@@ -11,6 +12,7 @@ export default (props) => {
const { parentFunction = (editorFn) => { editorFunction = editorFn;} } = props;
const [constraintSchema, setConstraintSchema] = useState();
const [initialValue, setInitialValue] = useState();
+ const [systemTime, setSystemTime] = useState();
//SU Constraint Editor Property Order,format and validation
const configureProperties = (properties) => {
for (const propertyKey in properties) {
@@ -69,7 +71,8 @@ export default (props) => {
}
};
//DateTime flatPicker component enabled with seconds
- const setDateTimeOption = (propertyValue) => {
+ const setDateTimeOption = async(propertyValue) => {
+ const systemTime = moment.utc((await UtilService.getUTC()));
propertyValue.format = 'datetime-local';
propertyValue.validationType = 'dateTime';
propertyValue.skipFormat = true;
@@ -83,6 +86,9 @@ export default (props) => {
"enableSeconds": true,
"time_24hr": true,
"allowInput": true,
+ "defaultDate": systemTime.format(UIConstants.CALENDAR_DEFAULTDATE_FORMAT),
+ "defaultHour": systemTime.hour(),
+ "defaultMinute": systemTime.minutes()
}
};
};
@@ -101,7 +107,7 @@ export default (props) => {
} else if(definitionName === 'timewindow') {
for (let property in schema.definitions.timewindow.properties) {
if(property === 'to' || property === 'from'){
- setDateTimeOption(schema.definitions.timewindow.properties[property]);
+ // setDateTimeOption(schema.definitions.timewindow.properties[property]);
if (property === 'from') {
schema.definitions.timewindow.properties[property].propertyOrder = 1;
} else {
@@ -148,7 +154,9 @@ export default (props) => {
}
}
- const constraintStrategy = () => {
+ const constraintStrategy = async() => {
+ const currentSystemTime = moment.utc(await UtilService.getUTC())
+ setSystemTime(currentSystemTime);
// const constraintTemplate = { ...props.constraintTemplate }
const constraintTemplate = _.cloneDeep(props.constraintTemplate);
if (constraintTemplate.schema) {
@@ -196,6 +204,9 @@ export default (props) => {
if (!props.constraintTemplate) {
return;
}
+ UtilService.getUTC().then(utcTime => {
+ setSystemTime(moment.utc(utcTime));
+ });
if (props.initValue) {
modifyInitiValue();
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/excelview.schedulingset.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/excelview.schedulingset.js
index 161657340ba91b073e152f4afd04f45212a892de..6e8e565a824195df4de6e33736a6715eea03e577 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/excelview.schedulingset.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/excelview.schedulingset.js
@@ -1723,12 +1723,13 @@ export class SchedulingSetCreate extends Component {
async saveSU() {
let newSUCount = 0;
let existingSUCount = 0;
+ let isUpdated = true;
try{
this.setState({
// saveDialogVisible: false,
confirmDialogVisible: false,
showSpinner: true
- })
+ });
let newSU = this.state.schedulingUnit;
let parameters = this.state.schedulingUnitList[0]['requirements_doc'].parameters;
@@ -1911,7 +1912,10 @@ export class SchedulingSetCreate extends Component {
if(taskdata){
taskDrafts = taskdata.data.results;
}
- await ScheduleService.updateSUDraftFromObservStrategy(observStrategy, newSU, taskDrafts, this.state.tasksToUpdate, tmpStationGroups);
+ let updateSu = await ScheduleService.updateSUDraftFromObservStrategy(observStrategy, newSU, taskDrafts, this.state.tasksToUpdate, tmpStationGroups);
+ if (updateSu && !updateSu.isSUUpdated) {
+ isUpdated = false;
+ }
existingSUCount++;
}
else if (suRow.id === 0 && this.isNotEmpty(suRow.suname) && this.isNotEmpty(suRow.sudesc)){
@@ -1921,7 +1925,10 @@ export class SchedulingSetCreate extends Component {
scheduling_constraints_template_id: newSU['scheduling_constraints_template_id'],
scheduling_set_id: newSU['scheduling_set_id']
}
- await ScheduleService.saveSUDraftFromObservStrategy(observStrategy, newSchedulueUnit, newConstraint, tmpStationGroups);
+ let updateSu = await ScheduleService.saveSUDraftFromObservStrategy(observStrategy, newSchedulueUnit, newConstraint, tmpStationGroups);
+ if (updateSu && !updateSu.isSUUpdated) {
+ isUpdated = false;
+ }
newSUCount++;
}
}
@@ -1932,7 +1939,13 @@ export class SchedulingSetCreate extends Component {
this.dialogType = "success";
this.dialogHeader = "Success";
this.showIcon = true;
- this.dialogMsg = '['+newSUCount+'] Scheduling Units are created & ['+existingSUCount+'] Scheduling Units are updated successfully.';
+ if (isUpdated) {
+ this.dialogMsg = '['+newSUCount+'] Scheduling Units are created & ['+existingSUCount+'] Scheduling Units are updated successfully.';
+ } else {
+ this.dialogHeader = "Warning";
+ this.dialogMsg = '['+newSUCount+'] Scheduling Units are created & ['+existingSUCount+'] Scheduling Units are updated successfully, and there are some Schedule Unit/Task failed to create/update';
+ }
+
this.dialogContent = "";
this.onCancel = this.close;
this.onClose = this.close;
@@ -1944,6 +1957,7 @@ export class SchedulingSetCreate extends Component {
}
}catch(err){
this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to create/update Scheduling Units'});
+ this.setState({showSpinner: false});
}
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/schedulingset.create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/schedulingset.create.js
index b9273de4b8be558f5f7cd8345c1dc7152c78f017..2b34d370565a6284dde6d7b40b222befe78a564c 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/schedulingset.create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/schedulingset.create.js
@@ -5,6 +5,7 @@ import UIConstants from '../../utils/ui.constants';
import { CustomDialog } from '../../layout/components/CustomDialog';
import ScheduleService from '../../services/schedule.service';
import { Growl } from 'primereact/components/growl/Growl';
+import { appGrowl } from './../../layout/components/AppGrowl';
export class SchedulingSet extends Component {
@@ -32,13 +33,13 @@ export class SchedulingSet extends Component {
schedulingSet['generator_doc'] = {};
schedulingSet['scheduling_unit_drafts'] = [];
const suSet = await ScheduleService.saveSchedulingSet(schedulingSet);
- if (suSet.id !== null) {
- this.growl.show({severity: 'success', summary: 'Success', detail: 'Scheduling Set is created successfully.'});
+ if (suSet.id && suSet.id !== null) {
+ appGrowl.show({severity: 'success', summary: 'Success', detail: 'Scheduling Set is created successfully.'});
this.setState({suSet: suSet, dialogVisible: true, });
this.props.onCancel();
- } else {
+ } /* else {
this.growl.show({severity: 'error', summary: 'Error Occured', detail: schedulingSet.message || 'Unable to save Scheduling Set'});
- }
+ } */
}
}},
{id:"no", title: 'Cancel', callback: this.props.onCancel} ];
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/reservation.create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/reservation.create.js
index 4809304616b85fe9a91f3828a8a16f5b29f64dda..e1b884053169d5a05dd9ab001e45af3c7ae0804a 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/reservation.create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/reservation.create.js
@@ -6,6 +6,7 @@ import { Growl } from 'primereact/components/growl/Growl';
import AppLoader from '../../layout/components/AppLoader';
import PageHeader from '../../layout/components/PageHeader';
import UIConstants from '../../utils/ui.constants';
+import Flatpickr from "react-flatpickr";
import { Calendar } from 'primereact/calendar';
import { InputMask } from 'primereact/inputmask';
import { Dropdown } from 'primereact/dropdown';
@@ -18,6 +19,9 @@ import ProjectService from '../../services/project.service';
import ReservationService from '../../services/reservation.service';
import UnitService from '../../utils/unit.converter';
import Jeditor from '../../components/JSONEditor/JEditor';
+import UtilService from '../../services/util.service';
+
+import "flatpickr/dist/flatpickr.css";
/**
* Component to create a new Reservation
@@ -38,7 +42,7 @@ export class ReservationCreate extends Component {
reservation: {
name: '',
description: '',
- start_time: '',
+ start_time: null,
duration: '',
project: (props.match?props.match.params.project:null) || null,
},
@@ -78,9 +82,11 @@ export class ReservationCreate extends Component {
async initReservation() {
const promises = [ ProjectService.getProjectList(),
ReservationService.getReservationTemplates(),
+ UtilService.getUTC()
];
let emptyProjects = [{url: null, name: "Select Project"}];
Promise.all(promises).then(responses => {
+ let systemTime = moment.utc(responses[2]);
this.projects = emptyProjects.concat(responses[0]);
this.reservationTemplates = responses[1];
@@ -95,8 +101,9 @@ export class ReservationCreate extends Component {
paramsSchema: schema,
isLoading: false,
reservationTemplate: reservationTemplate,
+ systemTime: systemTime
});
- });
+ });
}
@@ -209,7 +216,6 @@ export class ReservationCreate extends Component {
}
}
}
-
this.setState({errors: errors, validFields: validFields});
if (Object.keys(validFields).length === Object.keys(this.formRules).length) {
validForm = true;
@@ -232,7 +238,7 @@ export class ReservationCreate extends Component {
}
}
- saveReservation(){
+ async saveReservation(){
let reservation = this.state.reservation;
let project = this.projects.find(project => project.name === reservation.project);
reservation['start_time'] = moment(reservation['start_time']).format(UIConstants.CALENDAR_DATETIME_FORMAT);
@@ -240,13 +246,13 @@ export class ReservationCreate extends Component {
reservation['project']= project ? project.url: null;
reservation['specifications_template']= this.reservationTemplates[0].url;
reservation['specifications_doc']= this.paramsOutput;
- reservation = ReservationService.saveReservation(reservation);
- if (reservation && reservation !== null){
+ reservation = await ReservationService.saveReservation(reservation);
+ if (reservation && reservation.id){
const dialog = {header: 'Success', detail: 'Reservation is created successfully. Do you want to create another Reservation?'};
this.setState({ dialogVisible: true, dialog: dialog, paramsOutput: {}, showDialog: false, isDirty: false})
- } else {
+ }/* else {
this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to save Reservation', showDialog: false, isDirty: false});
- }
+ }*/
}
/**
@@ -356,20 +362,27 @@ export class ReservationCreate extends Component {
<div className="p-field p-grid">
<label htmlFor="reservationName" className="col-lg-2 col-md-2 col-sm-12">From Date <span style={{color:'red'}}>*</span></label>
<div className="col-lg-3 col-md-3 col-sm-12">
- <Calendar
- d dateFormat="yy-mm-dd"
- value= {this.state.reservation.start_time}
- onChange= {e => this.setParams('start_time',e.value)}
- data-testid="start_time"
- tooltip="Moment at which the reservation starts from, that is, when its reservation can run." tooltipOptions={this.tooltipOptions}
- showIcon={true}
- showTime= {true}
- showSeconds= {true}
- hourFormat= "24"
- />
-
- <label className={this.state.errors.from?"error":"info"}>
- {this.state.errors.start_time ? this.state.errors.start_time : ""}
+ <Flatpickr data-enable-time data-input options={{
+ "inlineHideInput": true,
+ "wrap": true,
+ "enableSeconds": true,
+ "time_24hr": true,
+ "allowInput": true,
+ "defaultDate": this.state.systemTime.format(UIConstants.CALENDAR_DEFAULTDATE_FORMAT),
+ "defaultHour": this.state.systemTime.hours(),
+ "defaultMinute": this.state.systemTime.minutes()
+ }}
+ title="Start of this reservation"
+ value={this.state.reservation.start_time}
+ onChange= {value => {this.setParams('start_time', value[0]?value[0]:this.state.reservation.start_time);
+ this.setReservationParams('start_time', value[0]?value[0]:this.state.reservation.start_time)}} >
+ <input type="text" data-input className={`p-inputtext p-component ${this.state.errors.start_time && this.state.touched.start_time?'input-error':''}`} />
+ <i className="fa fa-calendar" data-toggle style={{position: "absolute", marginLeft: '-25px', marginTop:'5px', cursor: 'pointer'}} ></i>
+ <i className="fa fa-times" style={{position: "absolute", marginLeft: '-50px', marginTop:'5px', cursor: 'pointer'}}
+ onClick={e => {this.setParams('start_time', ''); this.setReservationParams('start_time', '')}}></i>
+ </Flatpickr>
+ <label className={this.state.errors.start_time && this.state.touched.start_time?"error":"info"}>
+ {this.state.errors.start_time && this.state.touched.start_time ? this.state.errors.start_time : ""}
</label>
</div>
<div className="col-lg-1 col-md-1 col-sm-12"></div>
@@ -380,6 +393,8 @@ export class ReservationCreate extends Component {
value={this.state.reservation.duration}
mask="99:99:99"
placeholder="HH:mm:ss"
+ tooltip="Duration of this reservation. If it is empty, then this reservation is indefinite."
+ tooltipOptions={this.tooltipOptions}
onChange= {e => this.setParams('duration',e.value)}
ref={input =>{this.input = input}}
/>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
index 828386d19450af0473e199ad44430a7b4491e8ed..56437953e4ee0ed1bd91c69fda2fd9512bb7f703 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
@@ -845,9 +845,9 @@ export class TimelineView extends Component {
if (this.state.redirect) {
return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
}
- if (this.state.loader) {
- return <AppLoader />
- }
+ // if (this.state.loader) {
+ // return <AppLoader />
+ // }
const isSUDetsVisible = this.state.isSUDetsVisible;
const isTaskDetsVisible = this.state.isTaskDetsVisible;
const canExtendSUList = this.state.canExtendSUList;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js
index 095f5b009d342fdc124972bcd8ece09ac4eab7d3..9e110bc09653909d2045fea90ae3b68107cefaa7 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/cycle.service.js
@@ -100,7 +100,7 @@ const CycleService = {
return response.data;
} catch (error) {
console.log(error.response.data);
- return error.response.data;
+ //return error.response.data;
}
},
deleteCycleQuota: async function(cycleQuota) {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js
index 2ad00beed53a2655373cb373486180ab375ef8a1..5613c91deba01e0375f622e7e16700d872c9a494 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/project.service.js
@@ -68,10 +68,14 @@ const ProjectService = {
saveProject: async function(project, projectQuota) {
try {
const response = await axios.post(('/api/project/'), project);
- project = response.data
- for (let quota of projectQuota) {
+ project = response.data;
+ project['isQuotaCreated'] = true;
+ for (let quota of projectQuota) {
quota.project = project.url;
- this.saveProjectQuota(quota);
+ let response = await this.saveProjectQuota(quota);
+ if (response.status > 299) {
+ project['isQuotaCreated'] = false;
+ }
}
return response.data;
} catch (error) {
@@ -83,29 +87,33 @@ const ProjectService = {
updateProject: async function(id, project) {
try {
const response = await axios.put((`/api/project/${id}/`), project);
- return response.data;
+ project = response.data;
+ project['isUpdated'] = true;
+ return project;
} catch (error) {
- // console.log(error);
console.log(error.response.data);
- return error.response.data;
+ project = error.response.data;
+ project['isUpdated'] = false;
+ return project;
}
},
saveProjectQuota: async function(projectQuota) {
try {
const response = await axios.post(('/api/project_quota/'), projectQuota);
- return response.data;
+ return response;
} catch (error) {
console.error(error);
- return null;
+ return error.response;
}
},
updateProjectQuota: async function(projectQuota) {
+ const response = null;
try {
const response = await axios.put(`/api/project_quota/${projectQuota.id}/`, projectQuota);
return response.data;
} catch (error) {
console.error(error);
- return null;
+ return response;
}
},
deleteProjectQuota: async function(projectQuota) {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
index db6284425c56b108d1b5ff8dd08a341e6a78a9ef..eb1d3364f5d71f6eb31b17ba27803daafdd6f057 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
@@ -493,9 +493,10 @@ const ScheduleService = {
updateSUDraftFromObservStrategy: async function(observStrategy,schedulingUnit,tasks,tasksToUpdate,station_groups) {
try {
delete schedulingUnit['duration'];
-
+ schedulingUnit['isSUUpdated'] = false;
schedulingUnit = await this.updateSchedulingUnitDraft(schedulingUnit);
if (!schedulingUnit.error) {
+ schedulingUnit['isSUUpdated'] = true;
for (const taskToUpdate in tasksToUpdate) {
let task = tasks.find(task => { return task.name === taskToUpdate});
task.specifications_doc = observStrategy.template.tasks[taskToUpdate].specifications_doc;
@@ -515,6 +516,7 @@ const ScheduleService = {
return schedulingUnit;
} catch(error) {
console.error(error);
+ schedulingUnit['isSUUpdated'] = false;
return {
error: true,
message: 'Unable to Update Task Drafts'
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/utils/ui.constants.js b/SAS/TMSS/frontend/tmss_webapp/src/utils/ui.constants.js
index b4ca89eb6f0b31d99092b8d978ffb1e0e9c695f3..b2cdb71562603a663bddc1420395566d4a823afb 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/utils/ui.constants.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/utils/ui.constants.js
@@ -3,6 +3,15 @@ const UIConstants = {
timeline: {
types: { NORMAL: "NORMAL", WEEKVIEW:"WEEKVIEW"}
},
+ httpStatusMessages: {
+ 400: {severity: 'error', summary: 'Error', sticky: true, detail: 'Error while process request, please contact system admin'},
+ 401: {severity: 'error', summary: 'Error', sticky: true, detail: 'Not authenticated, Please retry to login with valid credential'},
+ 403: {severity: 'error', summary: 'Error', sticky: true, detail: 'Unknown request, please contact system admin'},
+ 404: {severity: 'error', summary: 'Error', sticky: true, detail: 'URL is not recognized, please contact system admin'},
+ 408: {severity: 'error', summary: 'Error', sticky: true, detail: 'Request is taking more time to response, please contact system admin'},
+ 500: {severity: 'error', summary: 'Error', sticky: true, detail: 'Internal Server Error, URL may not exists, please contact system admin'},
+ 503: {severity: 'error', summary: 'Error', sticky: true, detail: 'Server not available, please check system admin'},
+ },
CALENDAR_DATE_FORMAT: 'yy-mm-dd',
CALENDAR_DATETIME_FORMAT : 'YYYY-MM-DD HH:mm:ss',
CALENDAR_TIME_FORMAT: 'HH:mm:ss',