SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py

@@ -41,6 +41,7 @@ from typing import NamedTuple
 
 from lofar.sas.tmss.tmss.tmssapp import models
 from lofar.sas.tmss.tmss.exceptions import *
+from lofar.sas.tmss.tmss.tmssapp.reservations import get_active_station_reservations_in_timewindow
 
 ################## main data struct and methods ##################
 
@@ -238,23 +239,7 @@ def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUni
         return lower_bound
 
 
-def get_active_station_reservations_in_timewindow(lower_bound, upper_bound):
-    """
-    Retrieve a list of all active stations reservations, which are reserved between a timewindow
-    TODO: use filter like filter(start_time__lte=upper) filter(stop_time__gte=lower)
-    BUT can not use filter of property, so find another 'fast' solution (no loop), therefore stop_time has to move to
-    to the model. See TMSS-668
-    Also move this part to other module
-    """
-    lst_active_station_reservations = []
-    reservations = models.Reservation.objects.all()
-    for station_reservation in reservations:
-        if (station_reservation.duration is not None and \
-                station_reservation.start_time < upper_bound and station_reservation.stop_time > lower_bound) \
-           or (station_reservation.duration is None and station_reservation.start_time < upper_bound):
-            lst_active_station_reservations += station_reservation.specifications_doc["resources"]["stations"]
-
-    return lst_active_station_reservations
+
 
 
 def can_run_within_station_reservations(scheduling_unit: models.SchedulingUnitBlueprint) -> bool:

SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py

@@ -283,7 +283,9 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
                     target_rise_and_set_times = coordinates_timestamps_and_stations_to_target_rise_and_set(angle1=angle1, angle2=angle2, direction_type=direction_type, timestamps=timestamps, stations=tuple(stations), angle_to_horizon=min_elevation)
                     for station, times in target_rise_and_set_times.items():
                         for i in range(len(timestamps)):
-                            if not (timestamps[i] > times[0]['rise'] and timestamps[i] < times[0]['set']):
+                            if times[0]['always_above_horizon']:
+                                continue
+                            if times[0]['always_below_horizon'] or not (timestamps[i] > times[0]['rise'] and timestamps[i] < times[0]['set']):
                                 if task['specifications_template'] == 'calibrator observation':
                                     logger.info('min_calibrator_elevation=%s constraint is not met at timestamp=%s' % (min_elevation.rad, timestamps[i]))
                                 else:

SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py

@@ -735,8 +735,11 @@ class TestSkyConstraints(unittest.TestCase):
         self.distance_mock.return_value = self.distance_data
         self.addCleanup(self.distance_patcher.stop)
         
-        self.target_rise_and_set_data = {"CS002": [{"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0)},
-                                                   {"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0)}]}
+        self.target_rise_and_set_data = {"CS002": [{"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0), "always_above_horizon": False, "always_below_horizon": False},
+                                                   {"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0), "always_above_horizon": False, "always_below_horizon": False}]}
+        self.target_rise_and_set_data_always_above = {"CS002": [{"rise": None, "set": None, "always_above_horizon": True, "always_below_horizon": False}]}
+        self.target_rise_and_set_data_always_below = {"CS002": [{"rise": None, "set": None, "always_above_horizon": False, "always_below_horizon": True}]}
+        
         self.target_rise_and_set_patcher = mock.patch('lofar.sas.tmss.services.scheduling.constraints.template_constraints_v1.coordinates_timestamps_and_stations_to_target_rise_and_set')
         self.target_rise_and_set_mock = self.target_rise_and_set_patcher.start()
         self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data
@@ -760,21 +763,45 @@ class TestSkyConstraints(unittest.TestCase):
         
     # min_target_elevation
 
-    def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_true_when_met(self):
+    def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_true(self):
+        self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data
+
         self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.1}
         self.scheduling_unit_blueprint.save()
-        timestamp = datetime(2020, 1, 1, 10, 0, 0)
+        timestamp = datetime(2020, 1, 1, 10, 0, 0)  # target sets after obs ends (mocked response)
         returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
         self.assertTrue(returned_value)
 
-    def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_false_when_not_met(self):
-        self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.2}
+    def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_when_target_always_above_returns_true(self):
+        self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data_always_above
+
+        self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.1}
         self.scheduling_unit_blueprint.save()
-        timestamp = datetime(2020, 1, 1, 11, 0, 0)
+        timestamp = datetime(2020, 1, 1, 10, 0, 0)  # target is always up (mocked response)
+        returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
+        self.assertTrue(returned_value)
+
+    def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_false(self):
+        self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data
+
+        self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.1}
+        self.scheduling_unit_blueprint.save()
+        timestamp = datetime(2020, 1, 1, 11, 0, 0)  # target sets before obs ends (mocked response)
+        returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
+        self.assertFalse(returned_value)
+
+    def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_when_target_is_always_below_returns_false(self):
+        self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data_always_below
+
+        self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.1}
+        self.scheduling_unit_blueprint.save()
+        timestamp = datetime(2020, 1, 1, 10, 0, 0)  # target is never up (mocked response)
         returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
         self.assertFalse(returned_value)
 
 
+
+
 class TestTimeConstraints(TestCase):
     """
     Tests for the time constraint checkers used in dynamic scheduling with different boundaries
@@ -1035,9 +1062,10 @@ class TestReservedStations(unittest.TestCase):
     """
 
     @staticmethod
-    def create_station_reservation(additional_name, lst_stations, start_time=datetime(2100, 1, 1, 0, 0, 0), duration=86400):
+    def create_station_reservation(additional_name, lst_stations, start_time=datetime(2100, 1, 1, 0, 0, 0),
+                                                                   stop_time=datetime(2100, 1, 2, 0, 0, 0)):
         """
-        Create a station reservation with given list of stations, start_time and duration (optional)
+        Create a station reservation with given list of stations, start_time and stop_time (optional)
         Default duration is 24 hours (defined in seconds)
         """
         reservation_template = models.ReservationTemplate.objects.get(name="resource reservation")
@@ -1048,7 +1076,7 @@ class TestReservedStations(unittest.TestCase):
                                    specifications_template=reservation_template,
                                    specifications_doc=reservation_template_spec,
                                    start_time=start_time,
-                                   duration=duration)
+                                   stop_time=stop_time)
         return res
 
     def setUp(self) -> None:
@@ -1070,8 +1098,7 @@ class TestReservedStations(unittest.TestCase):
         Set (1) reservation start_time > SUB start_time and reservation stop_time > SUB stop_time
         """
         station_reservation.start_time = self.scheduling_unit_blueprint.start_time + timedelta(minutes=5)
-        reservation_stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
-        station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+        station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
         station_reservation.save()
 
     def set_2_reservation_start_time_lt_sub_start_time_and_stop_time_lt_sub_stop_time(self, station_reservation):
@@ -1079,8 +1106,7 @@ class TestReservedStations(unittest.TestCase):
         Set (2) reservation start_time < SUB start_time and reservation stop_time < SUB stop_time
         """
         station_reservation.start_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
-        reservation_stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
-        station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+        station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
         station_reservation.save()
 
     def set_3_reservation_start_time_gt_sub_start_time_and_stop_time_lt_sub_stop_time(self, station_reservation):
@@ -1088,8 +1114,7 @@ class TestReservedStations(unittest.TestCase):
         Set (3) reservation start_time > SUB start_time and reservation stop_time < SUB stop_time
         """
         station_reservation.start_time = self.scheduling_unit_blueprint.start_time + timedelta(minutes=5)
-        reservation_stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
-        station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+        station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time - timedelta(minutes=5)
         station_reservation.save()
 
     def set_4_reservation_start_time_lt_sub_start_time_and_stop_time_gt_sub_stop_time(self, station_reservation):
@@ -1097,8 +1122,7 @@ class TestReservedStations(unittest.TestCase):
         Set (4) reservation start_time < SUB start_time and reservation stop_time > SUB stop_time
         """
         station_reservation.start_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
-        reservation_stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
-        station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+        station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
         station_reservation.save()
 
     def set_5_reservation_start_time_and_stop_time_lt_sub_start_time(self, station_reservation):
@@ -1106,8 +1130,7 @@ class TestReservedStations(unittest.TestCase):
         Set (5) reservation start_time and reservation stop_time < SUB start_time
         """
         station_reservation.start_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=60)
-        reservation_stop_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
-        station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+        station_reservation.stop_time = self.scheduling_unit_blueprint.start_time - timedelta(minutes=5)
         station_reservation.save()
 
     def set_6_reservation_start_time_and_stop_time_gt_sub_stop_time(self, station_reservation):
@@ -1115,8 +1138,7 @@ class TestReservedStations(unittest.TestCase):
         Set (6) reservation start_time and reservation stop_time > SUB stop_time
         """
         station_reservation.start_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=5)
-        reservation_stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=65)
-        station_reservation.duration = (reservation_stop_time - station_reservation.start_time).total_seconds()
+        station_reservation.stop_time = self.scheduling_unit_blueprint.stop_time + timedelta(minutes=65)
         station_reservation.save()
 
     def update_station_groups_of_scheduling_unit_blueprint(self):
@@ -1234,7 +1256,7 @@ class TestReservedStations(unittest.TestCase):
         Test with different reservation start time and NO stop_time
         start_time after SUB stop_time 'can run' all others 'can NOT run'
         """
-        reservation_two_no_duration = self.create_station_reservation("Two-NoDuration", ["CS001", "CS002"], duration=None)
+        reservation_two_no_duration = self.create_station_reservation("Two-NoDuration", ["CS001", "CS002"], stop_time=None)
         # reservation start_time > SUB start_time and < SUB stop_time
         reservation_two_no_duration.start_time = self.scheduling_unit_blueprint.start_time + timedelta(minutes=5)
         reservation_two_no_duration.save()

SAS/TMSS/backend/src/tmss/tmssapp/CMakeLists.txt

@@ -10,6 +10,7 @@ set(_py_files
     subtasks.py
     tasks.py
     conversions.py
+    reservations.py
     )
 
 python_install(${_py_files}

SAS/TMSS/backend/src/tmss/tmssapp/conversions.py

@@ -126,9 +126,10 @@ def coordinates_timestamps_and_stations_to_target_rise_and_set(angle1: float, an
     :param stations: tuple of station names, e.g. ("CS002",)
     :param angle_to_horizon: the angle between horizon and given coordinates for which rise and set times are returned
     :return A dict that maps station names to a list of dicts with rise and set times for each requested date.
+            If rise and set are None, the target is always above or below horizon, and the respective boolean is True.
         E.g.
-        {"CS002": [{"rise": datetime(2020, 1, 1, 4, 0, 0), "set": datetime(2020, 1, 1, 11, 0, 0)},
-                   {"rise": datetime(2020, 1, 2, 4, 0, 0), "set": datetime(2020, 1, 2, 11, 0, 0)}]
+        {"CS002": [{"rise": datetime(2020, 1, 1, 4, 0, 0), "set": datetime(2020, 1, 1, 11, 0, 0), "always_above_horizon": False, "always_below_horizon": False},
+                   {"rise": datetime(2020, 1, 2, 4, 0, 0), "set": datetime(2020, 1, 2, 11, 0, 0), "always_above_horizon": False, "always_below_horizon": False}]
         }
     """
     if direction_type == "J2000":
@@ -140,10 +141,29 @@ def coordinates_timestamps_and_stations_to_target_rise_and_set(angle1: float, an
         for timestamp in timestamps:
             # todo: this can probably be made faster by moving the following logic to an own function with single station/timestamp as input and putting the lru_cache on there.
             observer = create_astroplan_observer_for_station(station)
-            target_set = observer.target_set_time(target=coord, time=Time(timestamp), horizon=angle_to_horizon, which='next', n_grid_points=TARGET_SET_RISE_PRECISION)
-            target_rise = observer.target_rise_time(target=coord, time=Time(target_set), horizon=angle_to_horizon, which='previous', n_grid_points=TARGET_SET_RISE_PRECISION)
+            try:
+                target_set = observer.target_set_time(target=coord, time=Time(timestamp), horizon=angle_to_horizon, which='next', n_grid_points=TARGET_SET_RISE_PRECISION)
+                target_rise = observer.target_rise_time(target=coord, time=Time(target_set), horizon=angle_to_horizon, which='previous', n_grid_points=TARGET_SET_RISE_PRECISION)
+                return_dict.setdefault(station, []).append(
+                    {"rise": target_rise.to_datetime(),
+                     "set": target_set.to_datetime(),
+                     "always_above_horizon": False,
+                     "always_below_horizon": False})
+            except TypeError as e:
+                if "numpy.float64" in str(e):
+                    # Note: when the target is always above or below horizon, astroplan excepts with the not very
+                    #  meaningful error: 'numpy.float64' object does not support item assignment
+                    # Determine whether the target is always above or below horizon so that we can return some useful
+                    # additional info, e.g. for scheduling purposes.
+                    is_up = observer.target_is_up(target=coord, time=Time(timestamp), horizon=angle_to_horizon)
+                    return_dict.setdefault(station, []).append(
+                        {"rise": None,
+                         "set": None,
+                         "always_above_horizon": is_up,
+                         "always_below_horizon": not is_up})
+                else:
+                    raise
 
-            return_dict.setdefault(station, []).append({"rise": target_rise.to_datetime(), "set": target_set.to_datetime()})
 
     return return_dict
 

SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py

@@ -453,7 +453,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
                 ('description', models.CharField(help_text='Short description for this reservation, used in overviews', max_length=255)),
                 ('start_time', models.DateTimeField(help_text='Start of this reservation.')),
-                ('duration', models.IntegerField(help_text='Duration of this reservation (in seconds). If null, then this reservation is indefinitely.', null=True)),
+                ('stop_time', models.DateTimeField(help_text='Stop time of this reservation. If null, then this reservation is indefinitely.', null=True)),
                 ('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Properties of this reservation')),
             ],
             options={

SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py

@@ -308,7 +308,7 @@ class ProjectQuota(Model):
 
 
 class ProjectQuotaArchiveLocation(Model):
-    project_quota = ForeignKey('ProjectQuota', null=False, related_name="project_quota", on_delete=PROTECT, help_text='Project to wich this quota belongs.')
+    project_quota = ForeignKey('ProjectQuota', null=False, related_name="project_quota_archive_location", on_delete=PROTECT, help_text='The ProjectQuota for this archive location')
     archive_location = ForeignKey('Filesystem', null=False, on_delete=PROTECT, help_text='Location of an archive LTA cluster.')
 
     def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
@@ -1040,18 +1040,18 @@ class Reservation(NamedCommon):
     project = ForeignKey('Project', null=True, related_name='reservations', on_delete=CASCADE, help_text='Reservation will be accounted for this project.')
     description = CharField(max_length=255, help_text='Short description for this reservation, used in overviews')
     start_time = DateTimeField(help_text='Start of this reservation.')
-    duration = IntegerField(null=True, help_text='Duration of this reservation (in seconds). If null, then this reservation is indefinitely.')
+    stop_time = DateTimeField(null=True, help_text='Stop of this reservation. If null, then this reservation is indefinitely.')
     specifications_doc = JSONField(help_text='Properties of this reservation')
     specifications_template = ForeignKey('ReservationTemplate', on_delete=CASCADE, help_text='Schema used for specifications_doc.')
 
-    # TODO add stop_time to the model and calculate either duration or stop_time (in serializer)
-    # See TMSS-668
     @property
-    def stop_time(self) -> datetime.datetime:
-        '''The stop_time based on start_time+duration if duration is known, else None'''
-        if self.duration:
-            return self.start_time + datetime.timedelta(seconds=self.duration)
-        return None
+    def duration(self) -> int:
+        '''return the overall duration (in seconds) of this task, if stop_time in None than duration ia also None
+        '''
+        if self.stop_time:
+            return (self.stop_time - self.start_time).total_seconds()
+        else:
+            return None
 
     def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
         annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')

SAS/TMSS/backend/src/tmss/tmssapp/populate.py

@@ -133,36 +133,158 @@ def populate_test_data():
 
 
 def populate_cycles(apps, schema_editor):
-    for nr in range(0, 18):
+    #  Cycle 0 deviates from any patterns
+    cycle = models.Cycle.objects.create(name="Cycle 00",
+                                        description="Lofar Cycle 0",
+                                        start=datetime(2013, 2, 11, 0, 0, 0, 0, tzinfo=timezone.utc),
+                                        stop=datetime(2013, 11, 14, 0, 0, 0, 0, tzinfo=timezone.utc))
+
+    models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time"),
+                                                             value=0.8 * cycle.duration.total_seconds()),
+                                           # rough guess. 80% of total time available for observing
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="CEP Processing Time"),
+                                                             value=0.8 * cycle.duration.total_seconds()),
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(name="LTA Storage"),
+                                                             value=0),  # needs to be filled in by user (SOS)
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Support Time"),
+                                                             value=0),  # needs to be filled in by user (SOS)
+
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time Commissioning"),
+                                                             value=0.05 * cycle.duration.total_seconds()),
+                                           # rough guess. 5% of total time available for observing
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time prio A"),
+                                                             value=0),  # needs to be filled in by user (SOS)
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time prio B"),
+                                                             value=0)  # needs to be filled in by user (SOS)
+                                           ])
+
+    #  Cycles 1-10 follow the same pattern
+    for nr in range(1, 11):
         cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
                                             description="Lofar Cycle %s" % nr,
-                                            start=datetime(2013+nr//2, 6 if nr%2==0 else 11, 1, 0, 0, 0, 0, tzinfo=timezone.utc),
-                                            stop=datetime(2013+(nr+1)//2, 6 if nr%2==1 else 11, 1, 0, 0, 0, 0, tzinfo=timezone.utc))
+                                            start=datetime(2013+nr//2, 5 if nr%2==0 else 11, 15, 0, 0, 0, 0, tzinfo=timezone.utc),
+                                            stop=datetime(2013+(nr+1)//2, 5 if nr%2==1 else 11, 14, 0, 0, 0, 0, tzinfo=timezone.utc))
 
         models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="observing_time"),
+                                                                 resource_type=ResourceType.objects.get(name="LOFAR Observing Time"),
                                                                  value=0.8*cycle.duration.total_seconds()), # rough guess. 80% of total time available for observing
                                                models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="cep_processing_time"),
+                                                                 resource_type=ResourceType.objects.get(name="CEP Processing Time"),
                                                                  value=0.8*cycle.duration.total_seconds()),
                                                models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="lta_storage"),
+                                                                 resource_type=ResourceType.objects.get(name="LTA Storage"),
                                                                  value=0), # needs to be filled in by user (SOS)
                                                models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="support_time"),
+                                                                 resource_type=ResourceType.objects.get(name="LOFAR Support Time"),
                                                                  value=0),  # needs to be filled in by user (SOS)
 
                                                models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="observing_time_commissioning"),
+                                                                 resource_type=ResourceType.objects.get(name="LOFAR Observing Time Commissioning"),
                                                                  value=0.05*cycle.duration.total_seconds()), # rough guess. 5% of total time available for observing
                                                models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="observing_time_prio_a"),
+                                                                 resource_type=ResourceType.objects.get(name="LOFAR Observing Time prio A"),
                                                                  value=0), # needs to be filled in by user (SOS)
                                                models.CycleQuota(cycle=cycle,
-                                                                 resource_type=ResourceType.objects.get(name="observing_time_prio_b"),
+                                                                 resource_type=ResourceType.objects.get(name="LOFAR Observing Time prio B"),
                                                                  value=0) # needs to be filled in by user (SOS)
                                                ])
 
+    #  Cycle 11 deviates from any patterns
+    cycle = models.Cycle.objects.create(name="Cycle 11",
+                                        description="Lofar Cycle 11",
+                                        start=datetime(2018, 11, 15, 0, 0, 0, 0, tzinfo=timezone.utc),
+                                        stop=datetime(2019, 5, 31, 0, 0, 0, 0, tzinfo=timezone.utc))
+
+    models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time"),
+                                                             value=0.8 * cycle.duration.total_seconds()),
+                                           # rough guess. 80% of total time available for observing
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="CEP Processing Time"),
+                                                             value=0.8 * cycle.duration.total_seconds()),
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LTA Storage"),
+                                                             value=0),  # needs to be filled in by user (SOS)
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Support Time"),
+                                                             value=0),  # needs to be filled in by user (SOS)
+
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time Commissioning"),
+                                                             value=0.05 * cycle.duration.total_seconds()),
+                                           # rough guess. 5% of total time available for observing
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time prio A"),
+                                                             value=0),  # needs to be filled in by user (SOS)
+                                           models.CycleQuota(cycle=cycle,
+                                                             resource_type=ResourceType.objects.get(
+                                                                 name="LOFAR Observing Time prio B"),
+                                                             value=0)  # needs to be filled in by user (SOS)
+                                           ])
+
+    #  Cycles 12-19 follow the same pattern
+    for nr in range(12, 20):
+        cycle = models.Cycle.objects.create(name="Cycle %02d" % nr,
+                                            description="Lofar Cycle %s" % nr,
+                                            start=datetime(2013 + nr // 2, 6 if nr % 2 == 0 else 12, 1, 0, 0, 0, 0,
+                                                           tzinfo=timezone.utc),
+                                            stop=datetime(2013 + (nr + 1) // 2, 5 if nr % 2 == 1 else 11,
+                                                          30 if nr % 2 == 0 else 31, 0, 0,
+                                                          0, 0, tzinfo=timezone.utc))
+
+        models.CycleQuota.objects.bulk_create([models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="LOFAR Observing Time"),
+                                                                 value=0.8 * cycle.duration.total_seconds()),
+                                               # rough guess. 80% of total time available for observing
+                                               models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="CEP Processing Time"),
+                                                                 value=0.8 * cycle.duration.total_seconds()),
+                                               models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="LTA Storage"),
+                                                                 value=0),  # needs to be filled in by user (SOS)
+                                               models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="LOFAR Support Time"),
+                                                                 value=0),  # needs to be filled in by user (SOS)
+
+                                               models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="LOFAR Observing Time Commissioning"),
+                                                                 value=0.05 * cycle.duration.total_seconds()),
+                                               # rough guess. 5% of total time available for observing
+                                               models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="LOFAR Observing Time prio A"),
+                                                                 value=0),  # needs to be filled in by user (SOS)
+                                               models.CycleQuota(cycle=cycle,
+                                                                 resource_type=ResourceType.objects.get(
+                                                                     name="LOFAR Observing Time prio B"),
+                                                                 value=0)  # needs to be filled in by user (SOS)
+                                               ])
+
+
 def populate_projects(apps, schema_editor):
     from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingSet_test_data
 
@@ -181,7 +303,7 @@ def populate_projects(apps, schema_editor):
         # for convenience, create a schedulingset for each project
         models.SchedulingSet.objects.create(**SchedulingSet_test_data(name="Test Scheduling Set", project=tmss_project))
 
-        project_quota = ProjectQuota.objects.create(project=tmss_project, value=1e12, resource_type=ResourceType.objects.get(name="lta_storage"))
+        project_quota = ProjectQuota.objects.create(project=tmss_project, value=1e12, resource_type=ResourceType.objects.get(name="LTA Storage"))
         sara_fs = Filesystem.objects.get(name="Lofar Storage (SARA)")
         models.ProjectQuotaArchiveLocation.objects.create(project_quota=project_quota, archive_location=sara_fs)
 
@@ -191,18 +313,7 @@ def populate_resources(apps, schema_editor):
     time_q = Quantity.objects.get(value=Quantity.Choices.TIME.value)
     number_q = Quantity.objects.get(value=Quantity.Choices.NUMBER.value)
 
-    ResourceType.objects.bulk_create([ResourceType(name="lta_storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
-                                      ResourceType(name="cep_storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=bytes_q),
-                                      ResourceType(name="cep_processing_time", description="Processing time on the CEP processing cluster (in seconds)", quantity=time_q),
-                                      ResourceType(name="observing_time", description="Observing time (in seconds)", quantity=time_q),
-                                      ResourceType(name="observing_time_prio_a", description="Observing time with priority A (in seconds)", quantity=time_q),
-                                      ResourceType(name="observing_time_prio_b", description="Observing time with priority B (in seconds)", quantity=time_q),
-                                      ResourceType(name="observing_time_commissioning", description="Observing time for Commissioning/DDT (in seconds)", quantity=time_q),
-                                      ResourceType(name="support_time", description="Support time by human (in seconds)", quantity=time_q),
-                                      ResourceType(name="number_of_triggers", description="Number of trigger events (as integer)", quantity=number_q),
-                                      # TODO these duplicates have names that front-end expects.
-                                      # TODO We should not have doubles.
-                                      ResourceType(name="LTA Storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
+    ResourceType.objects.bulk_create([ResourceType(name="LTA Storage", description="Amount of storage in the LTA (in bytes)", quantity=bytes_q),
                                       ResourceType(name="CEP Storage", description="Amount of storage on the CEP processing cluster (in bytes)", quantity=bytes_q),
                                       ResourceType(name="CEP Processing Time", description="Processing time on the CEP processing cluster (in seconds)", quantity=time_q),
                                       ResourceType(name="LOFAR Observing Time", description="Observing time (in seconds)", quantity=time_q),

SAS/TMSS/backend/src/tmss/tmssapp/reservations.py

+from lofar.sas.tmss.tmss.tmssapp import models
+
+
+def get_active_station_reservations_in_timewindow(lower_bound, upper_bound):
+    """
+    Retrieve a list of all active stations reservations, which are reserved between a timewindow
+    """
+    lst_active_station_reservations = []
+    for res in models.Reservation.objects.filter(start_time__lt=upper_bound, stop_time__gt=lower_bound).values('specifications_doc'):
+        lst_active_station_reservations += res["specifications_doc"]["resources"]["stations"]
+    for res in models.Reservation.objects.filter(start_time__lt=upper_bound, stop_time=None).values('specifications_doc'):
+        lst_active_station_reservations += res["specifications_doc"]["resources"]["stations"]
+    return list(set(lst_active_station_reservations))

SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json

@@ -9,8 +9,7 @@
         "pointing": {
           "direction_type": "J2000",
           "angle1": 0,
-          "angle2": 0,
-          "angle3": 0
+          "angle2": 0
         },
         "name": "calibrator1"
       },
@@ -80,8 +79,7 @@
         "tile_beam": {
           "direction_type": "J2000",
           "angle1": 0.42,
-          "angle2": 0.43,
-          "angle3": 0.44
+          "angle2": 0.43
         },
         "SAPs": [
           {
@@ -89,8 +87,7 @@
             "digital_pointing": {
               "direction_type": "J2000",
               "angle1": 0.24,
-              "angle2": 0.25,
-              "angle3": 0.26
+              "angle2": 0.25
             },
             "subbands": [
               349,
@@ -102,8 +99,7 @@
             "digital_pointing": {
               "direction_type": "J2000",
               "angle1": 0.27,
-              "angle2": 0.28,
-              "angle3": 0.29
+              "angle2": 0.28
             },
             "subbands": [
               349,
@@ -169,8 +165,7 @@
         "pointing": {
           "direction_type": "J2000",
           "angle1": 0,
-          "angle2": 0,
-          "angle3": 0
+          "angle2": 0
         },
         "name": "calibrator2"
       },

SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-pointing-1.json

@@ -42,12 +42,6 @@
           "title": "Angle 2",
           "description": "Second angle (e.g. DEC)",
           "default": 0
-        },
-        "angle3": {
-          "type": "number",
-          "title": "Angle 3",
-          "description": "Third angle (e.g. N in LMN)",
-          "default": 0
         }
       },
       "required": [

SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_constraints_template-constraints-1.json

@@ -111,13 +111,13 @@
           "properties": {
             "from": {
               "type": "number",
-              "minimum": -0.20943951,
-              "maximum": 0.20943951
+              "minimum": -86400,
+              "maximum": 86400
             },
             "to": {
               "type": "number",
-              "minimum": -0.20943951,
-              "maximum": 0.20943951
+              "minimum": -86400,
+              "maximum": 86400
             }
           },
           "additionalProperties": false

SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json

@@ -30,8 +30,7 @@
         "tile_beam": {
           "direction_type": "J2000",
           "angle1": 5.233660650313663,
-          "angle2": 0.7109404782526458,
-          "angle3": 0
+          "angle2": 0.7109404782526458
         },
         "SAPs": [
           {
@@ -40,8 +39,7 @@
             "digital_pointing": {
               "direction_type": "J2000",
               "angle1": 5.233660650313663,
-              "angle2": 0.7109404782526458,
-              "angle3": 0
+              "angle2": 0.7109404782526458
             },
             "subbands": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243]
           }

SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-beamforming-observation-scheduling-unit-observation-strategy.json

@@ -14,8 +14,7 @@
             "digital_pointing": {
               "direction_type": "J2000",
               "angle1": 5.233660650313663,
-              "angle2": 0.7109404782526458,
-              "angle3": 0
+              "angle2": 0.7109404782526458
             },
             "subbands": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243]
           }
@@ -26,8 +25,7 @@
         "tile_beam": {
           "direction_type": "J2000",
           "angle1": 5.233660650313663,
-          "angle2": 0.7109404782526458,
-          "angle3": 0
+          "angle2": 0.7109404782526458
         },
         "beamformers": [ {} ]
       },

SAS/TMSS/backend/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json

@@ -30,8 +30,7 @@
         "tile_beam": {
           "direction_type": "J2000",
           "angle1": 5.233660650313663,
-          "angle2": 0.7109404782526458,
-          "angle3": 0
+          "angle2": 0.7109404782526458
         },
         "SAPs": [
           {
@@ -40,8 +39,7 @@
             "digital_pointing": {
               "direction_type": "J2000",
               "angle1": 5.233660650313663,
-              "angle2": 0.7109404782526458,
-              "angle3": 0
+              "angle2": 0.7109404782526458
             },
             "subbands": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243]
           }

SAS/TMSS/backend/src/tmss/tmssapp/serializers/specification.py

@@ -171,10 +171,12 @@ class ProjectSerializer(DynamicRelationalHyperlinkedModelSerializer):
 
 
 class ProjectQuotaSerializer(DynamicRelationalHyperlinkedModelSerializer):
+    project_quota_archive_location = serializers.HyperlinkedRelatedField('projectquotaarchivelocation-detail', source='*', read_only=True)
+
     class Meta:
         model = models.ProjectQuota
         fields = '__all__'
-        extra_fields = ['resource_type']
+        extra_fields = ['resource_type', 'project_quota_archive_location']
 
 
 class ProjectQuotaArchiveLocationSerializer(DynamicRelationalHyperlinkedModelSerializer):
@@ -380,7 +382,7 @@ class ReservationSerializer(DynamicRelationalHyperlinkedModelSerializer):
     class Meta:
         model = models.Reservation
         fields = '__all__'
-        extra_fields = ['stop_time']
+        extra_fields = ['duration']
 
 
 class TaskBlueprintExtendedSerializer(TaskBlueprintSerializer):

SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py

@@ -99,7 +99,7 @@ def _add_pointings(pointing_a, pointing_b):
         raise SubtaskCreationException(
             "Cannot add pointings because direction types differ pointing_a=%s; pointing_b=%s" % (pointing_a, pointing_b))
     pointing = {"direction_type": pointing_a['direction_type']}
-    for angle in ['angle1', 'angle2', 'angle3']:
+    for angle in ['angle1', 'angle2']:
         pointing[angle] = pointing_a.get(angle, 0.0) + pointing_b.get(angle, 0.0)
     return pointing
 

SAS/TMSS/backend/test/CMakeLists.txt

@@ -36,6 +36,7 @@ if(BUILD_TESTING)
     lofar_add_test(t_permissions)
     lofar_add_test(t_permissions_system_roles)
     lofar_add_test(t_complex_serializers)
+    lofar_add_test(t_reservations)
 
     set_tests_properties(t_scheduling PROPERTIES TIMEOUT 300)
     set_tests_properties(t_tmssapp_scheduling_REST_API PROPERTIES TIMEOUT 300)

SAS/TMSS/backend/test/t_adapter.py

@@ -214,7 +214,7 @@ class SIPadapterTest(unittest.TestCase):
         specifications_doc['stations']['filter'] = "HBA_210_250"
         feedback_template = models.DataproductFeedbackTemplate.objects.get(name='feedback')
         # feedback_doc = get_default_json_object_for_schema(feedback_template.schema)  # todo <- fix the default generator, for some reason it does not produce valid json here...
-        feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'angle3': 0, 'direction_type': 'J2000'}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
+        feedback_doc = {'percentage_written': 100, 'frequency': {'subbands': [156], 'central_frequencies': [33593750.0], 'channel_width': 6103.515625, 'channels_per_subband': 32}, 'time': {'start_time': '2013-02-16T17:00:00', 'duration': 5.02732992172, 'sample_width': 2.00278016}, 'antennas': {'set': 'HBA_DUAL', 'fields': [{'type': 'HBA', 'field': 'HBA0', 'station': 'CS001'}, {'type': 'HBA', 'field': 'HBA1', 'station': 'CS001'}]}, 'target': {'pointing': {'angle1': 0, 'angle2': 0, 'direction_type': 'J2000'}}, 'samples': {'polarisations': ['XX', 'XY', 'YX', 'YY'], 'type': 'float', 'bits': 32, 'writer': 'standard', 'writer_version': '2.2.0', 'complex': True}, '$schema': 'http://127.0.0.1:8001/api/schemas/dataproductfeedbacktemplate/feedback/1#'}
         for dp in specifications_doc['stations']['digital_pointings']:
             dp['subbands'] = list(range(8))
         # Create SubTask(output)

SAS/TMSS/backend/test/t_conversions.py

@@ -362,6 +362,35 @@ class UtilREST(unittest.TestCase):
                 self.assertNotEqual(rise, rise_last)
             rise_last = rise
 
+    def test_util_target_rise_and_set_detects_when_target_above_horizon(self):
+
+        # assert always below and always above are usually false
+        r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.8&timestamps=2020-01-01&horizon=0.2', auth=AUTH)
+        self.assertEqual(r.status_code, 200)
+        r_dict = json.loads(r.content.decode('utf-8'))
+        self.assertIsNotNone(r_dict['CS002'][0]['rise'])
+        self.assertIsNotNone(r_dict['CS002'][0]['set'])
+        self.assertFalse(r_dict['CS002'][0]['always_below_horizon'])
+        self.assertFalse(r_dict['CS002'][0]['always_above_horizon'])
+
+        # assert rise and set are None and flag is true when target is always above horizon
+        r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.8&timestamps=2020-01-01&horizon=0.1', auth=AUTH)
+        self.assertEqual(r.status_code, 200)
+        r_dict = json.loads(r.content.decode('utf-8'))
+        self.assertIsNone(r_dict['CS002'][0]['rise'])
+        self.assertIsNone(r_dict['CS002'][0]['set'])
+        self.assertTrue(r_dict['CS002'][0]['always_above_horizon'])
+        self.assertFalse(r_dict['CS002'][0]['always_below_horizon'])
+
+        # assert rise and set are None and flag is true when target is always below horizon
+        r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=-0.5&timestamps=2020-01-01&horizon=0.2', auth=AUTH)
+        self.assertEqual(r.status_code, 200)
+        r_dict = json.loads(r.content.decode('utf-8'))
+        self.assertIsNone(r_dict['CS002'][0]['rise'])
+        self.assertIsNone(r_dict['CS002'][0]['set'])
+        self.assertFalse(r_dict['CS002'][0]['always_above_horizon'])
+        self.assertTrue(r_dict['CS002'][0]['always_below_horizon'])
+
 
 if __name__ == "__main__":
     os.environ['TZ'] = 'UTC'