TMSS-671: pass the scheduler._raise_if_triggered into dynamic scheduling...

TMSS-671: pass the scheduler._raise_if_triggered into dynamic scheduling contraint solver methods, so these can be interrupted

SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py

@@ -55,7 +55,7 @@ class ScoredSchedulingUnit():
         self.weighted_score = weighted_score
 
 
-def filter_scheduling_units_using_constraints(scheduling_units: [models.SchedulingUnitBlueprint], lower_bound: datetime, upper_bound: datetime) -> [models.SchedulingUnitBlueprint]:
+def filter_scheduling_units_using_constraints(scheduling_units: [models.SchedulingUnitBlueprint], lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> [models.SchedulingUnitBlueprint]:
     """
     Filter the given scheduling_units by whether their constraints are met within the given timewindow.
     If one or more scheduling units can run only within this time window and not after it, then only these exclusivly runnable scheduling units.
@@ -68,12 +68,14 @@ def filter_scheduling_units_using_constraints(scheduling_units: [models.Scheduli
     runnable_exclusive_in_this_window_scheduling_units = []
 
     for scheduling_unit in scheduling_units:
+        raise_if_interruped() # interrupts the scheduling loop
+
         try:
             # should not happen, these are non-nullable in db
             assert(scheduling_unit.draft is not None)
             assert(scheduling_unit.scheduling_constraints_template is not None)
 
-            if can_run_within_station_reservations(scheduling_unit) and can_run_within_timewindow(scheduling_unit, lower_bound, upper_bound):
+            if can_run_within_station_reservations(scheduling_unit) and can_run_within_timewindow(scheduling_unit, lower_bound, upper_bound, raise_if_interruped):
                 runnable_scheduling_units.append(scheduling_unit)
 
                 # if a schedulingunit cannot run after this window, then apparently its limited to run exclusively in this time window.
@@ -138,7 +140,7 @@ def sort_scheduling_units_scored_by_constraints(scheduling_units: [models.Schedu
 #                                                                                                 #
 ###################################################################################################
 
-def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     '''Check if the given scheduling_unit can run somewhere within the given time window depending on the sub's constrains-template/doc.'''
     constraints_template = scheduling_unit.scheduling_constraints_template
 
@@ -146,7 +148,7 @@ def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, l
     if constraints_template.name == 'constraints' and constraints_template.version == 1:
         # import here to prevent circular imports. Do not worry about performance loss, cause python only imports once and then uses a cache.
         from . import template_constraints_v1
-        return template_constraints_v1.can_run_within_timewindow(scheduling_unit, lower_bound, upper_bound)
+        return template_constraints_v1.can_run_within_timewindow(scheduling_unit, lower_bound, upper_bound, raise_if_interruped)
 
     # TODO: if we get more constraint templates or versions, then add a check here and import and use the new module with the constraint methods for that specific template. (strategy pattern)
 
@@ -196,7 +198,7 @@ def compute_scores(scheduling_units: [models.SchedulingUnitBlueprint], lower_bou
 
     return scored_scheduling_units
 
-def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime=None) -> datetime:
+def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime=None, raise_if_interruped=lambda: None) -> datetime:
     '''determine the earliest possible start_time for the given scheduling unit, taking into account all its constraints'''
     constraints_template = scheduling_unit.scheduling_constraints_template
 
@@ -204,7 +206,7 @@ def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBluep
     if constraints_template.name == 'constraints' and constraints_template.version == 1:
         # import here to prevent circular imports. Do not worry about performance loss, cause python only imports once and then uses a cache.
         from . import template_constraints_v1
-        return template_constraints_v1.get_earliest_possible_start_time(scheduling_unit, lower_bound)
+        return template_constraints_v1.get_earliest_possible_start_time(scheduling_unit, lower_bound, raise_if_interruped)
 
     # TODO: if we get more constraint templates or versions, then add a check here and import and use the new module with the constraint methods for that specific template. (strategy pattern)
 
@@ -212,12 +214,17 @@ def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBluep
                                     scheduling_unit.id, constraints_template.name, constraints_template.version))
 
 
-def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUnitBlueprint], lower_bound: datetime) -> datetime:
+def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUnitBlueprint], lower_bound: datetime, raise_if_interruped=lambda: None) -> datetime:
     '''deterimine the earliest possible starttime over all given scheduling units, taking into account all their constraints'''
-    try:
-        return min(get_earliest_possible_start_time(scheduling_unit, lower_bound) for scheduling_unit in scheduling_units if scheduling_unit.scheduling_constraints_template is not None)
-    except ValueError:
-        return lower_bound
+    min_earliest_possible_start_time = datetime.max
+    for scheduling_unit in scheduling_units:
+        raise_if_interruped()
+
+        if scheduling_unit.scheduling_constraints_template is not None:
+            earliest_possible_start_time = get_earliest_possible_start_time(scheduling_unit, lower_bound, raise_if_interruped)
+            if earliest_possible_start_time < min_earliest_possible_start_time:
+                min_earliest_possible_start_time = earliest_possible_start_time
+    return min_earliest_possible_start_time
 
 
 def can_run_within_station_reservations(scheduling_unit: models.SchedulingUnitBlueprint) -> bool:

SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py

@@ -38,24 +38,24 @@ from lofar.sas.tmss.tmss.exceptions import TMSSException
 
 from . import ScoredSchedulingUnit
 
-def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     '''determine if the given scheduling_unit can run withing the given timewindow evaluating all constraints from the "constraints" version 1 template'''
-    if not can_run_within_timewindow_with_time_constraints(scheduling_unit, lower_bound, upper_bound):
+    if not can_run_within_timewindow_with_time_constraints(scheduling_unit, lower_bound, upper_bound, raise_if_interruped):
         logger.info("SchedulingUnitBlueprint id=%s does not meet time constraints between %s and %s." % (scheduling_unit.id, lower_bound, upper_bound))
         return False
 
-    if not can_run_within_timewindow_with_daily_constraints(scheduling_unit, lower_bound, upper_bound):
+    if not can_run_within_timewindow_with_daily_constraints(scheduling_unit, lower_bound, upper_bound, raise_if_interruped):
         logger.info("SchedulingUnitBlueprint id=%s does not meet daily constraints between %s and %s." % (scheduling_unit.id, lower_bound, upper_bound))
         return False
 
-    if not can_run_within_timewindow_with_sky_constraints(scheduling_unit, lower_bound, upper_bound):
+    if not can_run_within_timewindow_with_sky_constraints(scheduling_unit, lower_bound, upper_bound, raise_if_interruped):
         logger.info("SchedulingUnitBlueprint id=%s does not meet sky constraints between %s and %s." % (scheduling_unit.id, lower_bound, upper_bound))
         return False
 
     return True
 
 
-def can_run_after(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime) -> bool:
+def can_run_after(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     '''Check if the given scheduling_unit can run somewhere after the given lowerbound timestamp depending on the sub's constrains-template/doc.'''
     constraints = scheduling_unit.scheduling_constraints_doc
     if constraints.get('scheduler', '') == 'dynamic':
@@ -80,7 +80,7 @@ def has_fixed_time_scheduler_constraint(scheduling_unit: models.SchedulingUnitBl
     return constraints.get('scheduler', '') == 'fixed_time'
 
 
-def can_run_within_timewindow_with_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_within_timewindow_with_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     """
     Checks whether it is possible to run the scheduling unit /somewhere/ in the given time window, considering the duration of the involved observation.
     :return: True if there is at least one possibility to place the scheduling unit in a way that all daily constraints are met over the runtime of the observation, else False.
@@ -88,6 +88,7 @@ def can_run_within_timewindow_with_daily_constraints(scheduling_unit: models.Sch
     duration = scheduling_unit.specified_main_observation_duration
     window_lower_bound = lower_bound
     while window_lower_bound + duration <= upper_bound:
+        raise_if_interruped()
         window_upper_bound = window_lower_bound + duration
         if can_run_anywhere_within_timewindow_with_daily_constraints(scheduling_unit, window_lower_bound, window_upper_bound):
             return True
@@ -96,7 +97,7 @@ def can_run_within_timewindow_with_daily_constraints(scheduling_unit: models.Sch
     return False
 
 
-def can_run_anywhere_within_timewindow_with_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_anywhere_within_timewindow_with_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     """
     Checks whether it is possible to place the scheduling unit arbitrarily in the given time window, i.e. the daily constraints must be met over the full time window.
     :return: True if all daily constraints are met over the entire time window, else False.
@@ -162,7 +163,7 @@ def can_run_anywhere_within_timewindow_with_daily_constraints(scheduling_unit: m
     return True
 
 
-def can_run_within_timewindow_with_time_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_within_timewindow_with_time_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     """
     Checks whether it is possible to run the scheduling unit /somewhere/ in the given time window,
     considering the duration of the involved observation.
@@ -184,6 +185,8 @@ def can_run_within_timewindow_with_time_constraints(scheduling_unit: models.Sche
         duration = timedelta(seconds=main_obs_task_spec.get('duration', main_obs_task_spec.get('target', {}).get('duration', 0)))
         window_lower_bound = lower_bound
         while window_lower_bound + duration <= upper_bound:
+            raise_if_interruped()
+
             window_upper_bound = window_lower_bound + duration
             if can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit, window_lower_bound, window_upper_bound):
                 return True
@@ -192,7 +195,7 @@ def can_run_within_timewindow_with_time_constraints(scheduling_unit: models.Sche
     return False
 
 
-def can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     """
     Checks whether it is possible to place the scheduling unit arbitrarily in the given time window,
     i.e. the time constraints must be met over the full time window.
@@ -249,7 +252,7 @@ def can_run_anywhere_within_timewindow_with_time_constraints(scheduling_unit: mo
     return can_run_at & can_run_before & can_run_with_after & can_run_between & can_run_not_between
 
 
-def can_run_within_timewindow_with_sky_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_within_timewindow_with_sky_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     """
     Checks whether it is possible to run the scheduling unit /somewhere/ in the given time window, considering the duration of the involved observation.
     :return: True if there is at least one possibility to place the scheduling unit in a way that all sky constraints are met over the runtime of the observation, else False.
@@ -261,14 +264,14 @@ def can_run_within_timewindow_with_sky_constraints(scheduling_unit: models.Sched
     window_lower_bound = lower_bound
     while window_lower_bound + duration <= upper_bound:
         window_upper_bound = window_lower_bound + duration
-        if can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit, window_lower_bound, window_upper_bound):
+        if can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit, window_lower_bound, window_upper_bound, raise_if_interruped):
             return True
         window_lower_bound += min(timedelta(hours=1), upper_bound - window_lower_bound)
 
     return False
 
 
-def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime) -> bool:
+def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, raise_if_interruped=lambda: None) -> bool:
     """
     Checks whether it is possible to place the scheduling unit arbitrarily in the given time window, i.e. the sky constraints must be met over the full time window.
     :return: True if all sky constraints are met over the entire time window, else False.
@@ -402,7 +405,7 @@ def get_longest_observation_task_name_from_specifications_doc(scheduling_unit: m
     return scheduling_unit.main_observation_task.name
 
 
-def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime=None) -> datetime:
+def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime=None, raise_if_interruped=lambda: None) -> datetime:
     constraints = scheduling_unit.scheduling_constraints_doc
 
     try:
@@ -430,6 +433,8 @@ def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBluep
             for station in stations:
                 sun_events = all_sun_events[station]
                 for event_name, event in list(sun_events.items()):
+                    raise_if_interruped()
+
                     if len(event) < len(timestamps):
                         logger.warning("get_earliest_possible_start_time for scheduling_unit id=%s: not all %s events could be computed for station %s lower_bound=%s.", scheduling_unit.id, event_name, station, lower_bound)
                         # use datetime.max as defaults, which have no influence on getting an earliest starttime

SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py

@@ -224,7 +224,7 @@ class Scheduler:
         # ensure upper is greater than or equal to lower
         upper_bound_stop_time = max(lower_bound_start_time, upper_bound_stop_time)
 
-        filtered_scheduling_units = filter_scheduling_units_using_constraints(scheduling_units, lower_bound_start_time, upper_bound_stop_time)
+        filtered_scheduling_units = filter_scheduling_units_using_constraints(scheduling_units, lower_bound_start_time, upper_bound_stop_time, self._raise_if_triggered)
 
         if filtered_scheduling_units:
             triggered_scheduling_units = [scheduling_unit for scheduling_unit in filtered_scheduling_units if scheduling_unit.interrupts_telescope]
@@ -315,7 +315,7 @@ class Scheduler:
                 # search again... (loop) with the remaining schedulable_units and new lower_bound_start_time
                 # it may be that in the mean time some scheduling_units are not (dynamically) schedulable anymore, fetch list again.
                 schedulable_units = get_dynamically_schedulable_scheduling_units(priority_queue=priority_queue)
-                lower_bound_start_time = get_min_earliest_possible_start_time(schedulable_units, lower_bound=lower_bound_start_time+timedelta(hours=1))
+                lower_bound_start_time = get_min_earliest_possible_start_time(schedulable_units, lower_bound_start_time+timedelta(hours=1), self._raise_if_triggered)
                 # TODO: update upper_bound_stop_time as well, stop when upper_bound_stop_time > cycle end.