diff --git a/SAS/DataManagement/Cleanup/CleanupService/service.py b/SAS/DataManagement/Cleanup/CleanupService/service.py
index 4234cff4cbd0fbccb4902fd66c1ea1243e4b098c..f43b5438fb1aa8780ae6719c98b9984caae31bd5 100644
--- a/SAS/DataManagement/Cleanup/CleanupService/service.py
+++ b/SAS/DataManagement/Cleanup/CleanupService/service.py
@@ -482,7 +482,7 @@ class TMSSEventMessageHandlerForCleanup(TMSSEventMessageHandler):
# when an ingest subtask finishes, then it is safe for the related cleanup subtask(s) to be started
subtasks = self._tmss_client.get_subtasks_in_same_scheduling_unit(subtask)
ingest_subtasks = [s for s in subtasks if s['subtask_type'] == 'ingest']
- unfinished_ingest_subtasks = [s for s in ingest_subtasks if s['state_value'] != 'finished']
+ unfinished_ingest_subtasks = [s for s in ingest_subtasks if s['state_value'] != 'finished' and s['obsolete_since'] is None]
if len(unfinished_ingest_subtasks) > 0:
logger.info("cleanup subtask id=%s is scheduled, but waiting for ingest id=%s to finish before queueing the cleanup subtask...",
diff --git a/SAS/TMSS/backend/services/scheduling/lib/constraints.py b/SAS/TMSS/backend/services/scheduling/lib/constraints.py
index 84003a51f427f43e3fedede05550957aded13f8e..5a65a637b02c19b97d9017f2f8176fea7ed9b995 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/constraints.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/constraints.py
@@ -40,7 +40,7 @@ from lofar.sas.tmss.tmss.tmssapp.conversions import *
from lofar.common.util import noop
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.exceptions import *
-from lofar.sas.tmss.tmss.tmssapp.subtasks import enough_stations_available
+from lofar.sas.tmss.tmss.tmssapp.subtasks import enough_stations_available, get_missing_stations
from lofar.sas.tmss.tmss.tmssapp.tasks import mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable
import logging
@@ -379,7 +379,7 @@ def sort_scheduling_units_scored_by_constraints(scheduling_units: [models.Schedu
x.scheduling_unit.created_at), reverse=True)
-def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, gridder: Gridder=None) -> bool:
+def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> bool:
'''determine if the given scheduling_unit can run withing the given timewindow evaluating all constraints from the "constraints" version 1 template
:param raise_if_interruped: a callable function which raises under an externally set condition (an 'interrupt' flag was set). This function is/can_be used to interrupt a long-running scheduling call to do an early exit and start a new scheduling call. Default used function is noop (no-operation), thus no interruptable behaviour.
'''
@@ -387,7 +387,7 @@ def can_run_within_timewindow(scheduling_unit: models.SchedulingUnitBlueprint, l
gridder = Gridder()
# Seek the earliest_possible_start_time. If existing and within window, then the unit can run within this window
- earliest_possible_start_time = get_earliest_possible_start_time(scheduling_unit, lower_bound, upper_bound, gridder)
+ earliest_possible_start_time = get_earliest_possible_start_time(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped)
if earliest_possible_start_time is not None:
earliest_possible_stop_time = earliest_possible_start_time + scheduling_unit.specified_main_observation_duration
if earliest_possible_start_time >= lower_bound and earliest_possible_stop_time <= upper_bound:
@@ -994,7 +994,7 @@ def evaluate_sky_min_distance_constraint(scheduling_unit: models.SchedulingUnitB
return result
@lru_cache(maxsize=10000)
-def get_earliest_possible_start_time_for_sky_min_elevation(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=Gridder()) -> datetime:
+def get_earliest_possible_start_time_for_sky_min_elevation(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=Gridder(), raise_if_interruped: Callable=noop) -> datetime:
# do expensive search from lower_bound until 24 hours later with small steps
# (sky constrains are (almost) cyclic over 24 hours).
# first occurrence where min_elevation constraint is met is taken as rough estimate of earliest_possible_start_time
@@ -1004,6 +1004,8 @@ def get_earliest_possible_start_time_for_sky_min_elevation(scheduling_unit: mode
upper_bound = lower_bound + timedelta(hours=24)
upper_bound = max(lower_bound, upper_bound)
while possible_start_time < upper_bound:
+ raise_if_interruped()
+
result = evaluate_sky_min_elevation_constraint(scheduling_unit, possible_start_time, gridder=gridder)
logger.debug('get_earliest_possible_start_time_for_sky_min_elevation %s', result)
@@ -1031,7 +1033,7 @@ def get_earliest_possible_start_time_for_sky_min_elevation(scheduling_unit: mode
return None
-def get_earliest_possible_start_time_for_sky_transit_offset(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None) -> datetime:
+def get_earliest_possible_start_time_for_sky_transit_offset(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> datetime:
# compute the transit time, and thus the optimal_start_time and earliest_possible_start_time
if gridder is None:
gridder = Gridder()
@@ -1044,6 +1046,8 @@ def get_earliest_possible_start_time_for_sky_transit_offset(scheduling_unit: mod
allow_quick_jump = True # see below, we can quick jump once, but use monotonous increments so ensure an exit of the while loop.
while possible_start_time < upper_bound:
+ raise_if_interruped()
+
gridded_possible_start_time = gridder.grid_time(possible_start_time)
result = evaluate_sky_transit_constraint(scheduling_unit, gridded_possible_start_time, gridder=gridder, which='next')
logger.debug('get_earliest_possible_start_time_for_sky_transit_offset %s', result)
@@ -1068,7 +1072,7 @@ def get_earliest_possible_start_time_for_sky_transit_offset(scheduling_unit: mod
return None
@lru_cache(maxsize=10000)
-def get_earliest_possible_start_time_for_sky_min_distance(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None) -> datetime:
+def get_earliest_possible_start_time_for_sky_min_distance(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> datetime:
# do expensive search from lower_bound until 24 hours later with small steps
# (sky constrains are (almost) cyclic over 24 hours).
# first occurrence where min_distance constraint is met is taken as rough estimate of earliest_possible_start_time
@@ -1081,6 +1085,8 @@ def get_earliest_possible_start_time_for_sky_min_distance(scheduling_unit: model
upper_bound = max(lower_bound, upper_bound)
while possible_start_time < upper_bound:
+ raise_if_interruped()
+
result = evaluate_sky_min_distance_constraint(scheduling_unit, possible_start_time, gridder=gridder)
logger.debug('get_earliest_possible_start_time_for_sky_min_distance %s', result)
@@ -1247,7 +1253,7 @@ def evaluate_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint,
return result
@lru_cache(maxsize=10000)
-def get_earliest_possible_start_time_for_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None) -> datetime:
+def get_earliest_possible_start_time_for_daily_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> datetime:
# search from lower_bound until 24 hours later with 6 hour steps
# (daily constrains are (almost) cyclic over 24 hours)
if gridder is None:
@@ -1255,28 +1261,35 @@ def get_earliest_possible_start_time_for_daily_constraints(scheduling_unit: mode
gridded_lower_bound = gridder.grid_time(lower_bound)
possible_start_time = gridded_lower_bound
while possible_start_time < lower_bound+timedelta(hours=24):
+ raise_if_interruped()
+
result = evaluate_daily_constraints(scheduling_unit, possible_start_time, gridder=gridder)
logger.debug('get_earliest_possible_start_time_for_daily_constraints %s', result)
if not result.has_constraint:
return None
- if result.earliest_possible_start_time is None or result.earliest_possible_start_time < lower_bound:
- # advance with a grid step, and evaluate again
- possible_start_time += gridder.as_timedelta()
- continue
-
if not result.is_constraint_met and result.earliest_possible_start_time is not None:
# advance straight to earliest_possible_start_time, and evaluate again to ensure the constraint is met
possible_start_time = gridder.grid_time(result.earliest_possible_start_time)
continue
if result.is_constraint_met:
- return result.earliest_possible_start_time
+ logger.debug('get_earliest_possible_start_time_for_daily_constraints(id=%s, lb=%s, up=%s) result=%s', scheduling_unit.id, lower_bound, upper_bound, result)
+ if result.earliest_possible_start_time >= lower_bound:
+ if upper_bound is None or result.earliest_possible_start_time < upper_bound:
+ return result.earliest_possible_start_time
+ else:
+ # do not advance past upper_bound
+ return None
+
+ # advance with a grid step, and evaluate again
+ possible_start_time += gridder.as_timedelta()
+
return None
-def get_earliest_possible_start_time_for_time_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None) -> datetime:
+def get_earliest_possible_start_time_for_time_constraints(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> datetime:
'''
'''
constraints = scheduling_unit.scheduling_constraints_doc
@@ -1292,10 +1305,11 @@ def get_earliest_possible_start_time_for_time_constraints(scheduling_unit: model
if 'after' in constraints['time']:
after = parser.parse(constraints['time']['after'], ignoretz=True)
- if lower_bound is not None:
- earliest_possible_start_times.add(max(lower_bound, after))
- else:
- earliest_possible_start_times.add(after)
+ if upper_bound is None or after <= upper_bound - scheduling_unit.specified_main_observation_duration:
+ if lower_bound is not None:
+ earliest_possible_start_times.add(max(lower_bound, after))
+ else:
+ earliest_possible_start_times.add(after)
if 'before' in constraints['time']:
before = parser.parse(constraints['time']['before'], ignoretz=True)
@@ -1329,6 +1343,10 @@ def get_earliest_possible_start_time_for_time_constraints(scheduling_unit: model
earliest_possible_start_times.add(potential_earliest_possible)
break
+ if not earliest_possible_start_times and not constraints.get('time'):
+ # an empty time constraint means it can just run at/after lower_bound
+ return lower_bound
+
if lower_bound is not None:
earliest_possible_start_times = [t for t in earliest_possible_start_times if t >= lower_bound]
@@ -1338,7 +1356,7 @@ def get_earliest_possible_start_time_for_time_constraints(scheduling_unit: model
if earliest_possible_start_times:
return max(earliest_possible_start_times)
- return lower_bound
+ return None
def get_at_constraint_timestamp(scheduling_unit: models.SchedulingUnitBlueprint) -> datetime:
@@ -1348,7 +1366,7 @@ def get_at_constraint_timestamp(scheduling_unit: models.SchedulingUnitBlueprint)
return at
return None
-def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None) -> datetime:
+def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> datetime:
'''
'''
_method_start_timestamp = datetime.utcnow()
@@ -1372,9 +1390,11 @@ def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBluep
get_earliest_possible_start_time_for_sky_transit_offset,
get_earliest_possible_start_time_for_sky_min_distance):
try:
- earliest_possible_start_time = get_earliest_possible_start_time_method(scheduling_unit, lower_bound, upper_bound, gridder)
+ earliest_possible_start_time = get_earliest_possible_start_time_method(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped)
if earliest_possible_start_time is not None:
earliest_possible_start_times.add(earliest_possible_start_time)
+ except SchedulerInterruptedException:
+ raise
except Exception as e:
logger.exception(e)
@@ -1574,7 +1594,7 @@ def compute_scheduling_unit_scores(scheduling_unit: models.SchedulingUnitBluepri
# return the actual (not the gridded) weighted_start_time
start_time=weighted_start_time)
-def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUnitBlueprint], lower_bound: datetime, upper_bound: datetime=None, raise_if_interruped: Callable=noop, gridder: Gridder=None) -> datetime:
+def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUnitBlueprint], lower_bound: datetime, upper_bound: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> datetime:
'''deterimine the earliest possible starttime over all given scheduling units, taking into account all their constraints
:param raise_if_interruped: a callable function which raises under an externally set condition (an 'interrupt' flag was set). This function is/can_be used to interrupt a long-running scheduling call to do an early exit and start a new scheduling call. Default used function is noop (no-operation), thus no interruptable behaviour.
'''
@@ -1585,7 +1605,7 @@ def get_min_earliest_possible_start_time(scheduling_units: [models.SchedulingUni
raise_if_interruped()
if scheduling_unit.scheduling_constraints_template is not None:
- earliest_possible_start_time = get_earliest_possible_start_time(scheduling_unit, lower_bound, upper_bound, gridder)
+ earliest_possible_start_time = get_earliest_possible_start_time(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped)
if earliest_possible_start_time is not None:
if min_earliest_possible_start_time is None or earliest_possible_start_time < min_earliest_possible_start_time:
min_earliest_possible_start_time = earliest_possible_start_time
@@ -1606,40 +1626,74 @@ def can_run_within_station_reservations(scheduling_unit: models.SchedulingUnitBl
return True
-def determine_unschedulable_reason_and_mark_unschedulable_if_needed(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, gridder: Gridder) -> models.SchedulingUnitBlueprint:
+def get_missing_stations_for_scheduling_unit(scheduling_unit: models.SchedulingUnitBlueprint) -> []:
+ observation_subtasks = models.Subtask.independent_subtasks().filter(task_blueprint__scheduling_unit_blueprint_id=scheduling_unit.id).filter(specifications_template__type__value=models.SubtaskType.Choices.OBSERVATION.value).all()
+ missing_stations = set()
+
+ for subtask in observation_subtasks:
+ for station in get_missing_stations(subtask):
+ missing_stations.add(station)
+
+ return sorted((list(missing_stations)))
+
+def determine_unschedulable_reason_and_mark_unschedulable_if_needed(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, gridder: Gridder, raise_if_interruped: Callable=noop) -> models.SchedulingUnitBlueprint:
try:
+ logger.debug("determine_unschedulable_reason_and_mark_unschedulable_if_needed: scheduling_unit id=%s", scheduling_unit.id)
+ if not can_run_within_station_reservations(scheduling_unit):
+ missing_stations = get_missing_stations_for_scheduling_unit(scheduling_unit)
+ msg = "Stations %s are reserved" % (missing_stations, )
+ return mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(scheduling_unit, msg)
+
if gridder is None:
gridder = Gridder()
- # this method relies on caching in the used methods in order to be fast.
- # check if the unit can run at all in the given window
- if not can_run_within_timewindow(scheduling_unit, lower_bound, upper_bound, gridder):
+
+ if not can_run_within_timewindow(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped):
# nope, can't run, so check each type of constraint
+ unmet_constraints = []
- at = get_at_constraint_timestamp(scheduling_unit)
- if at:
- lower_bound = at
- upper_bound = at
+ if 'time' in scheduling_unit.scheduling_constraints_doc:
+ if scheduling_unit.scheduling_constraints_doc['time'].get('between', []):
+ # recurse for each of the 'between' intervals until unschedulable
+ for between in scheduling_unit.scheduling_constraints_doc['time']['between']:
+ between_from = parser.parse(between["from"], ignoretz=True)
+ between_to = parser.parse(between["to"], ignoretz=True)
+ if between_from != lower_bound or between_to != upper_bound:
+ scheduling_unit = determine_unschedulable_reason_and_mark_unschedulable_if_needed(scheduling_unit, between_from, between_to, gridder)
+ if scheduling_unit.status.value == models.SchedulingUnitStatus.Choices.UNSCHEDULABLE.value:
+ return scheduling_unit
+
+ # check 'at' constraint
+ at = get_at_constraint_timestamp(scheduling_unit)
+ if at:
+ if at < lower_bound or at + scheduling_unit.specified_main_observation_duration > upper_bound:
+ msg = "constraint time.at='%s' falls outside of window ['%s', '%s']" % (round_to_second_precision(at),
+ round_to_second_precision(lower_bound),
+ round_to_second_precision(upper_bound))
+ return mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(scheduling_unit, msg)
+
+ # use the 'at' timestamp as bounds for the remaining checks below
+ lower_bound = at
+ upper_bound = at
- unmet_constraints = []
if 'sky' in scheduling_unit.scheduling_constraints_doc:
if 'min_elevation' in scheduling_unit.scheduling_constraints_doc['sky']:
- if get_earliest_possible_start_time_for_sky_min_elevation(scheduling_unit, lower_bound, upper_bound, gridder) is None:
- unmet_constraints.append("sky min elevation")
+ if get_earliest_possible_start_time_for_sky_min_elevation(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped) is None:
+ unmet_constraints.append("sky min_elevation")
if 'transit_offset' in scheduling_unit.scheduling_constraints_doc['sky']:
- if get_earliest_possible_start_time_for_sky_transit_offset(scheduling_unit, lower_bound, upper_bound, gridder) is None:
- unmet_constraints.append("sky transit offset")
+ if get_earliest_possible_start_time_for_sky_transit_offset(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped) is None:
+ unmet_constraints.append("sky transit_offset")
if 'min_distance' in scheduling_unit.scheduling_constraints_doc['sky']:
- if get_earliest_possible_start_time_for_sky_min_distance(scheduling_unit, lower_bound, upper_bound, gridder) is None:
- unmet_constraints.append("sky min distance")
+ if get_earliest_possible_start_time_for_sky_min_distance(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped) is None:
+ unmet_constraints.append("sky min_distance")
if 'time' in scheduling_unit.scheduling_constraints_doc:
- if get_earliest_possible_start_time_for_time_constraints(scheduling_unit, lower_bound, upper_bound, gridder) is None:
+ if get_earliest_possible_start_time_for_time_constraints(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped) is None:
unmet_constraints.append("time")
if 'daily' in scheduling_unit.scheduling_constraints_doc:
- if get_earliest_possible_start_time_for_daily_constraints(scheduling_unit, lower_bound, upper_bound, gridder) is None:
+ if get_earliest_possible_start_time_for_daily_constraints(scheduling_unit, lower_bound, upper_bound, gridder, raise_if_interruped) is None:
unmet_constraints.append("daily")
if unmet_constraints:
@@ -1649,10 +1703,13 @@ def determine_unschedulable_reason_and_mark_unschedulable_if_needed(scheduling_u
else:
mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(scheduling_unit, "sorry, unknown unschedulable reason.")
+ except SchedulerInterruptedException:
+ raise
except Exception as e:
logger.exception(e)
mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(scheduling_unit, str(e))
+ scheduling_unit.refresh_from_db()
return scheduling_unit
diff --git a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
index 3cf7383f932d42d00476c4c7275ddc57ce0cad43..831746b088bc57b819e627cbe62ea6e1cdec4b4c 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
@@ -257,13 +257,13 @@ class Scheduler:
logger.error(e)
mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(schedulable_unit, reason=str(e))
else:
- msg = "fixed_time-scheduled scheduling unit id=%d cannot be scheduled at '%s'" % (schedulable_unit.id, start_time)
- logger.warning(msg)
- mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(schedulable_unit, reason=msg)
+ logger.warning("fixed_time-scheduled scheduling unit id=%d cannot be scheduled at '%s'", schedulable_unit.id, start_time)
+ determine_unschedulable_reason_and_mark_unschedulable_if_needed(schedulable_unit, start_time, start_time)
self.log_schedule(log_level=logging.DEBUG)
except Exception as e:
logger.exception("Could not schedule fixed_time-scheduled scheduling unit id=%d: %s", schedulable_unit.id, e)
+ mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(schedulable_unit, reason=str(e))
else:
logger.info("there are no schedulable scheduling units with fixed_time at constraint for active projects to schedule")
@@ -342,7 +342,7 @@ class Scheduler:
logger.info("find_best_next_schedulable_unit: units meeting constraints in window ['%s', '%s']: %s", lower_bound_start_time, upper_bound_stop_time, ','.join([str(su.id) for su in sorted(filtered_scheduling_units, key=lambda x: x.id)]) or 'None')
if not filtered_scheduling_units:
- logger.warning("find_best_next_schedulable_unit: no units meeting constraints in window ['%s', '%s']", lower_bound_start_time, upper_bound_stop_time)
+ logger.info("find_best_next_schedulable_unit: no units meeting constraints in window ['%s', '%s']", lower_bound_start_time, upper_bound_stop_time)
return None
# then, check if there is a subset that can only run exclusively in this window and not later.
@@ -484,14 +484,14 @@ class Scheduler:
# nothing was found, or an error occurred.
# it may be that in the mean time some scheduling_units are not (dynamically) schedulable anymore, filter those out.
for su in candidate_units:
- determine_unschedulable_reason_and_mark_unschedulable_if_needed(su, lower_bound_start_time, upper_bound_stop_time, self.search_gridder)
+ determine_unschedulable_reason_and_mark_unschedulable_if_needed(su, lower_bound_start_time, upper_bound_stop_time, self.search_gridder, raise_if_interruped=self._raise_if_triggered)
# all units are refreshed and either schedulable or unschedulable.
# refresh list of schedulable_units to be considered in next round (only schedulable)
candidate_units = [su for su in candidate_units if su.status.value==models.SchedulingUnitStatus.Choices.SCHEDULABLE.value]
# advance the window
- min_earliest_possible_start_time = get_min_earliest_possible_start_time(candidate_units, lower_bound_start_time+timedelta(hours=1), lower_bound_start_time+timedelta(hours=25), self._raise_if_triggered, gridder=self.search_gridder)
+ min_earliest_possible_start_time = get_min_earliest_possible_start_time(candidate_units, lower_bound_start_time+timedelta(hours=1), lower_bound_start_time+timedelta(hours=25), gridder=self.search_gridder, raise_if_interruped=self._raise_if_triggered)
if min_earliest_possible_start_time is None:
lower_bound_start_time += timedelta(hours=6)
else:
@@ -620,7 +620,7 @@ class Scheduler:
scheduling_units.remove(placed_B_unit)
else:
# search again in a later timeslot
- min_earliest_possible_start_time = get_min_earliest_possible_start_time(scheduling_units, lower_bound_start_time+timedelta(minutes=60), lower_bound_start_time+timedelta(hours=25), self._raise_if_triggered, gridder=self.search_gridder)
+ min_earliest_possible_start_time = get_min_earliest_possible_start_time(scheduling_units, lower_bound_start_time+timedelta(minutes=60), lower_bound_start_time+timedelta(hours=25), gridder=self.search_gridder, raise_if_interruped=self._raise_if_triggered)
logger.info("lower_bound_start_time='%s', min_earliest_possible_start_time='%s'", lower_bound_start_time, min_earliest_possible_start_time)
if min_earliest_possible_start_time is not None and min_earliest_possible_start_time > lower_bound_start_time:
lower_bound_start_time = min_earliest_possible_start_time
diff --git a/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py b/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
index e00056c44dcb6152e5bcca55500ce2d8374cdea2..c9bd0d7660056b7a62087d4ea0bceb1c9e17d649 100755
--- a/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
@@ -98,7 +98,8 @@ class BaseDynamicSchedulingTestCase(unittest.TestCase):
weight_factor.weight = 0
weight_factor.save()
- def clean_environment(self):
+ @staticmethod
+ def clean_environment():
# wipe all scheduling_unit_drafts in between tests, so the tests don't influence each other
tmss_test_env.delete_scheduling_unit_drafts_cascade()
models.Reservation.objects.all().delete()
@@ -1508,6 +1509,58 @@ class TestDynamicScheduling(BaseDynamicSchedulingTestCase):
# this test passes when there are no failed templates
self.assertTrue(len(failed_templates)==0, msg='failed_templates: %s' % ([(t.name, t.version) for t in failed_templates]))
+ def test_unschedulable_reasons_due_to_unmet_constraints(self):
+ """
+ Test if the correct unschedulable_reason is set when a constraint is not met.
+ """
+ # use a short cycle, far in the future, for which we known how the target behaves in this period (transit, elevation, etc)
+ cycle = models.Cycle.objects.create(**Cycle_test_data(start=datetime(2030, 1, 1), stop=datetime(2030, 1, 7)))
+ project = models.Project.objects.create(**Project_test_data(name=str(uuid.uuid4()), project_state=models.ProjectState.objects.get(value=models.ProjectState.Choices.ACTIVE.value)))
+ project.cycles.add(cycle)
+ project.save()
+ scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data(project=project))
+ scheduling_unit_draft = self.create_simple_observation_scheduling_unit(str(uuid.uuid4()), scheduling_set=scheduling_set, obs_duration=3600)
+
+ # keep matters simple, use one station
+ obs_task_draft = scheduling_unit_draft.task_drafts.first()
+ obs_task_draft.specifications_doc['station_configuration']['station_groups'][0]['stations'] = ['CS002']
+ obs_task_draft.save()
+
+ scheduling_unit_blueprint = create_scheduling_unit_blueprint_and_tasks_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+
+ # set density_vs_optimal to 1, meaning "as close to transit as possible"
+ weight_factor, created = models.SchedulingConstraintsWeightFactor.objects.get_or_create(scheduling_constraints_template=models.SchedulingConstraintsTemplate.get_latest(name="constraints"), constraint_name="density_vs_optimal")
+ weight_factor.weight = 0
+ weight_factor.save()
+
+ for expected_reason, constraints in (("constraint time.at='2029-12-31 00:00:00' falls outside of window ['2030-01-01 00:00:00', '2030-01-08 00:00:00']", {'time': {'at': (cycle.start - timedelta(hours=24)).isoformat()}}),
+ ("time constraint is not met anywhere between 2030-01-01 00:00:00 and 2030-01-08 00:00:00", {'time': {'before': (cycle.start - timedelta(hours=24)).isoformat()}}),
+ ("time constraint is not met anywhere between 2030-01-01 00:00:00 and 2030-01-08 00:00:00", {'time': {'after': (cycle.stop + timedelta(hours=24)).isoformat()}}),
+ ("sky min_elevation constraint is not met anywhere between 2030-01-01 00:00:00 and 2030-01-08 00:00:00", {'sky': {'min_elevation': {'target': 1.57}}}),
+ ("sky min_distance constraint is not met anywhere between 2030-01-01 00:00:00 and 2030-01-08 00:00:00", {'sky': {'min_distance': {'sun': 1.57}}}),
+ ("sky transit_offset constraint is not met anywhere between 2030-01-01 00:00:00 and 2030-01-01 07:00:00", {'sky': {'transit_offset': {'from': -900, 'to': 900}}, 'time': {'between': [{'from': '2030-01-01T00:00:00Z', 'to': '2030-01-01T07:00:00Z'}]}}),
+ ("daily constraint is not met anywhere between 2030-01-01 00:00:00 and 2030-01-01 07:00:00", {'daily': {'require_day': True }, 'time': {'between': [{'from': '2030-01-01T00:00:00Z', 'to': '2030-01-01T07:00:00Z'}]}}),
+ ("daily constraint is not met anywhere between 2030-01-01 11:00:00 and 2030-01-01 15:00:00", {'daily': {'require_night': True }, 'time': {'between': [{'from': '2030-01-01T11:00:00Z', 'to': '2030-01-01T15:00:00Z'}]}}),
+ ("daily constraint is not met anywhere between 2030-01-01 07:00:00 and 2030-01-01 09:00:00", {'daily': {'avoid_twilight': True }, 'time': {'between': [{'from': '2030-01-01T07:00:00Z', 'to': '2030-01-01T09:00:00Z'}]}}),
+ ):
+ # reset unit...
+ mark_independent_subtasks_in_scheduling_unit_blueprint_as_schedulable(scheduling_unit_blueprint)
+ self.assertEqual('', scheduling_unit_blueprint.unschedulable_reason)
+ wipe_evaluate_constraints_caches()
+
+ # set the constraints
+ scheduling_unit_blueprint.scheduling_constraints_doc = {'scheduler': 'dynamic', **constraints}
+ scheduling_unit_blueprint.save()
+
+ # try to schedule, should fail.
+ scheduled_scheduling_units = self.scheduler.do_dynamic_schedule()
+ self.assertEqual(0, len(scheduled_scheduling_units))
+
+ # Assert the scheduling_unit has not been scheduled and that it has the correct expected unschedulable_reason
+ scheduling_unit_blueprint.refresh_from_db()
+ self.assertEqual('unschedulable', scheduling_unit_blueprint.status.value)
+ self.assertEqual(expected_reason, scheduling_unit_blueprint.unschedulable_reason)
+
class TestReservedStationsTimeWindows(BaseDynamicSchedulingTestCase):
"""
@@ -1525,6 +1578,8 @@ class TestReservedStationsTimeWindows(BaseDynamicSchedulingTestCase):
"""
@classmethod
def setUpClass(cls) -> None:
+ cls.clean_environment()
+
super().setUpClass()
# create a three re-usable variants scheduling_unit_blueprint, based on the "IM HBA - 1 Beam" strategy
@@ -1830,6 +1885,37 @@ class TestReservedStationsTimeWindows(BaseDynamicSchedulingTestCase):
start_time=self.scheduling_unit_blueprint.scheduled_start_time)
self.assertTrue(can_run_within_station_reservations(self.scheduling_unit_blueprint))
+ def test_unschedulable_reason_when_reserved_stations_block_unit(self):
+ """
+ Test station reservation when 2 station (CS001,CS002) are reserved and stations (CS001, CS002) are used in scheduling_unit
+ """
+ reservation_two = self.create_station_reservation("Two", ["CS001", "CS002"])
+ # reservation start_time > SUB start_time and reservation stop_time > SUB stop_time
+ self.set_1_reservation_start_time_gt_sub_start_time_and_stop_time_gt_sub_stop_time(reservation_two, self.scheduling_unit_blueprint_cs001_cs002)
+
+ # set the constraints for scheduling_unit_blueprint_cs001_cs002
+ self.scheduling_unit_blueprint_cs001_cs002.scheduling_constraints_doc['scheduler'] = 'dynamic'
+ self.scheduling_unit_blueprint_cs001_cs002.scheduling_constraints_doc['time'] = {
+ 'between': [ {"from": reservation_two.start_time.isoformat(),
+ "to": reservation_two.stop_time.isoformat()} ]}
+ self.scheduling_unit_blueprint_cs001_cs002.save()
+
+ # mark the other units as fixed time, so they won't interfere
+ self.scheduling_unit_blueprint.scheduling_constraints_doc = {'scheduler': 'fixed_time', 'time': {'at': (reservation_two.stop_time + timedelta(days=1)).isoformat()}}
+ self.scheduling_unit_blueprint.save()
+ self.scheduling_unit_blueprint_cs001.scheduling_constraints_doc = {'scheduler': 'fixed_time', 'time': {'at': (reservation_two.stop_time + timedelta(days=1)).isoformat()}}
+ self.scheduling_unit_blueprint_cs001.save()
+
+ # try to schedule, should fail.
+ wipe_evaluate_constraints_caches()
+ scheduled_scheduling_units = self.scheduler.do_dynamic_schedule()
+ self.assertEqual(0, len(scheduled_scheduling_units))
+
+ # Assert the scheduling_unit has not been scheduled and that it has the correct expected unschedulable_reason
+ self.scheduling_unit_blueprint_cs001_cs002.refresh_from_db()
+ self.assertEqual('unschedulable', self.scheduling_unit_blueprint_cs001_cs002.status.value)
+ self.assertEqual("Stations ['CS001', 'CS002'] are reserved", self.scheduling_unit_blueprint_cs001_cs002.unschedulable_reason)
+
class TestTriggers(BaseDynamicSchedulingTestCase):
"""
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
index 6a4891a87c2e13276033563c897b9bf929247316..615319ff50700e5b11ac846e6643814bf4e10978 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
@@ -1202,6 +1202,7 @@ def mark_independent_subtasks_in_task_blueprint_as_schedulable(task_blueprint: T
def mark_subtask_as_unschedulable(subtask: Subtask, reason: str):
'''Convenience method: Mark the subtask as unschedulable. Unschedules first if needed.'''
with transaction.atomic():
+ logger.info("marking subtask id=%s from scheduling_unit_id=%s as unschedulable. reason: %s", subtask.id, subtask.task_blueprint.scheduling_unit_blueprint.id, reason)
if subtask.state.value == SubtaskState.Choices.SCHEDULED.value:
unschedule_subtask(subtask, post_state=SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULABLE.value))
else:
@@ -1223,6 +1224,7 @@ def mark_subtasks_and_successors_as_unschedulable(subtask: Subtask, reason: str)
def mark_subtask_as_defined(subtask: Subtask):
'''Convenience method: Mark the subtask as defined, making it's task & scheduling_unit schedulable. Unschedules first if needed.'''
with transaction.atomic():
+ logger.info("marking subtask id=%s from scheduling_unit_id=%s as defined/schedulable.", subtask.id, subtask.task_blueprint.scheduling_unit_blueprint.id)
if subtask.state.value == SubtaskState.Choices.SCHEDULED.value:
unschedule_subtask(subtask, post_state=SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value))
else:
@@ -1736,7 +1738,7 @@ def convert_task_station_groups_specification_to_station_list_without_used_and_o
available_stations = requested_stations - unavailable_stations
missing_stations = requested_stations - available_stations
max_nr_missing = station_group.get('max_nr_missing', 0)
- if len(missing_stations) > max_nr_missing:
+ if raise_when_too_many_missing and len(missing_stations) > max_nr_missing:
# early exit. No need to evaluate more groups when one groups does not meet the requirements
raise TooManyStationsUnavailableException('Subtask id=%s is missing more than max_nr_missing=%s stations which are available between \'%s\' and \'%s\'\nunavailable=%s\nrequested=%s\navailable=%s' % (
subtask.id, max_nr_missing,
@@ -1771,15 +1773,14 @@ def get_missing_stations(subtask: Subtask) -> []:
if subtask.specifications_template.type.value != SubtaskType.Choices.OBSERVATION.value:
return []
- # the observation has to be scheduled or "further"
- if subtask.state.value in (SubtaskState.Choices.DEFINED.value, SubtaskState.Choices.SCHEDULING.value):
- return []
-
# fetch the requested stations from the spec (without removing the unavailable ones! and not raising!)
requested_stations = set(convert_task_station_groups_specification_to_station_list_without_used_and_or_reserved_stations(subtask, remove_reserved_stations=False, remove_used_stations=False, raise_when_too_many_missing=False))
- # fetch the used_stations from the subtask spec
- used_stations = set(subtask.specifications_doc.get('stations', {}).get('station_list', []))
+ # fetch the used_stations from the subtask spec, depends on if it's scheduled-and-further
+ if subtask.state.value in (SubtaskState.Choices.DEFINED.value, SubtaskState.Choices.SCHEDULING.value):
+ used_stations = set(convert_task_station_groups_specification_to_station_list_without_used_and_or_reserved_stations(subtask, remove_reserved_stations=True, remove_used_stations=True, raise_when_too_many_missing=False))
+ else:
+ used_stations = set(subtask.specifications_doc.get('stations', {}).get('station_list', []))
# missing is the difference
return sorted(list(requested_stations-used_stations))