diff --git a/SAS/TMSS/backend/services/scheduling/lib/constraints.py b/SAS/TMSS/backend/services/scheduling/lib/constraints.py
index 879eeb745a98c051500cce2a949674824bcbd6a7..b79648b7a09b4f016eaf283c428bbcf4a839642e 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/constraints.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/constraints.py
@@ -1228,11 +1228,11 @@ def evaluate_sky_min_distance_constraint(scheduling_unit: models.SchedulingUnitB
actual_distances = distances[body]
assert (len(actual_distances) == 1)
actual_distance = actual_distances[gridded_timestamp]
- logger.debug("min_distance: SUB id=%s task_id=%s task_name='%s' pointing='%s' distance=%.3f[deg] to body=%s %s min_distance=%.3f[deg] at '%s'",
- scheduling_unit.id, target_obs_task.id, target_obs_task.name, pointing, actual_distance.degree, body,
- '<' if actual_distance.rad < min_distance else '>=',
- Angle(min_distance, astropy.units.rad).degree,
- gridded_timestamp)
+ # logger.debug("min_distance: SUB id=%s task_id=%s task_name='%s' pointing='%s' distance=%.3f[deg] to body=%s %s min_distance=%.3f[deg] at '%s'",
+ # scheduling_unit.id, target_obs_task.id, target_obs_task.name, pointing, actual_distance.degree, body,
+ # '<' if actual_distance.rad < min_distance else '>=',
+ # Angle(min_distance, astropy.units.rad).degree,
+ # gridded_timestamp)
if actual_distance.rad < min_distance:
# constraint not met. update result, and do early exit.
@@ -1862,10 +1862,10 @@ def get_weighted_start_time(scheduling_unit: models.SchedulingUnitBlueprint, low
weighted_start_time = round_to_second_precision(weighted_start_time)
if can_run_at(scheduling_unit, weighted_start_time, gridder):
- logger.debug("get_weighted_start_time: SUB id=%s weighted='%s' - weight=%.3f earliest='%s' optimal='%s' latest='%s' - window=['%s', '%s']", scheduling_unit.id, weighted_start_time, density_vs_optimal.weight, earliest_possible_start_time, optimal_start_time, latest_possible_start_time, lower_bound, upper_bound)
+ logger.info("get_weighted_start_time: SUB id=%s weighted='%s' - weight=%.3f earliest='%s' optimal='%s' latest='%s' - window=['%s', '%s']", scheduling_unit.id, weighted_start_time, density_vs_optimal.weight, earliest_possible_start_time, optimal_start_time, latest_possible_start_time, lower_bound, upper_bound)
return weighted_start_time
- logger.debug("get_weighted_start_time: SUB id=%s returning earliest='%s' - window=['%s', '%s']", scheduling_unit.id, earliest_possible_start_time, lower_bound, upper_bound)
+ logger.info("get_weighted_start_time: SUB id=%s returning earliest='%s' - window=['%s', '%s']", scheduling_unit.id, earliest_possible_start_time, lower_bound, upper_bound)
return earliest_possible_start_time
logger.warning("get_weighted_start_time: SUB id=%s could not compute weighted_start_time - window=['%s', '%s']", scheduling_unit.id, lower_bound, upper_bound)
@@ -2033,28 +2033,36 @@ def can_run_without_used_stations(scheduling_unit: models.SchedulingUnitBlueprin
def get_blocking_scheduled_units(scheduling_unit: models.SchedulingUnitBlueprint, proposed_start_time: datetime=None) -> QuerySet:
- '''Get a list (tuple) of scheduled scheduling_units overlapping with the scheduled_observation_start/stop_time of the given scheduling_unit'''
+ return get_blocking_units(scheduling_unit, proposed_start_time, (models.SchedulingUnitStatus.Choices.SCHEDULED.value,))
+
+def get_blocking_scheduled_or_observing_units(scheduling_unit: models.SchedulingUnitBlueprint, proposed_start_time: datetime=None) -> QuerySet:
+ return get_blocking_units(scheduling_unit, proposed_start_time, (models.SchedulingUnitStatus.Choices.SCHEDULED.value, models.SchedulingUnitStatus.Choices.OBSERVING.value))
+
+def get_blocking_units(scheduling_unit: models.SchedulingUnitBlueprint, proposed_start_time: datetime = None, blocking_statuses: Iterable[str]=None) -> QuerySet:
from .dynamic_scheduling import DEFAULT_INTER_OBSERVATION_GAP
- scheduled_units = models.SchedulingUnitBlueprint.objects.filter(status__value=models.SchedulingUnitStatus.Choices.SCHEDULED.value).filter(obsolete_since__isnull=True)
+ units = models.SchedulingUnitBlueprint.objects.filter(obsolete_since__isnull=True)
+
+ if blocking_statuses:
+ units = units.filter(status__value__in=blocking_statuses)
lower_bound = (proposed_start_time or scheduling_unit.scheduled_observation_start_time) - DEFAULT_INTER_OBSERVATION_GAP
upper_bound = (proposed_start_time or scheduling_unit.scheduled_observation_start_time) + scheduling_unit.relative_observation_stop_time + DEFAULT_INTER_OBSERVATION_GAP
# do three stage filtering.
# First in the db which is fast, but sometimes yields too many overlapping units, because we overlap the entire unit instead of just the observation part...
- overlapping_scheduled_units = scheduled_units.filter(scheduled_stop_time__gt=lower_bound)
- overlapping_scheduled_units = overlapping_scheduled_units.filter(scheduled_start_time__lt=upper_bound)
+ overlapping_units = units.filter(scheduled_stop_time__gt=lower_bound)
+ overlapping_units = overlapping_units.filter(scheduled_start_time__lt=upper_bound)
# Second, loop over the small number of overlapping scheduled units, and only keep the ones overlapping in the observation part
# (scheduled_observation_start/stop_time properties are not available as db columns, they are evaluated in python)
- observation_overlapping_scheduled_units = [s for s in overlapping_scheduled_units.all()
+ observation_overlapping_units = [s for s in overlapping_units.all()
if s.scheduled_observation_stop_time > lower_bound and
s.scheduled_observation_start_time <= upper_bound]
# Third, loop over the small number of overlapping scheduled units, and only keep the ones sharing one or more stations
blocking_scheduled_unit_ids = set()
for obs_task in scheduling_unit.observation_tasks.filter(obsolete_since__isnull=True).all():
- for overlapping_scheduled_unit in observation_overlapping_scheduled_units:
+ for overlapping_scheduled_unit in observation_overlapping_units:
for overlapping_scheduled_obs_task in overlapping_scheduled_unit.observation_tasks.all():
if not enough_stations_available_for_task(obs_task, unavailable_stations=overlapping_scheduled_obs_task.used_stations):
blocking_scheduled_unit_ids.add(overlapping_scheduled_unit.id)
@@ -2065,7 +2073,7 @@ def get_blocking_scheduled_units(scheduling_unit: models.SchedulingUnitBlueprint
blocking_scheduled_unit_ids.remove(scheduling_unit.id)
# Finally, return the result as a queryset, so the caller can do further queries on it.
- return scheduled_units.filter(id__in=[x for x in blocking_scheduled_unit_ids]).all()
+ return units.filter(id__in=[x for x in blocking_scheduled_unit_ids]).all()
def determine_unschedulable_reason_and_mark_unschedulable_if_needed(scheduling_unit: models.SchedulingUnitBlueprint, lower_bound: datetime, upper_bound: datetime, proposed_start_time: datetime=None, gridder: Gridder=None, raise_if_interruped: Callable=noop) -> models.SchedulingUnitBlueprint:
@@ -2081,7 +2089,7 @@ def determine_unschedulable_reason_and_mark_unschedulable_if_needed(scheduling_u
msg = "Stations %s are reserved at start_time='%s'" % (','.join([str(s) for s in missing_stations]), proposed_start_time)
return mark_independent_subtasks_in_scheduling_unit_blueprint_as_unschedulable(scheduling_unit, msg)
- blocking_units = get_blocking_scheduled_units(scheduling_unit)
+ blocking_units = get_blocking_scheduled_or_observing_units(scheduling_unit)
if blocking_units.exists():
if len(blocking_units) == 1:
msg = "Scheduling unit id=%s is blocking this unit from being scheduled" % (blocking_units[0].id, )
diff --git a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
index b9459217c412dc5fdb456e265a62152d4b46540e..86a77a0c2eda0e5421a719b108f5aff219846fdd 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
@@ -323,11 +323,13 @@ class Scheduler:
# determine next possible start time for remaining scheduling_units
lower_bound_start_time = datetime.utcnow() + DEFAULT_NEXT_STARTTIME_GAP
if scheduled_unit:
- lower_bound_start_time = scheduled_unit.on_sky_stop_time + DEFAULT_INTER_OBSERVATION_GAP
+ # this scheduled_unit was the first and the best
+ # so update the search window to start from scheduled_unit.on_sky_start_time+GAP because we allow parallel observations.
+ lower_bound_start_time = max(lower_bound_start_time, scheduled_unit.on_sky_start_time + DEFAULT_INTER_OBSERVATION_GAP)
else:
scheduled_scheduling_units = get_scheduled_scheduling_units(datetime.utcnow(), scheduler='dynamic')
if scheduled_scheduling_units:
- lower_bound_start_time = max([s.on_sky_stop_time for s in scheduled_scheduling_units]) + DEFAULT_INTER_OBSERVATION_GAP
+ lower_bound_start_time = max(lower_bound_start_time, min([s.on_sky_start_time for s in scheduled_scheduling_units]) + DEFAULT_INTER_OBSERVATION_GAP)
# round up to next nearest minute
lower_bound_start_time += timedelta(seconds=60-lower_bound_start_time.second,
@@ -452,37 +454,25 @@ class Scheduler:
# search in a forward sliding window for the best scheduling_unit that can be scheduled
# once found and scheduled, exit. Otherwise slide window forward and try again.
# When scheduling 'just in time' we need to allow the other services/controllers/stations/boards some startup time: DEFAULT_NEXT_STARTTIME_GAP
- lower_bound_start_time = max(datetime.utcnow() + DEFAULT_NEXT_STARTTIME_GAP,
- min([su.earliest_possible_cycle_start_time for su in candidate_units]))
-
- # for normal A/B units (not triggered units), adjust lower_bound_start_time search window to any running/observing unit
- observing_units = get_observing_scheduling_units()
- if observing_units.exists() and schedulable_units != schedulable_units_triggered:
- last_on_sky_stop_time = observing_units.aggregate(Max('on_sky_stop_time'))['on_sky_stop_time__max']
- lower_bound_start_time = max(lower_bound_start_time, last_on_sky_stop_time + DEFAULT_NEXT_STARTTIME_GAP)
-
- # adjust lower_bound_start_time search window to any observed and ending-later unit
- observed_or_beyond_scheduling_units = get_observed_or_beyond_scheduling_units()
- if observed_or_beyond_scheduling_units.exists():
- last_on_sky_stop_time = observed_or_beyond_scheduling_units.aggregate(Max('on_sky_stop_time'))['on_sky_stop_time__max']
- lower_bound_start_time = max(lower_bound_start_time, last_on_sky_stop_time + DEFAULT_NEXT_STARTTIME_GAP)
+ # scanning from now+DEFAULT_NEXT_STARTTIME_GAP may include/overlap with already running and/or scheduled observations. That's ok. There are rules to handle the precedence who wins.
+ lower_bound_start_time = round_to_second_precision(datetime.utcnow() + DEFAULT_NEXT_STARTTIME_GAP)
# upper bound of search window is at least a 24h later, or up unit latest cycle end time
- upper_bound_stop_time = max(lower_bound_start_time + timedelta(days=1),
- max([su.latest_possible_cycle_stop_time for su in candidate_units]))
+ upper_bound_stop_time = round_to_second_precision(max(lower_bound_start_time + timedelta(days=1),
+ max([su.latest_possible_cycle_stop_time for su in candidate_units])))
window_lower_bound_start_time = lower_bound_start_time
- while window_lower_bound_start_time < upper_bound_stop_time:
+ window_upper_bound_stop_time = min(window_lower_bound_start_time + timedelta(hours=12),
+ upper_bound_stop_time-timedelta(seconds=1))
+ while window_upper_bound_stop_time < upper_bound_stop_time:
self._raise_if_triggered() # interrupts the scheduling loop for a next round
try:
# no need to irritate user in log files with sub-second scheduling precision
window_lower_bound_start_time = round_to_second_precision(window_lower_bound_start_time)
- # our sliding window only looks 12 hours ahead
- window_upper_bound_stop_time = round_to_second_precision(window_lower_bound_start_time + timedelta(hours=12))
# try to find the best next scheduling_unit
- logger.info("schedule_next_scheduling_unit: searching for best scheduling unit to schedule in window ['%s', '%s']", lower_bound_start_time, window_upper_bound_stop_time)
+ logger.info("schedule_next_scheduling_unit: searching for best scheduling unit to schedule in window ['%s', '%s']", window_lower_bound_start_time, window_upper_bound_stop_time)
best_scored_scheduling_unit = self.find_best_next_schedulable_unit(candidate_units,
window_lower_bound_start_time,
window_upper_bound_stop_time)
@@ -500,7 +490,16 @@ class Scheduler:
logger.info("schedule_next_scheduling_unit: found best candidate id=%s '%s' weighted_score=%.3f start_time=%s interrupts_telescope=%s",
best_scheduling_unit.id, best_scheduling_unit.name, best_scheduling_unit_score, best_start_time, best_scheduling_unit.interrupts_telescope)
- return self.try_schedule_unit(best_scored_scheduling_unit.scheduling_unit, best_scored_scheduling_unit.start_time)
+ scheduled_unit = self.try_schedule_unit(best_scored_scheduling_unit.scheduling_unit, best_scored_scheduling_unit.start_time)
+ if scheduled_unit is None:
+ # we had a best_scored_scheduling_unit, but it could not be scheduled here.
+ # advance the lower bound of the window a bit, so we can find a new best_scored_scheduling_unit,
+ # or if the same one wins it is scheduled also a bit later, where it might fit.
+ window_lower_bound_start_time += timedelta(hours=1)
+ else:
+ # done, just return the scheduled_unit
+ return scheduled_unit
+
else:
logger.info("schedule_next_scheduling_unit: no scheduling unit found which could be scheduled in window ['%s', '%s']", window_lower_bound_start_time, window_upper_bound_stop_time)
@@ -532,13 +531,9 @@ class Scheduler:
logger.info("schedule_next_scheduling_unit: no more candidate units...")
break
- # advance the window
- min_earliest_possible_start_time = get_min_earliest_possible_start_time(candidate_units, window_lower_bound_start_time+timedelta(hours=1), window_lower_bound_start_time+timedelta(hours=25), gridder=self.search_gridder, raise_if_interruped=self._raise_if_triggered)
- if min_earliest_possible_start_time is None:
- window_lower_bound_start_time += timedelta(hours=6)
- else:
- window_lower_bound_start_time = max(min_earliest_possible_start_time, window_lower_bound_start_time + timedelta(hours=1))
-
+ # advance the window at the upper side only so more candidates fit in
+ # our sliding window only looks 12 hours ahead
+ window_upper_bound_stop_time += timedelta(hours=3)
# search again... (while loop) with the remaining schedulable_units and new lower_bound_start_time
# nothing was found, or an error occurred.
@@ -566,8 +561,8 @@ class Scheduler:
unschededule_blocking_scheduled_units_if_needed_and_possible(scheduling_unit, start_time, self.fine_gridder)
unschededule_previously_scheduled_unit_if_needed_and_possible(scheduling_unit, start_time)
- # are there any blocking scheduled scheduling_units still in the way?
- blocking_scheduling_units = get_blocking_scheduled_units(scheduling_unit, start_time)
+ # are there any blocking scheduled or observing scheduling_units still in the way?
+ blocking_scheduling_units = get_blocking_scheduled_or_observing_units(scheduling_unit, start_time)
if blocking_scheduling_units.exists():
logger.warning("cannot schedule scheduling_unit id=%s '%s' at start_time=%s interrupts_telescope=%s because there are %d other units blocking it",