diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py b/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
index 142c1635f1cc1207c7700f525593b635de6eb939..8292f5d6c79ee372e9e27d660e22a6d77e607ed7 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
@@ -34,21 +34,22 @@ SUN_SET_RISE_ANGLE_TO_HORIZON = Angle(10, unit=astropy.units.deg)
SUN_SET_RISE_PRECISION = 30
-@lru_cache(maxsize=256, typed=False) # does not like lists, so use tuples to allow caching
def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tuple, angle_to_horizon: Angle=SUN_SET_RISE_ANGLE_TO_HORIZON,
create_when_not_found=False) -> dict:
"""
Retrieve for given stations and given timestamps the sunrise/sunset/day/night data as dictionary
If station/timestamp is already calculated it will be retrieved from database otherwise it will be calculated
- and added to the database for possible future retrieval (optional).
+ and added to the database for possible future retrieval (optional parameter must be true).
Storing the pre-calculated data into a database makes retrieval faster.
- TODO make a service to pre-calculate e.g. 1 year in advanced, currently only one week for all stations
- during populate of testenvironment
- TODO about the night.... when using a time in the timestamp
+
+ The day/sunrise/sunset is always on the date of the timestamp.
+ The night is usually the one _starting_ on the date of the time stamp, unless the given timestamp falls before
+ sunrise, in which case it is the night _ending_ on the timestamp date.
+
:param timestamps: tuple of datetimes, e.g. datetime(2020, 1, 1)
:param stations: tuple of station names, e.g. ("CS002")
:param angle_to_horizon: the angle between horizon and given coordinates for which rise and set times are returned
- :param: create_when_not_found: Add data to database if not found and so calculated for first time
+ :param: create_when_not_found: Add data to database if not found in database and so calculated for first time
:return A dict that maps station names to a nested dict that contains lists of start and end times for sunrise,
sunset, day and night, on each requested date.
E.g.
@@ -66,6 +67,7 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
"""
return_dict = {}
for station in stations:
+ observer = create_astroplan_observer_for_station(station)
for timestamp in timestamps:
# We can also check if ALL stations/timestamps are in DB once. Do it now in a loop for each
# station/timestamp, because we might missing something
@@ -81,12 +83,18 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
if station_timestamp_found:
logger.debug("StationTimeline data found in DB for station=%s, timestamp=%s" % (station,timestamp))
sunrise_dict = {"start": obj.sunrise_start, "end": obj.sunrise_end}
- sunset_dict = {"start": obj.sunset_start, "end": obj.sunset_start}
- day_dict = {"start": obj.day_start, "end": obj.day_end}
- night_dict = {"start": obj.night_start, "end": obj.night_end}
+ sunset_dict = {"start": obj.sunset_start, "end": obj.sunset_end}
else:
- observer = create_astroplan_observer_for_station(station)
- sunrise_dict, sunset_dict, day_dict, night_dict = get_sunrise_and_sunset_of_observer_day(observer, timestamp, angle_to_horizon)
+ # Not found in database so calculate it
+ try:
+ sunrise_dict, sunset_dict = calculate_and_get_sunrise_and_sunset_of_observer_day(observer, timestamp, angle_to_horizon)
+ except Exception as exp:
+ logger.warning("Can not calculate sunrise/sunset for station=%s, timestamp=%s" % (station,timestamp))
+ # raise exp
+ # Don't let it crash for now
+ # The stations SE607 and LV614 station has problems calculation on 2021-07-01....
+ # The SE607 also on 2021-06-04 ??
+ break
# Add to database
if create_when_not_found:
station_timeline = StationTimeline.objects.create(
@@ -95,14 +103,37 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
sunrise_start=sunrise_dict['start'],
sunrise_end=sunrise_dict['end'],
sunset_start=sunset_dict['start'],
- sunset_end=sunset_dict['end'],
- day_start=day_dict['start'],
- day_end=day_dict['end'],
- night_start=night_dict['start'],
- night_end=night_dict['end'])
+ sunset_end=sunset_dict['end'])
logger.debug("StationTimeline %s calculated and created for station=%s, timestamp=%s" %
(station_timeline, station, timestamp))
- # Create json overall result
+
+ # Derive day/night from sunset/sunrise
+ day_dict = {"start": sunrise_dict["end"], "end": sunset_dict["start"]}
+
+ if timestamp >= sunrise_dict["start"]:
+ # Determine next sunrise start
+ try:
+ obj_next = StationTimeline.objects.get(station_name=station,
+ timestamp=datetime.date(timestamp + timedelta(days=1)))
+ sunrise_next_start = obj_next.sunrise_start
+ except:
+ sunrise_next_start = observer.sun_rise_time(time=Time(sunrise_dict["end"]), horizon=-angle_to_horizon,
+ which='next',
+ n_grid_points=SUN_SET_RISE_PRECISION).to_datetime()
+ night_dict = {"start": sunset_dict["end"], "end": sunrise_next_start}
+ else:
+ # Determine previous sunset end
+ try:
+ obj_prev = StationTimeline.objects.get(station_name=station,
+ timestamp=datetime.date(timestamp - timedelta(days=1)))
+ sunset_previous_end = obj_prev.sunrise_start
+ except:
+ sunset_previous_end = observer.sun_set_time(time=Time(sunrise_dict["start"]), horizon=-angle_to_horizon,
+ which='previous',
+ n_grid_points=SUN_SET_RISE_PRECISION).to_datetime()
+ night_dict = {"start": sunset_previous_end, "end": sunrise_dict["start"]}
+
+ # Create overall result
return_dict.setdefault(station, {})
return_dict[station].setdefault("sunrise", []).append(sunrise_dict)
return_dict[station].setdefault("sunset", []).append(sunset_dict)
@@ -113,16 +144,13 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
@lru_cache(maxsize=256, typed=False)
-def get_sunrise_and_sunset_of_observer_day(observer, timestamp: datetime, angle_to_horizon: Angle) -> dict:
+def calculate_and_get_sunrise_and_sunset_of_observer_day(observer, timestamp: datetime, angle_to_horizon: Angle) -> dict:
"""
- Compute sunrise, sunset, day and night of the given observer object (station) at the given timestamp.
- The day/sunrise/sunset is always on the date of the timestamp.
- The night is usually the one _starting_ on the date of the time stamp, unless the given timestamp falls before
- sunrise, in which case it is the night _ending_ on the timestamp date.
+ Compute sunrise, sunset of the given observer object (station) at the given timestamp.
:param observer: observer object
:param timestamp: Datetime of a day (datetime(2020, 1, 1)
:param the angle between horizon and given coordinates for which rise and set times are returned
- :return: dictionaries (with 'start' and 'end' defined) of sunrise, sunset, day, night
+ :return: dictionaries (with 'start' and 'end' defined) of sunrise, sunset
"""
sunrise_start = observer.sun_rise_time(time=Time(datetime.combine(timestamp.date(), dtime(12, 0, 0))),
horizon=-angle_to_horizon, which='previous',
@@ -136,17 +164,8 @@ def get_sunrise_and_sunset_of_observer_day(observer, timestamp: datetime, angle_
sunrise_dict = {"start": sunrise_start.to_datetime(), "end": sunrise_end.to_datetime()}
sunset_dict = {"start": sunset_start.to_datetime(), "end": sunset_end.to_datetime()}
- day_dict = {"start": sunrise_end.to_datetime(), "end": sunset_start.to_datetime()}
- if timestamp >= sunrise_start:
- sunrise_next_start = observer.sun_rise_time(time=sunset_end, horizon=-angle_to_horizon, which='next',
- n_grid_points=SUN_SET_RISE_PRECISION)
- night_dict = {"start": sunset_end.to_datetime(), "end": sunrise_next_start.to_datetime()}
- else:
- sunset_previous_end = observer.sun_set_time(time=sunrise_start, horizon=-angle_to_horizon, which='previous',
- n_grid_points=SUN_SET_RISE_PRECISION)
- night_dict = {"start": sunset_previous_end.to_datetime(), "end": sunrise_start.to_datetime()}
- return sunrise_dict, sunset_dict, day_dict, night_dict
+ return sunrise_dict, sunset_dict
# todo: Depending on usage patterns, we should consider refactoring this a little so that we cache on a function with a single timestamp as input. Requests with similar (but not identical) timestamps or bodies currently make no use of cached results for the subset computed in previous requests.
@@ -289,21 +308,14 @@ def get_all_stations():
returns all possible stations.
Retrieve station names from station template by getting the Dutch and International stations,
then you should have it all.
- lst_stations = ["CS001", "CS002", "CS003", "CS004", "CS005", "CS006", "CS007", "CS011", "CS013", "CS017", "CS021",
- "CS024", "CS026", "CS028", "CS030", "CS031", "CS032", "CS101", "CS103", "CS201", "CS301", "CS302",
- "CS401", "CS501",
- "RS106", "RS205", "RS208", "RS210", "RS305", "RS306", "RS307", "RS310", "RS406", "RS407", "RS409",
- "RS503", "RS508", "RS509",
- "DE601", "DE602", "DE603", "DE604", "DE605", "FR606", "SE607", "UK608", "DE609", "PL610", "PL611",
- "PL612", "IE613", "LV614"]
"""
lst_stations = []
for station_group in ["Dutch", "International"]:
- station_schema_template = CommonSchemaTemplate.objects.get(name="stations", version=1)
- groups = station_schema_template.schema['definitions']['station_group']['anyOf']
try:
+ station_schema_template = CommonSchemaTemplate.objects.get(name="stations", version=1)
+ groups = station_schema_template.schema['definitions']['station_group']['anyOf']
selected_group = next(g for g in groups if g['title'].lower() == station_group.lower())
- except StopIteration:
- raise ValueError('No station_group with name "%s" found in the JSON schema.' % station_group)
- lst_stations.extend(selected_group['properties']['stations']['enum'][0])
+ lst_stations.extend(selected_group['properties']['stations']['enum'][0])
+ except Exception:
+ logger.warning("No stations schema found, sorry can not determine station list, return empty list")
return lst_stations
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
index 8a62e2862a4f83e2dac48a78fdf30406aa19fc15..52324656a177f74bb4c5467e9c94c689962a7994 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
@@ -904,10 +904,6 @@ class Migration(migrations.Migration):
('sunrise_end', models.DateTimeField(null=True, help_text='End time of the sunrise.')),
('sunset_start', models.DateTimeField(null=True, help_text='Start time of the sunset.')),
('sunset_end', models.DateTimeField(null=True, help_text='End time of the sunset.')),
- ('day_start', models.DateTimeField(null=True, help_text='Start time of the day.')),
- ('day_end', models.DateTimeField(null=True, help_text='End time of the day.')),
- ('night_start', models.DateTimeField(null=True, help_text='Start time of the night.')),
- ('night_end', models.DateTimeField(null=True, help_text='End time of the night.'))
],
),
migrations.AddConstraint(
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py
index b1b233acb4fa2c036b7f77a6b7fcf3699a1ba84a..3905cfeae06892743ba39bee500dce227e6e79e5 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py
@@ -11,7 +11,8 @@ from django.db.models import Model, CharField, DateTimeField, DateField
class StationTimeline(Model):
"""
- Represents computations of sunrise, sunset, day and night of the given stations at the given timestamps.
+ Represents computations of sunrise, sunset of the given stations at the given timestamps.
+ Day and night are derived from sunset/sunrise data.
The day/sunrise/sunset is always on the date of the timestamp.
The night is usually the one _starting_ on the date of the time stamp, unless the given timestamp falls
before sunrise, in which case it is the night _ending_ on the timestamp date.
@@ -23,8 +24,5 @@ class StationTimeline(Model):
sunrise_end = DateTimeField(null=True, help_text='End time of the sunrise.')
sunset_start = DateTimeField(null=True, help_text='Start time of the sunset.')
sunset_end = DateTimeField(null=True, help_text='End time of the sunset.')
- day_start = DateTimeField(null=True, help_text='Start time of the day.')
- day_end = DateTimeField(null=True, help_text='End time of the day.')
- night_start = DateTimeField(null=True, help_text='Start time of the night.')
- night_end = DateTimeField(null=True, help_text='End time of the night.')
+
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
index 18046daac1d7c76c09c8ab8e05341b0758a39b93..3b3147b6774851a059d16e46b9454f2559ff6db5 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
@@ -69,7 +69,7 @@ def populate_test_data():
from lofar.common.json_utils import get_default_json_object_for_schema
# Maybe move to 'migrations populate'
- populate_calculations(nbr_days=3)
+ #populate_calculations(nbr_days=3)
constraints_template = models.SchedulingConstraintsTemplate.objects.get(name="constraints")
constraints_spec = get_default_json_object_for_schema(constraints_template.schema)
@@ -485,18 +485,17 @@ def populate_system_test_users():
lta_user.groups.add(Group.objects.get(name='LTA User'))
-def populate_calculations(nbr_days=3):
+def populate_calculations(nbr_days=3, start_date=date.today()):
"""
- Calculate a week of station timeline data of all stations
- will take about a minute
- TODO create a service which will do this continuously ?
+ Populate station timeline data of all stations for given number of days the starting at given date
+ Note: If data is not in database yet, it will take about 6 seconds to calculate it for all (51) stations
"""
- starttime = datetime.utcnow()
- logger.info("Populate sunrise, sunset, day, night for ALL stations from today up to %d days" % nbr_days)
+ starttime_for_logging = datetime.utcnow()
+ logger.info("Populate sunrise and sunset for ALL known stations from %s up to %d days" % (start_date, nbr_days))
lst_timestamps = []
for i in range(0, nbr_days):
- dt = datetime.combine(date.today(), datetime.min.time()) + timedelta(days=i)
+ dt = datetime.combine(start_date, datetime.min.time()) + timedelta(days=i)
lst_timestamps.append(dt)
timestamps_and_stations_to_sun_rise_and_set(tuple(lst_timestamps), tuple(get_all_stations()), create_when_not_found=True)
- logger.info("Done in %.1fs", (datetime.utcnow()-starttime).total_seconds())
+ logger.info("Done in %.1fs", (datetime.utcnow()-starttime_for_logging).total_seconds())
diff --git a/SAS/TMSS/backend/test/test_utils.py b/SAS/TMSS/backend/test/test_utils.py
index ad64349b5c195fb1d7b5381cc2a4404a535711ae..9f6b7c1889f8359a39e60f38a779c4a86df891f4 100644
--- a/SAS/TMSS/backend/test/test_utils.py
+++ b/SAS/TMSS/backend/test/test_utils.py
@@ -585,6 +585,7 @@ def main_test_environment():
group.add_option('-f', '--feedbackservice', dest='feedbackservice', action='store_true', help='Enable feedbackservice to handle feedback from observations/pipelines which comes in via the (old qpid) otdb messagebus.')
group.add_option('--all', dest='all', action='store_true', help='Enable/Start all the services, upload schemas and testdata')
group.add_option('--simulate', dest='simulate', action='store_true', help='Simulate a run of the first example scheduling_unit (implies --data and --eventmessages and --ra_test_environment)')
+ group.add_option('--calculation_service', dest='calculation_service', action='store_true', help='Enable the (Pre-)Calculations service')
group = OptionGroup(parser, 'Messaging options')
parser.add_option_group(group)
@@ -656,6 +657,16 @@ def main_test_environment():
except KeyboardInterrupt:
return
+ # This is just a 'simple' timing service
+ if options.calculation_service:
+ stop_event = threading.Event()
+ create_calculation_service(stop_event=stop_event)
+ try:
+ stop_event.wait()
+ except KeyboardInterrupt:
+ print("KeyboardInterrupt")
+ return
+
waitForInterrupt()
@@ -958,5 +969,71 @@ def main_scheduling_unit_blueprint_simulator():
pass
+def create_calculation_service(stop_event: threading.Event):
+ """
+ First implementation of a simple time scheduler, starting with one major task and then run once a day
+ It created and start the populate of sunset/sunrise calculations
+ Should be organised a little bit differently and better!!!!!!!
+ """
+ print("create_calculation_service")
+
+ import threading, time, signal
+ from datetime import timedelta
+ # Import here otherwise you get
+ # "django.core.exceptions.ImproperlyConfigured: Requested setting INSTALLED_APPS, but settings are not configured. You must either define the environment variable DJANGO_SETTINGS_MODULE or call settings.configure() before accessing settings."
+ from lofar.sas.tmss.tmss.tmssapp.populate import populate_calculations
+
+ # Some parameters, currently as constants
+ WAIT_TIME_SECONDS = 300
+ NBR_DAYS_CALCULATE_AHEAD = 100
+ NBR_DAYS_BEFORE_TODAY = 1
+
+
+ class ProgramKilled(Exception):
+ pass
+
+ def populate_calculations_once_a_day():
+ days_offset = NBR_DAYS_CALCULATE_AHEAD - NBR_DAYS_BEFORE_TODAY
+ populate_calculations(nbr_days=1, start_date=datetime.date.today() + datetime.timedelta(days=days_offset))
+
+ def signal_handler(signum, frame):
+ raise ProgramKilled
+
+ class Job(threading.Thread):
+ def __init__(self, interval, execute, *args, **kwargs):
+ threading.Thread.__init__(self)
+ self.daemon = False
+ self.stopped = threading.Event()
+ self.interval = interval
+ self.execute = execute
+ self.args = args
+ self.kwargs = kwargs
+
+ def stop(self):
+ self.stopped.set()
+ self.join()
+
+ def run(self):
+ while not self.stopped.wait(self.interval.total_seconds()):
+ self.execute(*self.args, **self.kwargs)
+
+ signal.signal(signal.SIGTERM, signal_handler)
+ signal.signal(signal.SIGINT, signal_handler)
+ job = Job(interval=timedelta(seconds=WAIT_TIME_SECONDS), execute=populate_calculations_once_a_day)
+ job.start()
+
+ # Start one day before today
+ populate_calculations(nbr_days=NBR_DAYS_CALCULATE_AHEAD, start_date=datetime.date.today() - datetime.timedelta(days=NBR_DAYS_BEFORE_TODAY))
+
+ print("Execute Every %d sec" % WAIT_TIME_SECONDS )
+ while True:
+ try:
+ time.sleep(10)
+ except ProgramKilled:
+ print("Program killed: running cleanup code")
+ job.stop()
+ break
+
+
if __name__ == '__main__':
main_test_environment()
diff --git a/SAS/TMSS/client/lib/tmss_http_rest_client.py b/SAS/TMSS/client/lib/tmss_http_rest_client.py
index bcd7309b82b976177ce4f527435e2bea60b9cb09..8ca49cf4cbd16802330bcf504e21156298aff771 100644
--- a/SAS/TMSS/client/lib/tmss_http_rest_client.py
+++ b/SAS/TMSS/client/lib/tmss_http_rest_client.py
@@ -371,7 +371,7 @@ class TMSSsession(object):
if response.status_code == 201:
logger.info("created new template with name=%s: %s", name, json.loads(response.text)['url'])
else:
- raise Exception("Could not POST template with name=%s: %s" (name,response.text))
+ raise Exception("Could not POST template with name=%s: %s" % (name,response.text))
def process_feedback_and_set_to_finished_if_complete(self, subtask_id: int, feedback: str) -> {}:
'''Process the feedback_doc (which can be for one or more or all dataproducts), store/append it in the subtask's raw_feedback, and process it into json feedback per dataproduct. Sets the subtask to finished if all dataproducts are processed, which may require multiple postings of partial feedback docs.