From 7adf087d215c3bb65acd425f894c44e3fd0d74ab Mon Sep 17 00:00:00 2001
From: Roy de Goei <goei@astron.nl>
Date: Tue, 23 Mar 2021 07:44:18 +0100
Subject: [PATCH] TMSS-687: Add Create StationTimeLine to the model to store
sunrise, sunset, day, night calculation for all stations over a period
---
.../backend/src/tmss/tmssapp/conversions.py | 9 ++++
.../tmss/tmssapp/migrations/0001_initial.py | 15 ++++++
.../src/tmss/tmssapp/models/CMakeLists.txt | 1 +
.../src/tmss/tmssapp/models/__init__.py | 3 +-
.../src/tmss/tmssapp/models/calculations.py | 30 +++++++++++
SAS/TMSS/backend/src/tmss/tmssapp/populate.py | 50 ++++++++++++++++++-
.../tmss/tmssapp/serializers/CMakeLists.txt | 1 +
.../src/tmss/tmssapp/serializers/__init__.py | 3 +-
.../tmss/tmssapp/serializers/calculations.py | 15 ++++++
.../src/tmss/tmssapp/viewsets/CMakeLists.txt | 3 +-
.../src/tmss/tmssapp/viewsets/__init__.py | 3 +-
.../src/tmss/tmssapp/viewsets/calculations.py | 13 +++++
SAS/TMSS/backend/src/tmss/urls.py | 6 +++
13 files changed, 147 insertions(+), 5 deletions(-)
create mode 100644 SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py
create mode 100644 SAS/TMSS/backend/src/tmss/tmssapp/serializers/calculations.py
create mode 100644 SAS/TMSS/backend/src/tmss/tmssapp/viewsets/calculations.py
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py b/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
index ae926e172f4..819e3e4983a 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/conversions.py
@@ -5,6 +5,8 @@ from astropy.coordinates.earth import EarthLocation
from astropy.coordinates import Angle, get_body
import astropy.time
from functools import lru_cache
+from lofar.sas.tmss.tmss.tmssapp.models.calculations import StationTimeline
+
import logging
logger = logging.getLogger(__name__)
@@ -54,6 +56,13 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
return_dict = {}
for station in stations:
for timestamp in timestamps:
+ # TODO continue on this part
+ print("find %s %s" % (station, datetime.date(timestamp)))
+ if StationTimeline.objects.filter(station_name=station, timestamp=datetime.date(timestamp)).count() >0:
+ logger.info("Station %s %s gevonden" % (station,timestamp))
+ # so getit from database and fill in
+ # else calculate and add to database
+ #print("hey")
# todo: this can probably be made faster by moving the following logic to an own function with single station/timestamp as input and putting the lru_cache on there.
# This also means that we have to strip the time from the datetime. Can this be safely done?
observer = create_astroplan_observer_for_station(station)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
index 2df67c9b37f..8a62e2862a4 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
@@ -895,6 +895,21 @@ class Migration(migrations.Migration):
('second', models.ForeignKey(help_text='Second Task Blueprint to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='second_scheduling_relation', to='tmssapp.TaskBlueprint')),
],
),
+ migrations.CreateModel(
+ name='StationTimeline',
+ fields=[
+ ('station_name', models.CharField(max_length=16, null=False, editable=False, help_text='The LOFAR station name.')),
+ ('timestamp', models.DateField(editable=False, null=True, help_text='The date (YYYYMMDD).')),
+ ('sunrise_start', models.DateTimeField(null=True, help_text='Start time of the sunrise.')),
+ ('sunrise_end', models.DateTimeField(null=True, help_text='End time of the sunrise.')),
+ ('sunset_start', models.DateTimeField(null=True, help_text='Start time of the sunset.')),
+ ('sunset_end', models.DateTimeField(null=True, help_text='End time of the sunset.')),
+ ('day_start', models.DateTimeField(null=True, help_text='Start time of the day.')),
+ ('day_end', models.DateTimeField(null=True, help_text='End time of the day.')),
+ ('night_start', models.DateTimeField(null=True, help_text='Start time of the night.')),
+ ('night_end', models.DateTimeField(null=True, help_text='End time of the night.'))
+ ],
+ ),
migrations.AddConstraint(
model_name='taskrelationselectiontemplate',
constraint=models.UniqueConstraint(fields=('name', 'version'), name='taskrelationselectiontemplate_unique_name_version'),
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/CMakeLists.txt b/SAS/TMSS/backend/src/tmss/tmssapp/models/CMakeLists.txt
index 3496efd5735..f6e74f93da0 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/CMakeLists.txt
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/CMakeLists.txt
@@ -8,6 +8,7 @@ set(_py_files
scheduling.py
common.py
permissions.py
+ calculations.py
)
python_install(${_py_files}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/__init__.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/__init__.py
index 0b0546b8d4b..3eb788371d9 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/__init__.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/__init__.py
@@ -1,4 +1,5 @@
from .specification import *
from .scheduling import *
from .common import *
-from .permissions import *
\ No newline at end of file
+from .permissions import *
+from .calculations import *
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py
new file mode 100644
index 00000000000..b1b233acb4f
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/calculations.py
@@ -0,0 +1,30 @@
+"""
+This file contains the database models for calculations
+"""
+
+import os
+import logging
+logger = logging.getLogger(__name__)
+
+from django.db.models import Model, CharField, DateTimeField, DateField
+
+
+class StationTimeline(Model):
+ """
+ Represents computations of sunrise, sunset, day and night of the given stations at the given timestamps.
+ The day/sunrise/sunset is always on the date of the timestamp.
+ The night is usually the one _starting_ on the date of the time stamp, unless the given timestamp falls
+ before sunrise, in which case it is the night _ending_ on the timestamp date.
+ """
+ station_name = CharField(max_length=16, null=False, editable=False, help_text='The LOFAR station name.')
+ timestamp = DateField(editable=False, null=True, help_text='The date (YYYYMMDD).')
+
+ sunrise_start = DateTimeField(null=True, help_text='Start time of the sunrise.')
+ sunrise_end = DateTimeField(null=True, help_text='End time of the sunrise.')
+ sunset_start = DateTimeField(null=True, help_text='Start time of the sunset.')
+ sunset_end = DateTimeField(null=True, help_text='End time of the sunset.')
+ day_start = DateTimeField(null=True, help_text='Start time of the day.')
+ day_end = DateTimeField(null=True, help_text='End time of the day.')
+ night_start = DateTimeField(null=True, help_text='Start time of the night.')
+ night_end = DateTimeField(null=True, help_text='End time of the night.')
+
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
index d9352294552..aa30967cd2b 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
@@ -19,12 +19,13 @@ logger = logging.getLogger(__name__)
import inspect
import re
-from datetime import datetime, timezone
+from datetime import timezone, datetime, date
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp import viewsets
from lofar.sas.tmss.tmss.tmssapp.models.specification import *
from lofar.sas.tmss.tmss.tmssapp.models.scheduling import *
from lofar.sas.tmss.tmss.tmssapp.models.permissions import *
+from lofar.sas.tmss.tmss.tmssapp.conversions import timestamps_and_stations_to_sun_rise_and_set
from lofar.common import isTestEnvironment, isDevelopmentEnvironment
from concurrent.futures import ThreadPoolExecutor
from django.contrib.auth.models import User, Group, Permission
@@ -67,6 +68,9 @@ def populate_test_data():
from lofar.sas.tmss.tmss.tmssapp.subtasks import schedule_subtask
from lofar.common.json_utils import get_default_json_object_for_schema
+ # Maybe move to 'migrations populate'
+ populate_calculations()
+
constraints_template = models.SchedulingConstraintsTemplate.objects.get(name="constraints")
constraints_spec = get_default_json_object_for_schema(constraints_template.schema)
@@ -479,3 +483,47 @@ def populate_system_test_users():
guest_user.groups.add(Group.objects.get(name='Guest'))
lta_user = User.objects.create(username='lta_user', password='lta_user')
lta_user.groups.add(Group.objects.get(name='LTA User'))
+
+
+def populate_calculations():
+ """
+ Calculate a week of station timeline data of all stations
+ will take about a minute
+ TODO create a service which will do this continiously ?
+ """
+ starttime = datetime.utcnow()
+ logger.info("Populate sunrise, sunset, day, night for ALL stations from today until next week")
+ lst_timestamps = []
+ nbr_days = 7
+ for i in range(0, nbr_days):
+ dt = datetime.combine(date.today(), datetime.min.time()) + timedelta(days=i)
+ lst_timestamps.append(dt)
+
+ # Get all stations --> how to retrieve them?
+ lst_stations = ["CS001", "CS002", "CS003", "CS004", "CS005", "CS006", "CS007", "CS011", "CS013", "CS017", "CS021",
+ "CS024",
+ "CS026", "CS028", "CS030", "CS031", "CS032", "CS101", "CS103", "CS201", "CS301", "CS302", "CS401",
+ "CS501"]
+ # "RS104", "RS106", "RS205", "RS208", "RS210", "RS305", "RS306", "RS307", "RS310", "RS406", "RS407",
+ # "RS409",
+ # "RS410", "RS503", "RS508", "RS509",
+ # "DE601", "DE602", "DE603", "DE604", "DE605", "FR606", "SE607", "UK608", "DE609", "PL610", "PL611",
+ # "PL612",
+ # "IE613", "LV614"]
+
+ json_result = timestamps_and_stations_to_sun_rise_and_set(tuple(lst_timestamps), tuple(lst_stations))
+ for station in lst_stations:
+ for i in range(len(lst_timestamps)):
+ station_timeline = models.StationTimeline.objects.create(
+ station_name=station,
+ timestamp=lst_timestamps[i],
+ sunrise_start=json_result[station]['sunrise'][i]['start'],
+ sunrise_end=json_result[station]['sunrise'][i]['end'],
+ sunset_start=json_result[station]['sunset'][i]['start'],
+ sunset_end=json_result[station]['sunset'][i]['end'],
+ day_start=json_result[station]['day'][i]['start'],
+ day_end=json_result[station]['day'][i]['end'],
+ night_start= json_result[station]['night'][i]['start'],
+ night_end=json_result[station]['night'][i]['end'])
+ logger.debug("station_timeline %s created for station %s " % (station_timeline, station))
+ logger.info("Done in %.1fs", (datetime.utcnow()-starttime).total_seconds())
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/CMakeLists.txt b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/CMakeLists.txt
index 83a8174527b..f5f6fe38336 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/CMakeLists.txt
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/CMakeLists.txt
@@ -8,6 +8,7 @@ set(_py_files
widgets.py
common.py
permissions.py
+ calculations.py
)
python_install(${_py_files}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/__init__.py b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/__init__.py
index 0b0546b8d4b..3eb788371d9 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/__init__.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/__init__.py
@@ -1,4 +1,5 @@
from .specification import *
from .scheduling import *
from .common import *
-from .permissions import *
\ No newline at end of file
+from .permissions import *
+from .calculations import *
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/calculations.py b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/calculations.py
new file mode 100644
index 00000000000..8584228204e
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/calculations.py
@@ -0,0 +1,15 @@
+"""
+This file contains the serializers for conversion models
+"""
+
+import logging
+logger = logging.getLogger(__name__)
+
+from rest_framework import serializers
+from .. import models
+
+
+class StationTimelineSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.StationTimeline
+ fields = '__all__'
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/CMakeLists.txt b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/CMakeLists.txt
index 186d29924f2..ab71ce95fb8 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/CMakeLists.txt
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/CMakeLists.txt
@@ -8,7 +8,8 @@ set(_py_files
scheduling.py
permissions.py
project_permissions.py
- )
+ calculations.py
+ )
python_install(${_py_files}
DESTINATION lofar/sas/tmss/tmss/tmssapp/viewsets)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/__init__.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/__init__.py
index 0f7980fabfd..6f585af0a1c 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/__init__.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/__init__.py
@@ -1,4 +1,5 @@
from .specification import *
from .scheduling import *
from .permissions import *
-from .project_permissions import *
\ No newline at end of file
+from .project_permissions import *
+from .calculations import *
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/calculations.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/calculations.py
new file mode 100644
index 00000000000..fd7eb3fbfea
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/calculations.py
@@ -0,0 +1,13 @@
+from .. import models
+from .. import serializers
+from .lofar_viewset import LOFARViewSet
+
+
+#
+# Conversions ViewSets
+#
+
+class StationTimelineViewSet(LOFARViewSet):
+ queryset = models.StationTimeline.objects.all()
+ serializer_class = serializers.StationTimelineSerializer
+
diff --git a/SAS/TMSS/backend/src/tmss/urls.py b/SAS/TMSS/backend/src/tmss/urls.py
index 039b531a658..59624904853 100644
--- a/SAS/TMSS/backend/src/tmss/urls.py
+++ b/SAS/TMSS/backend/src/tmss/urls.py
@@ -224,6 +224,12 @@ router.register(r'sip_identifier', viewsets.SIPidentifierViewSet)
router.register(r'project_role', viewsets.ProjectRoleViewSet)
router.register(r'project_permission', viewsets.ProjectPermissionViewSet)
+
+# CONVERSIONS
+
+router.register(r'station_timeline', viewsets.StationTimelineViewSet)
+
+
urlpatterns.extend(router.urls)
frontend_urlpatterns = [
--
GitLab