Merge branch 'TMSS-435' into 'master'

Resolve TMSS-435

Closes TMSS-435

See merge request !269

Docker/lofar-ci/Dockerfile_ci_sas

@@ -16,7 +16,7 @@ RUN yum erase -y postgresql postgresql-server postgresql-devel && \
     cd /bin && ln -s /usr/pgsql-9.6/bin/initdb && ln -s /usr/pgsql-9.6/bin/postgres
 ENV PATH /usr/pgsql-9.6/bin:$PATH 
 
-RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil Django==3.0.9 djangorestframework==3.11.1 djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet pyxb==1.2.5 graphviz isodate astropy packaging django-debug-toolbar pymysql
+RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil Django==3.0.9 djangorestframework==3.11.1 djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet pyxb==1.2.5 graphviz isodate astropy packaging django-debug-toolbar pymysql astroplan
 
 #Viewflow package 
 RUN pip3 install django-material django-viewflow

SAS/TMSS/src/tmss/settings.py

@@ -137,7 +137,6 @@ DEBUG_TOOLBAR_CONFIG = {
 
 MIDDLEWARE = [
     'django.middleware.gzip.GZipMiddleware',
-    'debug_toolbar.middleware.DebugToolbarMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',

SAS/TMSS/src/tmss/tmssapp/conversions.py

@@ -3,6 +3,60 @@ import astropy.units
 from lofar.lta.sip import station_coordinates
 from datetime import datetime
 from astropy.coordinates.earth import EarthLocation
+from astropy.coordinates import Angle
+from astroplan.observer import Observer
+
+
+def create_astroplan_observer_for_station(station: str) -> Observer:
+    '''
+    returns an astroplan observer for object for a given station, located in the LBA center of the given station
+    :param station: a station name, e.g. "CS002"
+    :return: astroplan.observer.Observer object
+    '''
+
+    coords = station_coordinates.parse_station_coordinates()["%s_LBA" % station.upper()]
+    location = EarthLocation.from_geocentric(x=coords['x'], y=coords['y'], z=coords['z'],  unit=astropy.units.m)
+    observer = Observer(location, name="LOFAR", timezone="UTC")
+    return observer
+
+# default angle to the horizon at which the sunset/sunrise starts and ends, as per LOFAR definition.
+SUN_SET_RISE_ANGLE_TO_HORIZON = Angle(10, unit=astropy.units.deg)
+
+def timestamps_and_stations_to_sun_rise_and_set(timestamps: [datetime], stations: [str], angle_to_horizon: Angle=SUN_SET_RISE_ANGLE_TO_HORIZON) -> dict:
+    """
+    compute sunrise, sunset, day and night of the given stations at the given timestamps
+    :param timestamps: list of datetimes, e.g. [datetime(2020, 1, 1), datetime(2020, 1, 2)]
+    :param stations: list of station names, e.g. ["CS001"]
+    :return A dict that maps station names to a nested dict that contains lists of start and end times for sunrise, sunset, etc, on each requested date.
+        E.g.
+        {"CS001":
+            {   "sunrise": [{"start": (2020, 1, 1, 6, 0, 0)), "end": (2020, 1, 1, 6, 30, 0)},
+                            {"start": (2020, 1, 2, 6, 0, 0)), "end": (2020, 1, 2, 6, 30, 0)}],
+                "sunset": [{"start": (2020, 1, 1, 18, 0, 0)), "end": (2020, 1, 1, 18, 30, 0)},
+                           {"start": (2020, 1, 2, 18, 0, 0)), "end": (2020, 1, 2, 18, 30, 0)}],
+                "day": [{"start": (2020, 1, 1, 6, 30, 0)), "end": (2020, 1, 1, 18, 00, 0)},
+                        {"start": (2020, 1, 2, 6, 30, 0)), "end": (2020, 1, 2, 18, 00, 0)}],
+                "night": [{"start": (2020, 1, 1, 18, 30, 0)), "end": (2020, 1, 2, 6, 0, 0)},
+                          {"start": (2020, 1, 2, 18,3 0, 0)), "end": (2020, 1, 3, 6, 0, 0)}],
+            }
+        }
+    """
+    return_dict = {}
+    for station in stations:
+        for timestamp in timestamps:
+            observer = create_astroplan_observer_for_station(station)
+            sunrise_start = observer.sun_rise_time(time=Time(timestamp), which='previous')
+            if sunrise_start.to_datetime().date() < timestamp.date():
+                sunrise_start = observer.sun_rise_time(time=Time(timestamp), horizon=-angle_to_horizon, which='next')
+            sunrise_end = observer.sun_rise_time(time=Time(timestamp), horizon=angle_to_horizon, which='next')
+            sunset_start = observer.sun_set_time(time=sunrise_end, horizon=angle_to_horizon, which='next')
+            sunset_end = observer.sun_set_time(time=sunrise_end, horizon=-angle_to_horizon, which='next')
+            sunrise_next_start = observer.sun_rise_time(time=sunset_end, horizon=-angle_to_horizon, which='next')
+            return_dict.setdefault(station, {}).setdefault("sunrise", []).append({"start": sunrise_start.to_datetime(), "end": sunrise_end.to_datetime()})
+            return_dict[station].setdefault("sunset", []).append({"start": sunset_start.to_datetime(), "end": sunset_end.to_datetime()})
+            return_dict[station].setdefault("day", []).append({"start": sunrise_end.to_datetime(), "end": sunset_start.to_datetime()})
+            return_dict[station].setdefault("night", []).append({"start": sunset_end.to_datetime(), "end": sunrise_next_start.to_datetime()})
+    return return_dict
 
 
 def local_sidereal_time_for_utc_and_station(timestamp: datetime = None,

SAS/TMSS/src/tmss/tmssapp/views.py

@@ -4,15 +4,17 @@ from django.http import HttpResponse, JsonResponse, Http404
 from django.shortcuts import get_object_or_404, render
 from lofar.sas.tmss.tmss.tmssapp import models
 from lofar.common.json_utils import get_default_json_object_for_schema
+from lofar.common.datetimeutils import formatDatetime
 from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset
 from drf_yasg.utils import swagger_auto_schema
+from drf_yasg.openapi import Parameter
 from rest_framework.permissions import AllowAny
 from rest_framework.decorators import authentication_classes, permission_classes
 from django.apps import apps
 
 from datetime import datetime
 import dateutil.parser
-from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_and_station, local_sidereal_time_for_utc_and_longitude
+from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_and_station, local_sidereal_time_for_utc_and_longitude, timestamps_and_stations_to_sun_rise_and_set
 
 def subtask_template_default_specification(request, subtask_template_pk:int):
     subtask_template = get_object_or_404(models.SubtaskTemplate, pk=subtask_template_pk)
@@ -84,10 +86,24 @@ def get_stations_in_group(request, template_name:str, template_version:str, stat
                          'stations': stations})
 
 
+@permission_classes([AllowAny])
+@authentication_classes([AllowAny])
+@swagger_auto_schema(responses={200: 'An isoformat timestamp of the current UTC clock of the system'},
+                     operation_description="Get the current system time in UTC")
 def utc(request):
     return HttpResponse(datetime.utcnow().isoformat(), content_type='text/plain')
 
-
+@permission_classes([AllowAny])
+@authentication_classes([AllowAny])
+@swagger_auto_schema(responses={200: 'The LST time in hms format at the given UTC time and station or longitude'},
+                     operation_description="Get LST time for UTC time and station or longitude",
+                     manual_parameters=[Parameter(name='station', required=False, type='string', in_='query',
+                                                  description="A station names (defaults to CS002)"),
+                                        Parameter(name='timestamp', required=False, type='string', in_='query',
+                                                  description="A timestamp in isoformat (defaults to utcnow)"),
+                                        Parameter(name='longitude', required=False, type='float', in_='query',
+                                                  description="A longitude")
+                                        ])
 def lst(request):
     # Handling optional parameters via django paths in urls.py is a pain, we access them on the request directly instead.
     timestamp = request.GET.get('timestamp', None)
@@ -109,4 +125,33 @@ def lst(request):
         lst_lon = local_sidereal_time_for_utc_and_station(timestamp)
 
     # todo: do we want to return a dict, so users can make sure their parameters were parsed correctly instead?
-    return HttpResponse(str(lst_lon), content_type='text/plain')
\ No newline at end of file
+    return HttpResponse(str(lst_lon), content_type='text/plain')
+
+
+@permission_classes([AllowAny])
+@authentication_classes([AllowAny])
+@swagger_auto_schema(responses={200: 'A JSON object with sunrise, sunset, day and night of the given stations at the given timestamps'},
+                     operation_description="Get sunrise, sunset, day and night for stations and timestamps",
+                     manual_parameters=[Parameter(name='stations', required=False, type='string', in_='query',
+                                                  description="comma-separated list of station names"),
+                                        Parameter(name='timestamps', required=False, type='string', in_='query',
+                                                  description="comma-separated list of isoformat timestamps")])
+def get_sun_rise_and_set(request):
+    """
+    returns sunrise and sunset at the given stations and timestamps, or today at LOFAR core if none specified.
+    example request: /api/util/sun_rise_and_set?stations=CS002,CS005&timestamps=2020-05-01,2020-09-09T11-11-00
+    """
+    timestamps = request.GET.get('timestamps', None)
+    stations = request.GET.get('stations', None)
+    if timestamps is None:
+        timestamps = [datetime.utcnow()]
+    else:
+        timestamps = timestamps.split(',')
+        timestamps = [dateutil.parser.parse(timestamp) for timestamp in timestamps]  #  isot to datetime
+    if stations is None:
+        stations = ['CS002']
+    else:
+        stations = stations.split(',')
+
+    return JsonResponse(timestamps_and_stations_to_sun_rise_and_set(timestamps, stations))
+

SAS/TMSS/src/tmss/urls.py

@@ -65,6 +65,7 @@ urlpatterns = [
     path('schemas/<str:template>/<str:name>/<str:version>/', views.get_template_json_schema, name='get_template_json_schema'),
     path('station_groups/<str:template_name>/<str:template_version>/<str:station_group>', views.get_stations_in_group, name='get_stations_in_group'), #TODO: how to make trailing slash optional?
     path('station_groups/<str:template_name>/<str:template_version>/<str:station_group>/', views.get_stations_in_group, name='get_stations_in_group'),
+    path('util/sun_rise_and_set', views.get_sun_rise_and_set, name='get_sun_rise_and_set'),
     path(r'util/utc', views.utc, name="system-utc"),
     path(r'util/lst', views.lst, name="conversion-lst"),
 ]

SAS/TMSS/test/t_conversions.py

@@ -26,6 +26,7 @@ import logging
 import requests
 import dateutil.parser
 import astropy.coordinates
+import json
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
@@ -127,6 +128,43 @@ class UtilREST(unittest.TestCase):
         lon_str2 = r2.content.decode('utf8')
         self.assertNotEqual(lon_str1, lon_str2)
 
+    def test_util_sun_rise_and_set_returns_json_structure_with_defaults(self):
+        r = requests.get(BASE_URL + '/util/sun_rise_and_set', auth=AUTH)
+        self.assertEqual(r.status_code, 200)
+        r_dict = json.loads(r.content.decode('utf-8'))
+
+        # assert defaults to core and today
+        self.assertIn('CS002', r_dict.keys())
+        sunrise_start = dateutil.parser.parse(r_dict['CS002']['sunrise'][0]['start'])
+        self.assertEqual(datetime.date.today(), sunrise_start.date())
+
+    def test_util_sun_rise_and_set_considers_stations(self):
+        stations = ['CS005', 'RS305', 'DE609']
+        r = requests.get(BASE_URL + '/util/sun_rise_and_set?stations=%s' % ','.join(stations), auth=AUTH)
+        self.assertEqual(r.status_code, 200)
+        r_dict = json.loads(r.content.decode('utf-8'))
+
+        # assert station is included in response and timestamps differ
+        sunset_start_last = None
+        for station in stations:
+            self.assertIn(station, r_dict.keys())
+            sunset_start = dateutil.parser.parse(r_dict[station]['sunset'][0]['start'])
+            if sunset_start_last:
+                self.assertNotEqual(sunset_start, sunset_start_last)
+            sunset_start_last = sunset_start
+
+    def test_util_sun_rise_and_set_considers_timestamps(self):
+        timestamps = ['2020-01-01', '2020-02-22T16-00-00', '2020-3-11', '2020-01-01']
+        r = requests.get(BASE_URL + '/util/sun_rise_and_set?timestamps=%s' % ','.join(timestamps), auth=AUTH)
+        self.assertEqual(r.status_code, 200)
+        r_dict = json.loads(r.content.decode('utf-8'))
+
+        # assert all requested timestamps are included in response (sunrise on same day)
+        for i in range(len(timestamps)):
+            expected_date = dateutil.parser.parse(timestamps[i]).date()
+            response_date = dateutil.parser.parse(r_dict['CS002']['sunrise'][i]['start']).date()
+            self.assertEqual(expected_date, response_date)
+
 
 if __name__ == "__main__":
     os.environ['TZ'] = 'UTC'