diff --git a/SAS/TMSS/backend/services/CMakeLists.txt b/SAS/TMSS/backend/services/CMakeLists.txt
index d28add1121af1b33ff814fac4c6f5ba9c5a2e9c1..fcf450a17b3faff9e242cc8c0d0160b362129b36 100644
--- a/SAS/TMSS/backend/services/CMakeLists.txt
+++ b/SAS/TMSS/backend/services/CMakeLists.txt
@@ -10,4 +10,5 @@ lofar_add_package(TMSSSchedulingService scheduling)
lofar_add_package(TMSSSlackWebhookService slack_webhook)
lofar_add_package(TMSSWebSocketService websocket)
lofar_add_package(TMSSWorkflowService workflow_service)
+lofar_add_package(TMSSReportRefreshService report_refresh)
diff --git a/SAS/TMSS/backend/services/report_refresh/CMakeLists.txt b/SAS/TMSS/backend/services/report_refresh/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..de02703279e29aa6db07e43737780a3a9a5525e3
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/CMakeLists.txt
@@ -0,0 +1,14 @@
+lofar_package(TMSSWebSocketService 0.1 DEPENDS TMSSClient PyCommon pyparameterset PyMessaging) # also depends on TMSSBackend, but that dependency is added implicitely because this is a child package
+
+lofar_find_package(PythonInterp 3.6 REQUIRED)
+
+IF(NOT SKIP_TMSS_BUILD)
+ include(FindPythonModule)
+ find_python_module(SimpleWebSocketServer REQUIRED) # sudo pip3 install SimpleWebSocketServer
+
+ add_subdirectory(lib)
+ add_subdirectory(test)
+ENDIF(NOT SKIP_TMSS_BUILD)
+
+add_subdirectory(bin)
+
diff --git a/SAS/TMSS/backend/services/report_refresh/bin/CMakeLists.txt b/SAS/TMSS/backend/services/report_refresh/bin/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c61137bc0b7a662ae864ff6faa37f35d9d8eafca
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/bin/CMakeLists.txt
@@ -0,0 +1,2 @@
+lofar_add_bin_scripts(tmss_report_refresh_service)
+
diff --git a/SAS/TMSS/backend/services/report_refresh/bin/tmss_report_refresh_service b/SAS/TMSS/backend/services/report_refresh/bin/tmss_report_refresh_service
new file mode 100755
index 0000000000000000000000000000000000000000..e7e317c64f6b2e073a66b17ec916d589a7453a93
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/bin/tmss_report_refresh_service
@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+
+from lofar.sas.tmss.services.report_refresh import main
+
+if __name__ == "__main__":
+ main()
diff --git a/SAS/TMSS/backend/services/report_refresh/lib/CMakeLists.txt b/SAS/TMSS/backend/services/report_refresh/lib/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6d5d8984446c293661930b5f1e819db34a4c0c46
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/lib/CMakeLists.txt
@@ -0,0 +1,10 @@
+lofar_find_package(PythonInterp 3.4 REQUIRED)
+include(PythonInstall)
+
+set(_py_files
+ report_refresh.py
+ )
+
+python_install(${_py_files}
+ DESTINATION lofar/sas/tmss/services)
+
diff --git a/SAS/TMSS/backend/services/report_refresh/lib/report_refresh.py b/SAS/TMSS/backend/services/report_refresh/lib/report_refresh.py
new file mode 100644
index 0000000000000000000000000000000000000000..4202ca24ff3071be4e5abc8984430bea71033461
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/lib/report_refresh.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+# subtask_scheduling.py
+#
+# Copyright (C) 2015
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+from datetime import timedelta
+import logging
+import os
+from optparse import OptionParser, OptionGroup
+
+logger = logging.getLogger(__name__)
+
+from lofar.sas.tmss.client.tmssbuslistener import *
+from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
+
+class TMSSEventMessageHandlerForReportRefresh(TMSSEventMessageHandler):
+
+ def __init__(self, tmss_client_credentials_id: str="TMSSClient"):
+ super().__init__()
+ self.tmss_client = TMSSsession.create_from_dbcreds_for_ldap(tmss_client_credentials_id)
+
+ def start_handling(self):
+ self.tmss_client.open()
+
+ def stop_handling(self):
+ self.tmss_client.close()
+
+ def onSubTaskStatusChanged(self, id: int, status: str):
+
+ # if end state has been reached, request refresh of cycle report
+ if status in ['finished', 'cancelled', 'error']:
+ logger.info('Subtask id=%s has reached end state=%s' % (id, status))
+ from lofar.sas.tmss.tmss.tmssapp.models import Subtask
+ subtask = Subtask.objects.get(id=id)
+ for cycle in subtask.project.cycles.all():
+ logger.info('Requesting recalculation of report for cycle=%s because related subtask=%s has reached its end state=%s' % (cycle.pk, id, status))
+ self.tmss_client.get_path_as_json_object('/cycle/%s/report/?refresh=true' % cycle.pk)
+
+
+def create_service(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, tmss_client_credentials_id: str="TMSSClient"):
+ return TMSSBusListener(handler_type=TMSSEventMessageHandlerForReportRefresh,
+ handler_kwargs={'tmss_client_credentials_id': tmss_client_credentials_id},
+ exchange=exchange, broker=broker)
+
+
+def main():
+ # make sure we run in UTC timezone
+ os.environ['TZ'] = 'UTC'
+
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+ # Check the invocation arguments
+ parser = OptionParser('%prog [options]',
+ description='run the tmss_report_refresh_service which listens for TMSS event messages on the messagebus, and triggers a refresh of the cycle report on relevant changes.')
+
+ group = OptionGroup(parser, 'Messaging options')
+ group.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER,
+ help='Address of the message broker, default: %default')
+ group.add_option('-e', "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME,
+ help="exchange where the TMSS event messages are published. [default: %default]")
+ parser.add_option_group(group)
+
+ group = OptionGroup(parser, 'Django options')
+ parser.add_option_group(group)
+ group.add_option('-C', '--credentials', dest='dbcredentials', type='string', default=os.environ.get('TMSS_DBCREDENTIALS', 'TMSSClient'), help='django dbcredentials name, default: %default')
+
+ (options, args) = parser.parse_args()
+
+ from lofar.sas.tmss.tmss import setup_and_check_tmss_django_database_connection_and_exit_on_error
+ setup_and_check_tmss_django_database_connection_and_exit_on_error(options.dbcredentials)
+
+ with create_service(options.exchange, options.broker, options.dbcredentials):
+ waitForInterrupt()
+
+if __name__ == '__main__':
+ main()
diff --git a/SAS/TMSS/backend/services/report_refresh/requirements.txt b/SAS/TMSS/backend/services/report_refresh/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..601bc79fa354b84d437e1e17971621db27f43089
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/requirements.txt
@@ -0,0 +1,3 @@
+SimpleWebSocketServer # MIT
+Django # BSD3
+djangorestframework # BSD
\ No newline at end of file
diff --git a/SAS/TMSS/backend/services/report_refresh/test/CMakeLists.txt b/SAS/TMSS/backend/services/report_refresh/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7de2bd757aec594a18182e59a2d89674d70dc655
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/test/CMakeLists.txt
@@ -0,0 +1,9 @@
+# $Id: CMakeLists.txt 32679 2015-10-26 09:31:56Z schaap $
+
+if(BUILD_TESTING)
+ include(FindPythonModule)
+
+ include(LofarCTest)
+
+ lofar_add_test(t_report_refresh_service)
+endif()
diff --git a/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.py b/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.py
new file mode 100755
index 0000000000000000000000000000000000000000..c28354cefb0a259f4f0c377c408218d7802ddb2e
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+import time
+import unittest
+import uuid
+from datetime import datetime
+
+import logging
+logger = logging.getLogger('lofar.' + __name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+from lofar.messaging.messagebus import TemporaryExchange, BusListenerJanitor
+
+# create temporary queue
+TEST_UUID = uuid.uuid1()
+tmp_exchange = TemporaryExchange("%s_%s" % (__name__, TEST_UUID))
+tmp_exchange.open()
+
+# override DEFAULT_BUSNAME (before test env start)
+import lofar
+lofar.messaging.config.DEFAULT_BUSNAME = tmp_exchange.address
+
+# start test env
+from lofar.sas.tmss.test.test_environment import TMSSTestEnvironment
+tmss_test_env = TMSSTestEnvironment(exchange=tmp_exchange.address, populate_test_data=False, populate_schemas=False, start_websocket=False, start_postgres_listener=True, enable_viewflow=False)
+tmss_test_env.start()
+
+# import what's needed for the test (after test env start)
+from lofar.sas.tmss.services.report_refresh import create_service, TMSSEventMessageHandlerForReportRefresh
+from lofar.common.test_utils import integration_test
+from lofar.sas.tmss.test.tmss_test_data_django_models import *
+from lofar.sas.tmss.tmss.tmssapp import models
+
+def tearDownModule():
+ tmss_test_env.stop()
+ tmp_exchange.close()
+
+@integration_test
+class TestReportRefreshService(unittest.TestCase):
+ '''
+ Tests for the ReportRefreshService
+ '''
+
+ def test_refort_refresh_on_ending_subtask(self):
+ '''
+ This test starts a report refresh service. Creates and cancels a subtasl to check if this triggers the creation
+ of a cycle report in the database.
+ '''
+
+ # test data setup
+ project = models.Project.objects.create(**Project_test_data(name=str(uuid.uuid4())))
+ cycle = models.Cycle.objects.create(**Cycle_test_data(name=str(uuid.uuid4())))
+ project.cycles.set([cycle])
+ project.save()
+ scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data(project=project))
+ scheduling_unit_template = models.SchedulingUnitTemplate.get_latest(name='scheduling unit')
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data(name=str(uuid.uuid4()), template=scheduling_unit_template, scheduling_set=scheduling_set))
+ scheduling_unit_blueprint = models.SchedulingUnitBlueprint.objects.create(**SchedulingUnitBlueprint_test_data(name=str(uuid.uuid4()), draft=scheduling_unit_draft))
+ task_blueprint = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(name=str(uuid.uuid4()), scheduling_unit_blueprint=scheduling_unit_blueprint))
+ subtask_obs = models.Subtask.objects.create(**Subtask_test_data(state=models.SubtaskState.objects.get(value="defining"),
+ subtask_template=models.SubtaskTemplate.get_latest(name='observation control'),
+ task_blueprint=task_blueprint))
+ subtask_obs.state = models.SubtaskState.objects.get(value="defined")
+ subtask_obs.save()
+
+ # create and start the service (the object under test)
+ service = create_service(exchange=tmp_exchange.address, tmss_client_credentials_id=tmss_test_env.client_credentials.dbcreds_id)
+ with BusListenerJanitor(service):
+ # assert we did not have anything in the cache for the test cycle
+ self.assertEqual(models.CycleReport.objects.filter(cycle=cycle).count(), 0)
+
+ # trigger the service with a subtask status update
+ subtask_obs.state = models.SubtaskState.objects.get(value="error")
+ subtask_obs.save()
+
+ # wait for the service to do its thing
+ timeout = 30
+ start = datetime.utcnow()
+ while models.CycleReport.objects.filter(cycle=cycle).count() == 0:
+ if datetime.utcnow() - start > timedelta(seconds=timeout):
+ raise TimeoutError("timeout while waiting for cycle report refresh to get triggered.")
+ logger.info('waiting for cycle report refresh to get triggered.')
+ time.sleep(0.5)
+ while 'weekly_efficiency' not in models.CycleReport.objects.filter(cycle=cycle).first().data:
+ if datetime.utcnow() - start > timedelta(seconds=timeout):
+ raise TimeoutError("timeout while waiting for refreshed cycle report to get created.")
+ logger.info('waiting for refreshed cycle report to get created.')
+ time.sleep(0.5)
+
+ # assert we have a cache entry, so the refresh request worked
+ self.assertEqual(models.CycleReport.objects.filter(cycle=cycle).count(), 1)
+
+ # assert we have a valid report and not just a placeholder, so the actual cycle report generation worked
+ self.assertIn('weekly_efficiency', models.CycleReport.objects.filter(cycle=cycle).first().data)
+
+
+if __name__ == '__main__':
+ #run the unit tests
+ unittest.main()
diff --git a/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.run b/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.run
new file mode 100755
index 0000000000000000000000000000000000000000..1850968465f90b2db304a8cf12eb63afa73bbbf3
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.run
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "*tmss*" t_report_refresh_service.py
+
diff --git a/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.sh b/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.sh
new file mode 100755
index 0000000000000000000000000000000000000000..945fe16351f1507be81e11f57c27c908e1028288
--- /dev/null
+++ b/SAS/TMSS/backend/services/report_refresh/test/t_report_refresh_service.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_report_refresh_service
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0049_cyclereport.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0049_cyclereport.py
new file mode 100644
index 0000000000000000000000000000000000000000..eac1f1a73aabbb9d0e7595febc1ddd8800b6bae5
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0049_cyclereport.py
@@ -0,0 +1,32 @@
+# Generated by Django 3.0.9 on 2023-09-19 11:20
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('tmssapp', '0048_indices'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='CycleReport',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('cycle', models.OneToOneField(help_text='Cycle to which these quota apply.', on_delete=django.db.models.deletion.PROTECT, related_name='report', to='tmssapp.Cycle')),
+ ('data', django.contrib.postgres.fields.jsonb.JSONField(help_text='The report data')),
+ ('start', models.DateTimeField(help_text='Moment at which the reporting period starts.')),
+ ('stop', models.DateTimeField(help_text='Moment at which the reporting period ends.')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ],
+ ),
+ migrations.AddConstraint(
+ model_name='cyclereport',
+ constraint=models.UniqueConstraint(fields=('cycle', 'start', 'stop'),
+ name='cyclereport_unique_cycle_start_stop'))
+ ]
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
index 68b06075bc53eb1c6916acd096e2ccff6119e6fe..11b25f6fad982d1b84fec49b08b0748bdad254d6 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
@@ -33,6 +33,7 @@ from lofar.common.util import subdict_of_pointer_items
from .permissions import TMSSUser as User
import typing
import re
+import collections, time, threading
#
# I/O
@@ -858,6 +859,50 @@ class Cycle(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommonPK):
super().save(force_insert, force_update, using, update_fields)
+class CycleReport(BasicCommon):
+ '''This table caches cycle report data, which are costly to compute and easily time out.
+ We create these on demand and keep them in the database, since async requests are not supported by DRF.'''
+ cycle = OneToOneField('Cycle', related_name="report", on_delete=PROTECT, help_text='Cycle to which these quota apply.')
+ start = DateTimeField(help_text='Moment at which the reporting period starts.')
+ stop = DateTimeField(help_text='Moment at which the reporting period ends.')
+ data = JSONField(help_text='The report data')
+
+ class Meta:
+ constraints = [UniqueConstraint(fields=['cycle', 'start', 'stop'], name='%(class)s_unique_cycle_start_stop')]
+
+ @staticmethod
+ def __process_cycle_queue(cycle_queue: collections.deque):
+ '''background thread method to process cycles on the given queue into CycleReport entries'''
+ from lofar.sas.tmss.tmss.tmssapp.adapters.reports import create_cycle_report
+ while True:
+ try:
+ cycle, start, stop = cycle_queue.popleft()
+ report = CycleReport.objects.get(cycle=cycle, start=start, stop=stop)
+ report.data=create_cycle_report(cycle, start, stop)
+ report.save()
+ logger.info("CycleReport.process_cycle_queue: created CycleReport for cycle pk=%s", cycle.pk)
+ except IndexError:
+ # wait for work...
+ time.sleep(10)
+ except Exception as e:
+ logger.error("CycleReport.process_cycle_queue: %s", e)
+
+ # a queue, processing thread and lock for processing submitted cycles in the background
+ __cycle_queue = collections.deque()
+ __cycle_processor_thread = threading.Thread(target=__process_cycle_queue, daemon=True,
+ kwargs={'cycle_queue': __cycle_queue})
+ __cycle_processor_thread.start()
+ __lock = threading.Lock()
+
+ @classmethod
+ def submit_cycle_for_report_creation(cls, cycle, start, stop):
+ '''submit a cycle for which we want to create a report, which will be generated in a background thread'''
+ with cls.__lock:
+ if [cycle, start, stop] not in cls.__cycle_queue:
+ logger.info("submitting cycle id=%s to queue for background report creation...", cycle.pk)
+ cls.__cycle_queue.append([cycle, start, stop])
+
+
class CycleQuota(Model):
cycle = ForeignKey('Cycle', related_name="quota", on_delete=PROTECT, help_text='Cycle to which these quota apply.')
value = FloatField(help_text='Resource Quota value')
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
index 8c8b80cc842de549c63a536eb4a4c204cec4358b..0ab0a32c5c3e56e367890b56eb92f1602213f9ad 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
@@ -57,6 +57,7 @@ import dateutil
from django.core.exceptions import ObjectDoesNotExist
from django import forms
+
logger = logging.getLogger(__name__)
@@ -393,6 +394,7 @@ class CycleViewSet(LOFARViewSet):
cycle = get_object_or_404(models.Cycle, pk=pk)
# Check start and stop times
start, stop = request.query_params.get('start', None), request.query_params.get('stop', None)
+ refresh = request.query_params.get('refresh', None)
if start:
try:
start = dateutil.parser.parse(start, ignoretz=True)
@@ -409,9 +411,15 @@ class CycleViewSet(LOFARViewSet):
if not stop or stop > cycle.stop:
stop = cycle.stop
- result = create_cycle_report(cycle, start, stop)
- return Response(result, status=status.HTTP_200_OK)
-
+ if refresh:
+ # wipe all reports for this cycle (not just request period), so we have no outdated info in the cache.
+ logger.info('Refresh requested, wiping all cached reports for cycle=%s' % cycle.pk)
+ models.CycleReport.objects.filter(cycle=cycle).delete()
+ report = models.CycleReport.objects.filter(cycle=cycle, start=start, stop=stop).first()
+ if not report:
+ report = models.CycleReport.objects.create(cycle=cycle, start=start, stop=stop, data="Your report is getting prepared. Please check again in a few minutes")
+ models.CycleReport.submit_cycle_for_report_creation(cycle, start, stop)
+ return Response(report.data, status=status.HTTP_200_OK)
class CycleQuotaViewSet(LOFARViewSet):
queryset = models.CycleQuota.objects.all()
diff --git a/SAS/TMSS/backend/test/t_reports.py b/SAS/TMSS/backend/test/t_reports.py
index 6288d1eb16cfa35127d2eef5e079714c3f1bce3d..072072df825e9344479a2faae6d2402a8e43f74d 100755
--- a/SAS/TMSS/backend/test/t_reports.py
+++ b/SAS/TMSS/backend/test/t_reports.py
@@ -18,6 +18,7 @@
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
import os
+import time
import unittest
import requests
@@ -384,7 +385,14 @@ class ReportTest(unittest.TestCase):
Test create_cycle_report extra action.
"""
with tmss_test_env.create_tmss_client() as client:
- result = client.get_path_as_json_object('/cycle/%s/report' % self.cycle.pk)
+ for trial in range(10):
+ result = client.get_path_as_json_object('/cycle/%s/report' % self.cycle.pk)
+ if "report is getting prepared" not in result:
+ break
+ logger.info('waiting for report generation to complete...')
+ time.sleep(2)
+
+ self.assertNotIn("report is getting prepared", result)
# Assertions
diff --git a/SAS/TMSS/backend/test/test_environment.py b/SAS/TMSS/backend/test/test_environment.py
index 7a37aa1bb8cd0587b2d08f39cbbf1a5643639c58..148078e9ef16a408891d5f9cb0bd765dab8044b6 100644
--- a/SAS/TMSS/backend/test/test_environment.py
+++ b/SAS/TMSS/backend/test/test_environment.py
@@ -234,6 +234,7 @@ class TMSSTestEnvironment:
start_feedback_service: bool=False,
start_workflow_service: bool=False, enable_viewflow: bool=False,
start_precalculations_service: bool=False,
+ start_report_refresh_service: bool=False,
ldap_dbcreds_id: str=None, db_dbcreds_id: str=None, client_dbcreds_id: str=None):
self._exchange = exchange
self._broker = broker
@@ -288,6 +289,9 @@ class TMSSTestEnvironment:
self._start_precalculations_service = start_precalculations_service
self.precalculations_service = None
+ self._start_report_refresh_service = start_report_refresh_service
+ self.report_refresh_service = None
+
# Check for correct Django version, should be at least 3.0
if django.VERSION[0] < 3:
print("\nWARNING: YOU ARE USING DJANGO VERSION '%s', WHICH WILL NOT SUPPORT ALL FEATURES IN TMSS!\n" %
@@ -387,6 +391,12 @@ class TMSSTestEnvironment:
except Exception as e:
logger.exception(e)
+ if self._start_report_refresh_service:
+ from lofar.sas.tmss.services.report_refresh import create_service as create_report_refresh_service
+ self.report_refresh_service = create_report_refresh_service(exchange=self._exchange, broker=self._broker)
+ service_threads.append(threading.Thread(target=self.report_refresh_service.start_listening()))
+ service_threads[-1].start()
+
# wait for all services to be fully started in their background threads
for thread in service_threads:
thread.join()
@@ -691,6 +701,7 @@ def main_test_environment():
group.add_option('-w', '--websockets', dest='websockets', action='store_true', help='Enable json updates pushed via websockets')
group.add_option('-f', '--feedbackservice', dest='feedbackservice', action='store_true', help='Enable feedbackservice to handle feedback from observations/pipelines which comes in via the (old qpid) otdb messagebus.')
group.add_option('-C', '--precalculations_service', dest='precalculations_service', action='store_true', help='Enable the PreCalculations service')
+ group.add_option('-r', '--report_refresh_service', dest='report_refresh_service', action='store_true', help='Enable the ReportRefresh service')
group.add_option('--all', dest='all', action='store_true', help='Enable/Start all the services, upload schemas')
group.add_option('--simulate', dest='simulate', action='store_true', help='Simulate a run of the for each scheduled scheduling_unit (implies --eventmessages)')
@@ -736,6 +747,7 @@ def main_test_environment():
enable_viewflow=options.viewflow_app or options.viewflow_service or options.all,
start_workflow_service=options.viewflow_service or options.all,
start_precalculations_service=options.precalculations_service or options.all,
+ start_report_refresh_service=options.report_refresh_service or options.all,
ldap_dbcreds_id=options.LDAP_ID, db_dbcreds_id=options.DB_ID, client_dbcreds_id=options.REST_CLIENT_ID) as tmss_test_env:
# print some nice info for the user to use the test servers...
diff --git a/SAS/TMSS/deploy/docker-compose.yml b/SAS/TMSS/deploy/docker-compose.yml
index 8548e66686889364e0aafef0968d2d4f53fa5c39..833fc28248106f918a4572f4cb3cc80dbceb3b1d 100644
--- a/SAS/TMSS/deploy/docker-compose.yml
+++ b/SAS/TMSS/deploy/docker-compose.yml
@@ -301,3 +301,26 @@ services:
driver: journald
options:
tag: tmss_lobster
+ report_refresh:
+ container_name: tmss_report_refresh
+ image: tmss_report_refresh
+ build:
+ context: ./app
+ dockerfile: Dockerfile
+ args:
+ SOURCE_IMAGE: ${SOURCE_IMAGE}
+ HOME: "/localhome/lofarsys"
+ restart: unless-stopped
+ env_file:
+ - env
+ environment:
+ - USER=lofarsys
+ - HOME=/localhome/lofarsys
+ command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_report_refresh_service'
+ depends_on:
+ db_migrate:
+ condition: service_completed_successfully
+ logging:
+ driver: journald
+ options:
+ tag: tmss_report_refresh
\ No newline at end of file