Task #10986: Reintegrated task branch RT-Task10986 (fixed endless loop in...

Task #10986: Reintegrated task branch RT-Task10986 (fixed endless loop in TriggerEmailService) into trunk

.gitattributes

@@ -5308,7 +5308,7 @@ SAS/TriggerEmailService/Common/config.py -text
 SAS/TriggerEmailService/Server/CMakeLists.txt -text
 SAS/TriggerEmailService/Server/bin/CMakeLists.txt -text
 SAS/TriggerEmailService/Server/bin/TriggerEmailService -text
-SAS/TriggerEmailService/Server/bin/TriggerEmailService.ini_stopped -text
+SAS/TriggerEmailService/Server/bin/TriggerEmailService.ini -text
 SAS/TriggerEmailService/Server/lib/CMakeLists.txt -text
 SAS/TriggerEmailService/Server/lib/Templates.py -text
 SAS/TriggerEmailService/Server/lib/TriggerEmailService.py -text

SAS/TriggerEmailService/Server/lib/TriggerEmailService.py

@@ -31,6 +31,7 @@ from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
 from datetime import timedelta, datetime
+import time
 from lofar.sas.otdb.OTDBBusListener import OTDBBusListener
 from lofar.common.util import waitForInterrupt
 from lofar.messaging.messagebus import AbstractBusListener
@@ -75,6 +76,10 @@ def email(recipients, subject, body, attachment, attachment_name):
     s.quit()
 
 
+class MoMIdError(Exception):
+    pass
+
+
 class OTDBTriggerListener(OTDBBusListener):
     def __init__(self, momquery_rpc=MoMQueryRPC(), busname=DEFAULT_OTDB_NOTIFICATION_BUSNAME,
                  subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, broker=None, **kwargs):
@@ -106,19 +111,39 @@ class OTDBTriggerListener(OTDBBusListener):
         self.when_trigger_send_email(otdb_id, REJECTED_TEMPLATE_SUBJECT, REJECTED_TEMPLATE_BODY)
 
     def when_trigger_send_email(self, otdb_id, template_subject, template_body):
-        mom_id = None
+        try:
+            mom_id, trigger_id = self._get_mom_and_trigger_id(otdb_id)
 
-        while not mom_id:  # we are sometimes to fast for MOM
-            mom_id = self.mom_rpc_client.getMoMIdsForOTDBIds(otdb_id)[otdb_id]
+            if trigger_id:
+                self._send_email(otdb_id, mom_id, trigger_id, template_subject, template_body)
+
+        except MoMIdError:
+            logger.error("Could not retrieve a mom_id for otdb_id: %s", otdb_id)
+
+    def _get_mom_and_trigger_id(self, otdb_id):
+        mom_id = self._try_get_mom_id(otdb_id)
+
+        if not mom_id:
+            raise MoMIdError
 
         trigger_id = self.mom_rpc_client.get_trigger_id(mom_id)['trigger_id']
 
-        if trigger_id:
-            logger.info("Emailing otdb_id: %s, mom_id: %s, trigger_id: %s, template_subject: %s, template_body: %s",
-                        otdb_id, mom_id, trigger_id, template_subject, template_body)
-            self._send_email(otdb_id, mom_id, trigger_id, template_subject, template_body)
+        return mom_id, trigger_id
+
+    def _try_get_mom_id(self, otdb_id):
+        # sometimes we are too fast for MoM so we need to retry
+        mom_id = None
+        for _ in xrange(10):
+            mom_id = self.mom_rpc_client.getMoMIdsForOTDBIds(otdb_id)[otdb_id]
+            if mom_id:
+                break
+            time.sleep(3)
+        return mom_id
 
     def _send_email(self, otdb_id, mom_id, trigger_id, template_subject, template_body):
+        logger.info("Emailing otdb_id: %s, mom_id: %s, trigger_id: %s, template_subject: %s, template_body: %s",
+                    otdb_id, mom_id, trigger_id, template_subject, template_body)
+
         subject, body = self._fill_template(otdb_id, mom_id, trigger_id, template_subject, template_body)
         recipients = self._get_recipients(mom_id)
 

SAS/TriggerEmailService/Server/test/t_TriggerEmailService.py

@@ -73,6 +73,10 @@ class TestOTDBTriggerListener(unittest.TestCase):
         self.addCleanup(email_patcher.stop)
         self.email_mock = email_patcher.start()
 
+        logger_patcher = mock.patch('lofar.sas.TriggerEmailService.TriggerEmailService.logger')
+        self.addCleanup(logger_patcher.stop)
+        self.logger_mock = logger_patcher.start()
+
     @mock.patch('lofar.sas.otdb.OTDBBusListener.OTDBBusListener.start_listening')
     def test_start_listening_should_open_momquery_rpc(self, super_mock):
         listener = OTDBTriggerListener(self.momqueryrpc_mock)
@@ -236,6 +240,48 @@ class TestOTDBTriggerListener(unittest.TestCase):
         self.assertIn(str(self.trigger_id), self.email_mock.call_args[0][2])
         self.assertIn(self.project_name, self.email_mock.call_args[0][2])
 
+    # when_trigger_send_email tests
+
+    @mock.patch('time.sleep', return_value=None)
+    def test_when_trigger_send_email_should_limit_the_amount_of_requests(self, _):
+        self.momqueryrpc_mock.getMoMIdsForOTDBIds.return_value = {self.obs_sas_id: None}
+
+        listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+        listener.onObservationAborted(self.obs_sas_id, None)
+
+        self.assertEqual(10, self.momqueryrpc_mock.getMoMIdsForOTDBIds.call_count)
+
+    @mock.patch('time.sleep', return_value=None)
+    def test_when_trigger_send_email_should_log_when_no_mom_id_can_be_found(self, _):
+        self.momqueryrpc_mock.getMoMIdsForOTDBIds.return_value = {self.obs_sas_id: None}
+
+        listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+        listener.onObservationAborted(self.obs_sas_id, None)
+
+        self.logger_mock.error.assert_any_call("Could not retrieve a mom_id for otdb_id: %s", self.obs_sas_id)
+
+    @mock.patch('time.sleep', return_value=None)
+    def test_when_trigger_send_email_should_wait_three_seconds_between_retries(self, sleep_mock):
+        self.momqueryrpc_mock.getMoMIdsForOTDBIds.return_value = {self.obs_sas_id: None}
+
+        listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+        listener.onObservationAborted(self.obs_sas_id, None)
+
+        sleep_mock.assert_called_with(3)
+        self.assertEqual(10, sleep_mock.call_count)
+
+    def test_when_trigger_send_email_should_log_when_sending_email(self):
+        listener = OTDBTriggerListener(self.momqueryrpc_mock)
+
+        listener.when_trigger_send_email(self.obs_sas_id, "", "")
+
+        self.logger_mock.info.assert_any_call(
+            "Emailing otdb_id: %s, mom_id: %s, trigger_id: %s, template_subject: %s, template_body: %s",
+            self.obs_sas_id, self.obs_mom_id, self.trigger_id, "", "")
+
 
 class TestTriggerNotificationListener(unittest.TestCase):
     project_name = "test_lofar"