Task #10339: applied patch from 2.19: when setting status and message, set...

Task #10339: applied patch from 2.19: when setting status and message, set status first and then message, otherwise mom can't handle it.

LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/momclient.py

@@ -74,7 +74,7 @@ class MoMClient:
 
             self.__logged_in = True
         except Exception as e:
-            raise Exception("Logging into MoM failed: " + str(e))
+            raise Exception("Logging into MoM on %s failed: %s" % (self.__momURLlogin, str(e)))
 
     def __logout(self):
         try:
@@ -92,32 +92,60 @@ class MoMClient:
 
     def setStatus(self, export_id, status_id, message=None):
         try:
-            if not self.__logged_in:
-                self.__login()
-
-            params = {"exportId" : export_id, "status" : status_id}
-            if message:
-                if len(message) > 100:
-                    logger.info('truncating message to 100 characters because MoM cannot store more')
-                    message = message[:97] + '...'
-
-                params['message'] = message
-
-            statusUrl = self.__momURLsetStatus + '?' + urllib.urlencode(params)
-            logger.debug("updating MoM: " + statusUrl)
-            response = self.__browser.open(statusUrl)
-            reply = response.readlines()
-            if reply == ['ok']:
-                logger.info('MoMClient.setStatus updated status of %s to %s', export_id, jobState2String(int(status_id)))
-                return True
-            else:
-                logger.error('MoMClient.setStatus could not update status of %s to %s using url: %s  reply: %s',
-                             export_id,
-                             jobState2String(int(status_id)),
-                             statusUrl,
-                             reply)
-                self.__logout()
-                return False
+            # mom is quite reluctant in updating the status
+            # often it returns a login page, even when you're logged in
+            # so, upon error, retry a couple of times with a pause, else just return
+            for mom_retry in range(self.MAX_MOM_RETRIES):
+                if not self.__logged_in:
+                    self.__login()
+
+                params = {"exportId" : export_id, "status" : status_id}
+                statusUrl = self.__momURLsetStatus + '?' + urllib.urlencode(params)
+                logger.debug("updating MoM: " + statusUrl)
+                response = self.__browser.open(statusUrl)
+                reply = response.readlines()
+                if reply == ['ok']:
+                    logger.info('MoMClient.setStatus updated status of %s to %s', export_id, jobState2String(int(status_id)))
+
+                    if message:
+                        # even though the mom api suggests that one could update the status and message at the same time
+                        # this does not work. If the status changes, and there is a message, then the status is ignored.
+                        # So, let's fool mom, and update the status again, this time including the message
+                        # In this second call, the status won't change, because we already just changed it.
+                        if len(message) > 100:
+                            logger.info('truncating message to 100 characters because MoM cannot store more')
+                            message = message[:97] + '...'
+
+                        params['message'] = message
+
+                        statusUrl = self.__momURLsetStatus + '?' + urllib.urlencode(params)
+                        logger.debug("updating MoM: " + statusUrl)
+                        response = self.__browser.open(statusUrl)
+                        reply = response.readlines()
+                        if reply == ['ok']:
+                            logger.info('MoMClient.setStatus updated status of %s to %s with message: %s',
+                                        export_id,
+                                        jobState2String(int(status_id)),
+                                        message)
+                        # if the message update did not succeed, we don't really care
+                        # the status update already succeeded, and that's important.
+                    return True
+                else:
+                    logger.error('MoMClient.setStatus could not update status of %s to %s using url: %s  reply: %s',
+                                export_id,
+                                jobState2String(int(status_id)),
+                                statusUrl,
+                                reply)
+                    self.__logout()
+
+                    if 'DOCTYPE HTML PUBLIC' in reply:
+                        logger.error('MoM returned login screen instead of SIP for archive_id=%s mom_id=%s using url %s and data %s', archive_id, mom_id, self.__momURLgetSIP, data)
+
+                        wait_secs = (mom_retry+1)*(mom_retry+1)*10
+                        logger.info('Retrying to setStatus for archiveId %s in %s seconds', archive_id, wait_secs)
+                        time.sleep(wait_secs)
+                        continue #jump back to for mom_retry in range(self.MAX_MOM_RETRIES)
+
         except Exception as e:
             logger.error('MoMClient.setStatus could not update status of %s to %s: %s', export_id,
                                                                                         jobState2String(int(status_id)),