TMSS-666: retry

SAS/TMSS/client/lib/mains.py

@@ -121,7 +121,7 @@ def main_schedule_subtask():
 
     try:
         with TMSSsession.create_from_dbcreds_for_ldap() as session:
-            pprint(session.schedule_subtask(args.subtask_id))
+            pprint(session.schedule_subtask(args.subtask_id, retry_count=3))
     except Exception as e:
         print(e)
         exit(1)
@@ -134,7 +134,7 @@ def main_unschedule_subtask():
 
     try:
         with TMSSsession.create_from_dbcreds_for_ldap() as session:
-            pprint(session.unschedule_subtask(args.subtask_id))
+            pprint(session.unschedule_subtask(args.subtask_id, retry_count=3))
     except Exception as e:
         print(e)
         exit(1)
@@ -147,7 +147,7 @@ def main_cancel_subtask():
 
     try:
         with TMSSsession.create_from_dbcreds_for_ldap() as session:
-            pprint(session.cancel_subtask(args.subtask_id))
+            pprint(session.cancel_subtask(args.subtask_id, retry_count=3))
     except Exception as e:
         print(e)
         exit(1)

SAS/TMSS/client/lib/tmss_http_rest_client.py

@@ -25,6 +25,9 @@ class TMSSsession(object):
     OPENID = "openid"
     BASICAUTH = "basicauth"
 
+    POST_RETRY_COUNT = 0 # default number of retries (excluding the first normal attempt)
+    POST_RETRY_INTERVAL = 10 # default number of seconds between POST retries
+
     def __init__(self, username, password, host, port: int=8000, authentication_method=OPENID):
         self.session = requests.session()
         self.username = username
@@ -237,13 +240,12 @@ class TMSSsession(object):
             return result_object
         return result
 
-    def post_to_url_and_get_result_as_as_string(self, full_url: str, json_data:dict=None) -> str:
+    def post_to_url_and_get_result_as_as_string(self, full_url: str, json_data:dict=None, retry_count: int=POST_RETRY_COUNT, retry_interval: float=POST_RETRY_INTERVAL) -> str:
         '''post to the given full_url including http://<base_url>, and return the response as plain text
         Try to post, automatically retry 3 times with 10sec interval upon failure.
         '''
-        ATTEMPT_COUNT = 3
-        RETRY_INTERVAL = 10
-        for attempt_nr in range(ATTEMPT_COUNT):
+        attempt_count = retry_count+1
+        for attempt_nr in range(attempt_count):
             response = self.session.post(url=full_url, timeout=100000, json=json_data)
             logger.info("%s %s %s in %.1fms%s on %s", response.request.method.upper(), response.status_code, responses.get(response.status_code),
                                                       response.elapsed.total_seconds()*1000, ' SLOW!' if response.elapsed > timedelta(seconds=1) else '',
@@ -253,8 +255,8 @@ class TMSSsession(object):
                 result = response.content.decode('utf-8')
                 return result
 
-            if attempt_nr < ATTEMPT_COUNT:
-                time.sleep(RETRY_INTERVAL)
+            if attempt_nr < retry_count:
+                time.sleep(retry_interval)
 
         # ugly error message parsing
         content = response.text
@@ -265,14 +267,14 @@ class TMSSsession(object):
 
         raise Exception("Could not post to %s - %s %s - %s" % (full_url, response.status_code, responses.get(response.status_code), error_msg))
 
-    def post_to_url_and_get_result_as_json_object(self, full_url: str, json_data:dict=None) -> object:
+    def post_to_url_and_get_result_as_json_object(self, full_url: str, json_data:dict=None, retry_count: int=POST_RETRY_COUNT, retry_interval: float=POST_RETRY_INTERVAL) -> object:
         '''post to the given full_url (including http://<base_url>), and return the response as native object (usually a dict or a list of dicts)'''
-        result = self.post_to_url_and_get_result_as_as_string(full_url, json_data=json_data)
+        result = self.post_to_url_and_get_result_as_as_string(full_url, json_data=json_data, retry_count=retry_count, retry_interval=retry_interval)
         return json.loads(result)
 
-    def post_to_path_and_get_result_as_json_object(self, path: str, json_data:dict=None) -> object:
+    def post_to_path_and_get_result_as_json_object(self, path: str, json_data:dict=None, retry_count: int=POST_RETRY_COUNT, retry_interval: float=POST_RETRY_INTERVAL) -> object:
         '''post to the given path, and return the response as native object (usually a dict or a list of dicts)'''
-        return self.post_to_url_and_get_result_as_json_object(self.get_full_url_for_path(path=path), json_data=json_data)
+        return self.post_to_url_and_get_result_as_json_object(self.get_full_url_for_path(path=path), json_data=json_data, retry_count=retry_count, retry_interval=retry_interval)
 
     def _get_template(self, template_type_name: str, name: str, version: int=None) -> dict:
         '''get the template of the given type as dict for the given name (and version)'''
@@ -353,42 +355,42 @@ class TMSSsession(object):
             return result.content.decode('utf-8')
         raise Exception("Could not specify observation for task %s.\nResponse: %s" % (task_id, result))
 
-    def schedule_subtask(self, subtask_id: int, start_time: datetime=None) -> {}:
+    def schedule_subtask(self, subtask_id: int, start_time: datetime=None, retry_count: int=0) -> {}:
         """schedule the subtask for the given subtask_id at the given start_time. If start_time==None then already (pre)set start_time is used.
         returns the scheduled subtask upon success, or raises."""
         if start_time is not None:
             self.session.patch(self.get_full_url_for_path('subtask/%s' % subtask_id), {'start_time': datetime.utcnow()})
-        return self.post_to_path_and_get_result_as_json_object('subtask/%s/schedule' % subtask_id)
+        return self.post_to_path_and_get_result_as_json_object('subtask/%s/schedule' % (subtask_id), retry_count=retry_count)
 
-    def unschedule_subtask(self, subtask_id: int) -> {}:
+    def unschedule_subtask(self, subtask_id: int, retry_count: int=0) -> {}:
         """unschedule the subtask for the given subtask_id.
         returns the unscheduled subtask upon success, or raises."""
-        return self.post_to_path_and_get_result_as_json_object('subtask/%s/unschedule' % subtask_id)
+        return self.post_to_path_and_get_result_as_json_object('subtask/%s/unschedule' % (subtask_id), retry_count=retry_count)
 
-    def cancel_subtask(self, subtask_id: int) -> {}:
+    def cancel_subtask(self, subtask_id: int, retry_count: int=0) -> {}:
         """cancel the subtask for the given subtask_id, either preventing it to start, or to kill it while running.
         returns the cancelled subtask upon success, or raises."""
-        return self.post_to_path_and_get_result_as_json_object('subtask/%s/cancel' % subtask_id)
+        return self.post_to_path_and_get_result_as_json_object('subtask/%s/cancel' % (subtask_id), retry_count=retry_count)
 
-    def cancel_task_blueprint(self, task_blueprint_id: int) -> {}:
+    def cancel_task_blueprint(self, task_blueprint_id: int, retry_count: int=0) -> {}:
         """cancel the task_blueprint for the given task_blueprint_id, either preventing it to start, or to kill it while running.
         returns the cancelled task_blueprint upon success, or raises."""
-        return self.post_to_path_and_get_result_as_json_object('task_blueprint/%s/cancel' % task_blueprint_id)
+        return self.post_to_path_and_get_result_as_json_object('task_blueprint/%s/cancel' % (task_blueprint_id), retry_count=retry_count)
 
-    def cancel_scheduling_unit_blueprint(self, scheduling_unit_blueprint_id: int) -> {}:
+    def cancel_scheduling_unit_blueprint(self, scheduling_unit_blueprint_id: int, retry_count: int=0) -> {}:
         """cancel the scheduling_unit_blueprint for the given scheduling_unit_blueprint_id, either preventing it to start, or to kill it while running.
         returns the cancelled scheduling_unit_blueprint upon success, or raises."""
-        return self.post_to_path_and_get_result_as_json_object('scheduling_unit_blueprint/%s/cancel' % scheduling_unit_blueprint_id)
+        return self.post_to_path_and_get_result_as_json_object('scheduling_unit_blueprint/%s/cancel' % (scheduling_unit_blueprint_id), retry_count=retry_count)
 
-    def create_blueprints_and_subtasks_from_scheduling_unit_draft(self, scheduling_unit_draft_id: int) -> {}:
+    def create_blueprints_and_subtasks_from_scheduling_unit_draft(self, scheduling_unit_draft_id: int, retry_count: int=0) -> {}:
         """create a scheduling_unit_blueprint, its specified taskblueprints and subtasks for the given scheduling_unit_draft_id.
         returns the scheduled subtask upon success, or raises."""
-        return self.post_to_path_and_get_result_as_json_object('scheduling_unit_draft/%s/create_blueprints_and_subtasks' % scheduling_unit_draft_id)
+        return self.post_to_path_and_get_result_as_json_object('scheduling_unit_draft/%s/create_blueprints_and_subtasks' % (scheduling_unit_draft_id), retry_count=retry_count)
 
-    def create_scheduling_unit_draft_from_strategy_template(self, scheduling_unit_observing_strategy_template_id: int, parent_scheduling_set_id: int) -> {}:
+    def create_scheduling_unit_draft_from_strategy_template(self, scheduling_unit_observing_strategy_template_id: int, parent_scheduling_set_id: int, retry_count: int=0) -> {}:
         """create a scheduling_unit_blueprint, its specified taskblueprints and subtasks for the given scheduling_unit_draft_id.
         returns the created scheduling_unit_draft upon success, or raises."""
-        return self.post_to_path_and_get_result_as_json_object('scheduling_unit_observing_strategy_template/%s/create_scheduling_unit?scheduling_set_id=%s' % (scheduling_unit_observing_strategy_template_id, parent_scheduling_set_id))
+        return self.post_to_path_and_get_result_as_json_object('scheduling_unit_observing_strategy_template/%s/create_scheduling_unit?scheduling_set_id=%s' % (scheduling_unit_observing_strategy_template_id, parent_scheduling_set_id), retry_count=retry_count)
 
     def get_schedulingunit_draft(self, scheduling_unit_draft_id: str, extended: bool=True) -> dict:
         '''get the schedulingunit_draft as dict for the given scheduling_unit_draft_id. When extended==True then you get the full scheduling_unit,task,subtask tree.'''