L2SS-1444: Check exit status of system commands

SAS/TMSS/backend/services/lobster/lib/message_handler.py

@@ -49,6 +49,14 @@ LOOK_AHEAD_WINDOW = timedelta(minutes=3)
 POLL_INTERVAL     = timedelta(seconds=15)
 
 
+def _system(cmdline: str):
+    """Wraps os.system to raise a RuntimeError on failure."""
+
+    exit_status = system(cmdline)
+    if exit_status != 0:
+        raise RuntimeError(f"Execution failed, got exit status {exit_status} when executing: {cmdline}")
+
+
 class L2TMSSObservationControlMessageHandler(TMSSEventMessageHandler):
 
     def __init__(self, tmss_client_credentials_id: str=None):
@@ -146,12 +154,12 @@ class L2TMSSObservationControlMessageHandler(TMSSEventMessageHandler):
                 parset.write(l2parset)
 
             # copy it to COBALT
-            system(f"scp {tmpdir}/{PARSET_FILENAME} {COBALT_HEADNODE}:{COBALT_PARSET_DIR}/{PARSET_FILENAME}")
+            _system(f"scp {tmpdir}/{PARSET_FILENAME} {COBALT_HEADNODE}:{COBALT_PARSET_DIR}/{PARSET_FILENAME}")
 
             # kickstart the observation on COBALT to start and stop at the
             # times as provided in the parset.
             # first 3 parameters are historical and ignored
-            system(f"ssh {COBALT_HEADNODE} {COBALT_START_SCRIPT} 1 2 3 {COBALT_PARSET_DIR}/{PARSET_FILENAME} {subtask_id}")
+            _system(f"ssh {COBALT_HEADNODE} {COBALT_START_SCRIPT} 1 2 3 {COBALT_PARSET_DIR}/{PARSET_FILENAME} {subtask_id}")
 
 
     def enqueue_scheduled_observation_subtask(self, subtask: dict):