diff --git a/SAS/TMSS/backend/services/copy_service/copy_service.py b/SAS/TMSS/backend/services/copy_service/copy_service.py index 8da9f0bde1dc06455c066235b74f19f774246bb0..9609ee80241390ded10c2b8e73c3db2f43fc4655 100755 --- a/SAS/TMSS/backend/services/copy_service/copy_service.py +++ b/SAS/TMSS/backend/services/copy_service/copy_service.py @@ -80,44 +80,47 @@ class TMSSCopyServiceEventMessageHandler(TMSSEventMessageHandler): if subtask['state_value'] != 'queued': return - # determine destination host and root_dir - destination = subtask['specifications_doc']['destination'] - dst_host = destination[:destination.find(':')] if ':' in destination else '' - dst_root_dir = '/'+destination.lstrip(dst_host + ':').split('/')[1] - # strip unneeded localhost to prevent unneeded ssh calls - dst_host = dst_host.lstrip('localhost:').lstrip('127.0.0.1:') - - df_cmd = ['df', dst_root_dir] - if dst_host: - df_cmd = wrap_command_in_ssh_call(df_cmd, dst_host) - - logger.info("checking free disk space for copy-subtask id=%s, executing: %s", subtask['id'], ' '.join(df_cmd)) - - # run df cmd, and parse output for total free disk space - df_result = check_output_returning_strings(df_cmd) - logger.info(str(df_result)) - df_result_line = df_result.splitlines()[-1] - logger.info(str(df_result_line)) - df_result_line_parts = df_result_line.split(' ') - logger.info(str(df_result_line_parts)) - df_bytes = int(df_result_line_parts[3]) - input_dp_sizes = self.tmss_client.get_url_as_json_object(subtask['url'].rstrip('/')+'/input_dataproducts?fields=size') - total_size = sum(x['size'] for x in input_dp_sizes) - - if df_bytes > total_size: - logger.info("enough free disk space available for copy-subtask id=%s destination=%s df=%d needed=%d", subtask['id'], destination, df_bytes, total_size) - - # clear previously set "not enough free disk space available" error_reason if set - if subtask.get('error_reason'): - self.tmss_client.do_request_and_get_result_as_string('PATCH', subtask['url'], {'error_reason': None}) - - # run it - self.run_copy_subtask(subtask) - else: - msg = "not enough free disk space available to start copy-subtask id=%s df=%d needed=%d" % (subtask['id'], df_bytes, total_size) - logger.info(msg) - self.tmss_client.do_request_and_get_result_as_string('PATCH', subtask['url'], {'error_reason': msg}) + try: + # determine destination host and root_dir + destination = subtask['specifications_doc']['destination'] + dst_host = destination[:destination.find(':')] if ':' in destination else '' + dst_root_dir = '/'+destination.lstrip(dst_host + ':').split('/')[1] + # strip unneeded localhost to prevent unneeded ssh calls + dst_host = dst_host.lstrip('localhost:').lstrip('127.0.0.1:') + + df_cmd = ['df', dst_root_dir] + if dst_host: + df_cmd = wrap_command_in_ssh_call(df_cmd, dst_host) + + logger.info("checking free disk space for copy-subtask id=%s, executing: %s", subtask['id'], ' '.join(df_cmd)) + + # run df cmd, and parse output for total free disk space + df_result = check_output_returning_strings(df_cmd) + df_result_line = df_result.splitlines()[-1] + df_result_line_parts = df_result_line.split() + df_bytes = int(df_result_line_parts[3]) + input_dp_sizes = self.tmss_client.get_url_as_json_object(subtask['url'].rstrip('/')+'/input_dataproducts?fields=size') + total_size = sum(x['size'] for x in input_dp_sizes) + + if df_bytes > total_size: + logger.info("enough free disk space available for copy-subtask id=%s destination=%s df=%d needed=%d", subtask['id'], destination, df_bytes, total_size) + + # clear previously set "not enough free disk space available" error_reason if set + if subtask.get('error_reason'): + self.tmss_client.do_request_and_get_result_as_string('PATCH', subtask['url'], {'error_reason': None}) + + # run it + self.run_copy_subtask(subtask) + else: + msg = "not enough free disk space available to start copy-subtask id=%s df=%d needed=%d" % (subtask['id'], df_bytes, total_size) + logger.info(msg) + self.tmss_client.do_request_and_get_result_as_string('PATCH', subtask['url'], {'error_reason': msg}) + except Exception as e: + logger.exception(str(e)) + # try to run it anyway, maybe it fails on not enough disk space. + # if it fails while running, then it results in an error status, and the user can take appropriate action. + self.run_copy_subtask(subtask) def run_copy_subtask(self, subtask): if subtask['subtask_type'] != 'copy':