diff --git a/test_jobs/check_workflow_results.py b/test_jobs/check_workflow_results.py index 23ec9dfe3aeb80887087ea87e8ca8f0cf328dcfd..e9dab56d1e93edb5e841bf75145576af6311d456 100755 --- a/test_jobs/check_workflow_results.py +++ b/test_jobs/check_workflow_results.py @@ -9,28 +9,57 @@ import h5py import sys import numpy as np + +def check_all_files_present(dcmp, leftname='left', rightname='right'): + """ + Checks recursively that all files are present in both compared directories + + Parameters + ---------- + dcmp : dircmp object + The result of dircmp(left, right) + leftname : str, optional + Name of the left part of the input dcmp + rightname : str, optional + Name of the right part of the input dcmp + + Returns + ------- + agree : bool + True if all files are present, False if not + """ + agree = True + if dcmp.left_only or dcmp.right_only: + if dcmp.left_only: + print('ERROR: The following files are present in the {0} but not in the ' + '{1}: {2}'.format(leftname, rightname, dcmp.left_only)) + if dcmp.right_only: + print('ERROR: The following files are present in the {0} but not in the ' + '{1}: {2}'.format(rightname, leftname, dcmp.right_only)) + agree = False + for sub_dcmp in dcmp.subdirs.values(): + if not check_all_files_present(sub_dcmp, leftname=leftname, rightname=rightname): + agree = False + return agree + + def main(check, control): """ Checks the outputs of an end-to-end workflow job Parameters ---------- - check_path : str + check : str Path to output results directory of job - control_path : str + control : str Path to control results directory """ error = False - #Check that all expected output files are present + # Check that all expected output files are present dcmp = dircmp(check, control) - if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0: - if len(dcmp.left_only) > 0: - print('ERROR: The following files are present in the output but not in the ' - 'control: {}'.format(dcmp.left_only)) - if len(dcmp.right_only) > 0: - print('ERROR: The following files are present in the control but not in the ' - 'output: {}'.format(dcmp.right_only)) + all_present = check_all_files_present(dcmp, leftname='output', rightname='control') + if not all_present: error = True check_path = check + '/cal_values/cal_solutions.h5' @@ -40,17 +69,17 @@ def main(check, control): # Check that the calibration solutions match the control ones for matching frequencies for solsetname in check_h5: - soltabnames = [name for name in check_h5[solsetname] - if name not in ('source', 'antenna')] + soltabnames = [name for name in check_h5[solsetname] + if name not in ('source', 'antenna')] try: control_h5[solsetname] - except: + except KeyError: print("Error: solset {} not present in control.".format(solsetname)) sys.exit(1) for soltabname in soltabnames: try: control_h5[solsetname][soltabname] - except: + except KeyError: print("Error: soltab {} not present in control.".format(soltabname)) sys.exit(1) check_soltabval = check_h5[solsetname][soltabname]['val'] @@ -63,13 +92,13 @@ def main(check, control): control_soltabfreq = control_h5[solsetname][soltabname]['freq'][:] matches = np.isclose(control_soltabfreq[:, np.newaxis], check_soltabfreq) matching_freq_indices = np.where(matches)[0] - matching_vals = np.take(control_soltabval, - matching_freq_indices, + matching_vals = np.take(control_soltabval, + matching_freq_indices, axis=freq_axis_index) - if not np.allclose(check_soltabval, matching_vals, + if not np.allclose(check_soltabval, matching_vals, rtol=1e-03, atol=1e-03, equal_nan=True): error = True - print("Val array of soltab {} does not match the control".format(soltabname)) + print("ERROR: Val array of soltab {} does not match the control".format(soltabname)) with open("check_soltab.{}.val".format(soltabname), "w") as f: f.write(str(check_soltabval[:])) with open("control_soltab.{}.val".format(soltabname), "w") as f: @@ -80,6 +109,7 @@ def main(check, control): control_h5.close() sys.exit(error) + if __name__ == '__main__': descriptiontext = "Checks the output of a workflow run.\n"