Merge branch 'improve_ci_checks' into 'master'

Improve CI checks

See merge request eosc/prefactor3-cwl!51

Former-commit-id: 6e4702f6

test_jobs/check_workflow_results.py

@@ -9,28 +9,57 @@ import h5py
 import sys
 import numpy as np
 
+
+def check_all_files_present(dcmp, leftname='left', rightname='right'):
+    """
+    Checks recursively that all files are present in both compared directories
+
+    Parameters
+    ----------
+    dcmp : dircmp object
+        The result of dircmp(left, right)
+    leftname : str, optional
+        Name of the left part of the input dcmp
+    rightname : str, optional
+        Name of the right part of the input dcmp
+
+    Returns
+    -------
+    agree : bool
+        True if all files are present, False if not
+    """
+    agree = True
+    if dcmp.left_only or dcmp.right_only:
+        if dcmp.left_only:
+            print('ERROR: The following files are present in the {0} but not in the '
+                  '{1}: {2}'.format(leftname, rightname, dcmp.left_only))
+        if dcmp.right_only:
+            print('ERROR: The following files are present in the {0} but not in the '
+                  '{1}: {2}'.format(rightname, leftname, dcmp.right_only))
+        agree = False
+    for sub_dcmp in dcmp.subdirs.values():
+        if not check_all_files_present(sub_dcmp, leftname=leftname, rightname=rightname):
+            agree = False
+    return agree
+
+
 def main(check, control):
     """
     Checks the outputs of an end-to-end workflow job
 
     Parameters
     ----------
-    check_path : str
+    check : str
         Path to output results directory of job
-    control_path : str
+    control : str
         Path to control results directory
     """
     error = False
 
-    #Check that all expected output files are present
+    # Check that all expected output files are present
     dcmp = dircmp(check, control)
-    if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0:
-        if len(dcmp.left_only) > 0:
-            print('ERROR: The following files are present in the output but not in the '
-                  'control: {}'.format(dcmp.left_only))
-        if len(dcmp.right_only) > 0:
-            print('ERROR: The following files are present in the control but not in the '
-                  'output: {}'.format(dcmp.right_only))
+    all_present = check_all_files_present(dcmp, leftname='output', rightname='control')
+    if not all_present:
         error = True
 
     check_path = check + '/cal_values/cal_solutions.h5'
@@ -40,17 +69,17 @@ def main(check, control):
 
     # Check that the calibration solutions match the control ones for matching frequencies
     for solsetname in check_h5:
-        soltabnames = [name for name in check_h5[solsetname] 
-                            if name not in ('source', 'antenna')]
+        soltabnames = [name for name in check_h5[solsetname]
+                       if name not in ('source', 'antenna')]
         try:
             control_h5[solsetname]
-        except:
+        except KeyError:
             print("Error: solset {} not present in control.".format(solsetname))
             sys.exit(1)
         for soltabname in soltabnames:
             try:
                 control_h5[solsetname][soltabname]
-            except:
+            except KeyError:
                 print("Error: soltab {} not present in control.".format(soltabname))
                 sys.exit(1)
             check_soltabval = check_h5[solsetname][soltabname]['val']
@@ -63,13 +92,13 @@ def main(check, control):
                 control_soltabfreq = control_h5[solsetname][soltabname]['freq'][:]
                 matches = np.isclose(control_soltabfreq[:, np.newaxis], check_soltabfreq)
                 matching_freq_indices = np.where(matches)[0]
-                matching_vals = np.take(control_soltabval, 
-                                        matching_freq_indices, 
+                matching_vals = np.take(control_soltabval,
+                                        matching_freq_indices,
                                         axis=freq_axis_index)
-            if not np.allclose(check_soltabval, matching_vals, 
+            if not np.allclose(check_soltabval, matching_vals,
                                rtol=1e-03, atol=1e-03, equal_nan=True):
                 error = True
-                print("Val array of soltab {} does not match the control".format(soltabname))
+                print("ERROR: Val array of soltab {} does not match the control".format(soltabname))
                 with open("check_soltab.{}.val".format(soltabname), "w") as f:
                     f.write(str(check_soltabval[:]))
                 with open("control_soltab.{}.val".format(soltabname), "w") as f:
@@ -80,6 +109,7 @@ def main(check, control):
     control_h5.close()
     sys.exit(error)
 
+
 if __name__ == '__main__':
     descriptiontext = "Checks the output of a workflow run.\n"