Improve script that checks workflow results

The existing script to compare workflow results bails out of first error.
This improved version tries to do as many checks as possible. It also
saves the values of differing solution tables in human-readable format
for later inspection.


Former-commit-id: 1160a283
Former-commit-id: 99b9ff16

test_jobs/check_workflow_results.py

@@ -22,6 +22,7 @@ def main(results_path, control_path):
     control_path : str
         Path to control results directory
     """
+    error = False
     # Check that all expected output files are present
     dcmp = dircmp(results_path, control_path)
     if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0:
@@ -31,7 +32,7 @@ def main(results_path, control_path):
         if len(dcmp.right_only) > 0:
             print('ERROR: The following files are present in the control but not in the '
                   'output: {}'.format(dcmp.right_only))
-        sys.exit(1)
+        error = True
 
     # Check that the calibration solutions match the control ones
     check_h5parm = os.path.join(results_path, 'cal_values', 'cal_solutions.h5')
@@ -44,10 +45,19 @@ def main(results_path, control_path):
             control_soltab = control_solset.getSoltab(check_soltab.name)
             if not np.allclose(check_soltab.val, control_soltab.val, rtol=1e-03,
                                atol=1e-03, equal_nan=True):
+                error = True
                 print('ERROR: Val array of soltab {} of solset {} does not match '
                       'the control'.format(check_soltab.name, check_solset.name))
-                sys.exit(1)
-
+                with open("check_soltab.{}.val".format(check_soltab.name), "w") as f:
+                    f.write(str(check_soltab.val))
+                with open("control_soltab.{}.val".format(control_soltab.name), "w") as f:
+                    f.write(str(control_soltab.val))
+            else:
+                print('INFO: Val array of soltab {} of solset {} matches '
+                      'the control'.format(check_soltab.name, check_solset.name))
+    control_h5.close()
+    check_h5.close()
+    sys.exit(error)
 
 if __name__ == '__main__':
     descriptiontext = "Checks the ouput of a workflow run.\n"