From 166adf91d6b8a2a5235edcd8b5ec4e0956a2d327 Mon Sep 17 00:00:00 2001
From: Marcel Loose <loose@astron.nl>
Date: Tue, 28 Sep 2021 11:07:59 +0200
Subject: [PATCH] Improve script that checks workflow results

The existing script to compare workflow results bails out of first error.
This improved version tries to do as many checks as possible. It also
saves the values of differing solution tables in human-readable format
for later inspection.


Former-commit-id: 1160a2834b2eb37ccd06d559338954b6394d89f7
Former-commit-id: 99b9ff162a1378c0594ff94f7e1e61a411445cef
---
 test_jobs/check_workflow_results.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/test_jobs/check_workflow_results.py b/test_jobs/check_workflow_results.py
index 44c8327d..5b694a44 100755
--- a/test_jobs/check_workflow_results.py
+++ b/test_jobs/check_workflow_results.py
@@ -22,6 +22,7 @@ def main(results_path, control_path):
     control_path : str
         Path to control results directory
     """
+    error = False
     # Check that all expected output files are present
     dcmp = dircmp(results_path, control_path)
     if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0:
@@ -31,7 +32,7 @@ def main(results_path, control_path):
         if len(dcmp.right_only) > 0:
             print('ERROR: The following files are present in the control but not in the '
                   'output: {}'.format(dcmp.right_only))
-        sys.exit(1)
+        error = True
 
     # Check that the calibration solutions match the control ones
     check_h5parm = os.path.join(results_path, 'cal_values', 'cal_solutions.h5')
@@ -44,10 +45,19 @@ def main(results_path, control_path):
             control_soltab = control_solset.getSoltab(check_soltab.name)
             if not np.allclose(check_soltab.val, control_soltab.val, rtol=1e-03,
                                atol=1e-03, equal_nan=True):
+                error = True
                 print('ERROR: Val array of soltab {} of solset {} does not match '
                       'the control'.format(check_soltab.name, check_solset.name))
-                sys.exit(1)
-
+                with open("check_soltab.{}.val".format(check_soltab.name), "w") as f:
+                    f.write(str(check_soltab.val))
+                with open("control_soltab.{}.val".format(control_soltab.name), "w") as f:
+                    f.write(str(control_soltab.val))
+            else:
+                print('INFO: Val array of soltab {} of solset {} matches '
+                      'the control'.format(check_soltab.name, check_solset.name))
+    control_h5.close()
+    check_h5.close()
+    sys.exit(error)
 
 if __name__ == '__main__':
     descriptiontext = "Checks the ouput of a workflow run.\n"
-- 
GitLab