diff --git a/test_jobs/check_workflow_results.py b/test_jobs/check_workflow_results.py
index 74a584021b8dfd5c91e5cb35eb284bf84b7592f7..13d508b451794c21f5af3d81c66399c2dc436af2 100755
--- a/test_jobs/check_workflow_results.py
+++ b/test_jobs/check_workflow_results.py
@@ -20,6 +20,7 @@ def main(check, control):
     control_path : str
         Path to control results directory
     """
+    error = False
 
     #Check that all expected output files are present
     dcmp = dircmp(check, control)
@@ -30,6 +31,7 @@ def main(check, control):
         if len(dcmp.right_only) > 0:
             print('ERROR: The following files are present in the control but not in the '
                   'output: {}'.format(dcmp.right_only))
+        error = True
 
     check_path = check + '/cal_values/cal_solutions.h5'
     control_path = control + '/cal_values/cal_solutions.h5'
@@ -65,6 +67,7 @@ def main(check, control):
                                         axis=freq_axis_index)
             if not np.allclose(check_soltabval, matching_vals, 
                                rtol=1e-03, atol=1e-03, equal_nan=True):
+                error = True
                 print("Val array of soltab {} does not match the control".format(soltabname))
                 with open("check_soltab.{}.val".format(soltabname), "w") as f:
                     f.write(str(check_soltabval[:]))
@@ -74,7 +77,7 @@ def main(check, control):
                 print("Val array of soltab {} matches the control".format(soltabname))
     check_h5.close()
     control_h5.close()
-    sys.exit(0)
+    sys.exit(error)
 
 if __name__ == '__main__':
     descriptiontext = "Checks the output of a workflow run.\n"