Merge branch 'scatter_and_aggregate' into 'master'

Scatter and aggregate

See merge request !18

aggregate_and_plot.cwl

@@ -15,4 +15,20 @@ inputs:
     inputBinding:
       position: 1  # The first argument after the script will be the workdir
 
-outputs: []
+outputs:
+- id: plots
+  type: File[]
+  outputBinding:
+    glob: "*.png"
+- id: outputs
+  type: File
+  outputBinding:
+    glob: "outputs.txt"
+- id: quality
+  type: File
+  outputBinding:
+    glob: "outputs.json"
+- id: inspect
+  type: File
+  outputBinding:
+    glob: "inspect.h5"

aggregate_and_plot.py

 #!/usr/bin/env python3
 import argparse
+import subprocess
+import json
 from lofar_quality.manipulate.common import join_inspect_ds
 from lofar_quality.inspect.inspect_dataset import plot_inspect_ds
+from lofar_quality.inspect.v2 import extract_quality_metrics
+from lofar_quality.inspect.common import store_metrics
 from pathlib import Path
 import logging
 
@@ -31,15 +35,19 @@ def main():
                 logging.info(f"Could not find the following input file: {line}")
         logging.info(f"Found {len(h5_paths)} h5 files")
 
-    h5_output_path = workdir / "inspect.h5"
+    h5_output_path = "inspect.h5"
     join_inspect_ds(h5_paths, h5_output_path)
     plot_paths = plot_inspect_ds(h5_output_path, workdir, min_elevation=15)
 
-    with open(workdir / "outputs.txt", "w") as file:
-        file.write(str(h5_output_path.absolute()) + "\n")
+    quality = {"quality": extract_quality_metrics(h5_output_path)}
+    store_metrics("outputs.json", quality)
+
+    with open("outputs.txt", "w") as file:
+        file.write(str(h5_output_path) + "\n")
         for file_path in plot_paths:
-            path = Path(file_path).absolute()
+            path = Path(file_path)
             file.write(str(path) + "\n")
 
 if __name__ == "__main__":
     main()
+

aggregate_and_plot.sh

@@ -14,4 +14,4 @@ else
 fi
 export SINGULARITY_BIND="/project:/project"
 export TOIL_SLURM_ARGS=""
-toil-cwl-runner "${SCRIPT_DIR}/imaging_compress_pipeline.git/aggregate_and_plot.cwl" --workdir $SCRIPT_DIR --logDebug --singularity --batchSystem slurm --batchLogsDir ./logs --jobStore ./job_store
+toil-cwl-runner "${SCRIPT_DIR}/imaging_compress_pipeline.git/aggregate_and_plot.cwl" --workdir $SCRIPT_DIR --outdir $SCRIPT_DIR --logDebug --singularity --batchSystem slurm --batchLogsDir ./logs --jobStore ./job_store --clean always