Merge branch 'scatter_and_aggregate' into 'master'

Make dir references absolute

See merge request !16

aggregate_and_plot.cwl

@@ -3,12 +3,16 @@ class: CommandLineTool
 
 requirements:
   - class: DockerRequirement
-    dockerPull: git.astron.nl:5000/ldv/imaging_compress_pipeline:v0.4.1
+    dockerPull: git.astron.nl:5000/ldv/imaging_compress_pipeline:v0.5.1
 
 baseCommand: python3
 arguments:
-  - /src/aggregate_and_plot.py
+  - $(inputs.workdir)/imaging_compress_pipeline.git/aggregate_and_plot.py
 
-inputs: {}
+inputs:
+  workdir:
+    type: string
+    inputBinding:
+      position: 1  # The first argument after the script will be the workdir
 
 outputs: []

aggregate_and_plot.py

 #!/usr/bin/env python3
+import argparse
 from lofar_quality.manipulate.common import join_inspect_ds
 from lofar_quality.inspect.inspect_dataset import plot_inspect_ds
 from pathlib import Path
@@ -7,28 +8,38 @@ import logging
 FORMAT = '%(levelname)-15s %(asctime)s: %(message)s'
 logging.basicConfig(format=FORMAT, level="INFO")
 
-if __name__ == "__main__":
+def main():
+    # Set up argument parser
+    parser = argparse.ArgumentParser(description="Process H5 files and output results to a specified directory.")
+    parser.add_argument('workdir', type=str, help='Path to the working directory')
+
+    # Parse the arguments
+    args = parser.parse_args()
+    workdir = Path(args.workdir)  # The first command-line argument
+
     h5_paths = []
-    with open("/mnt/workdir/inputs.txt", "r") as file:
+    with open(workdir / "inputs.txt", "r") as file:
         for line in file:
-            line = str(line.strip()) # remove white space
+            line = str(line.strip())  # Remove white space
             path = Path(line)
             if path.exists():
                 if path.suffix.lower() == ".h5":
                     h5_paths.append(line)
                 else:
                     logging.info(f"File exists but is not an h5 file: {line}")
-
             else:
                 logging.info(f"Could not find the following input file: {line}")
         logging.info(f"Found {len(h5_paths)} h5 files")
 
-    h5_output_path = "/mnt/workdir/inspect.h5"
+    h5_output_path = workdir / "inspect.h5"
     join_inspect_ds(h5_paths, h5_output_path)
-    plot_paths = plot_inspect_ds(h5_output_path, "/mnt/workdir", min_elevation=15)
+    plot_paths = plot_inspect_ds(h5_output_path, workdir, min_elevation=15)
 
-    with open("/mnt/workdir/outputs.txt", "w") as file:
-        file.write(str(Path(h5_output_path).absolute()) + "\n")
+    with open(workdir / "outputs.txt", "w") as file:
+        file.write(str(h5_output_path.absolute()) + "\n")
         for file_path in plot_paths:
             path = Path(file_path).absolute()
             file.write(str(path) + "\n")
+
+if __name__ == "__main__":
+    main()

aggregate_and_plot.sh

 #!/usr/bin/sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR
+TOIL_VENV_DIR="${SCRIPT_DIR}/toil_venv"
 if [ -d "toil_venv" ]; then
     echo "Activating existing virtual environment."
-    source toil_venv/bin/activate
+    source "${TOIL_VENV_DIR}/bin/activate"
 else
     echo "Creating new virtual environment for using toil"
-    python3.9 -m venv toil_venv
-    source toil_venv/bin/activate
+    python3.9 -m venv $TOIL_VENV_DIR
+    source "${TOIL_VENV_DIR}/bin/activate"
     pip install --upgrade pip setuptools wheel
     pip install toil[cwl] singularity
 fi
-export SINGULARITY_BIND="$PWD:/mnt/workdir,/project:/project"
-toil-cwl-runner --singularity imaging_compress_pipeline.git/aggregate_and_plot.cwl
+export SINGULARITY_BIND="/project:/project"
+export TOIL_SLURM_ARGS="--exclusive"
+toil-cwl-runner "${SCRIPT_DIR}/imaging_compress_pipeline.git/aggregate_and_plot.cwl" --workdir $SCRIPT_DIR --logDebug --singularity --batchSystem slurm --batchLogsDir ./logs --jobStore ./job_store