diff --git a/aggregate_and_plot.cwl b/aggregate_and_plot.cwl index 34b3f89bcca576989ef79a46df735c18efd137f4..f55e0552068db2fa71d62be06d544a194364e69f 100644 --- a/aggregate_and_plot.cwl +++ b/aggregate_and_plot.cwl @@ -3,12 +3,16 @@ class: CommandLineTool requirements: - class: DockerRequirement - dockerPull: git.astron.nl:5000/ldv/imaging_compress_pipeline:v0.4.1 + dockerPull: git.astron.nl:5000/ldv/imaging_compress_pipeline:v0.5.1 baseCommand: python3 arguments: - - /src/aggregate_and_plot.py + - $(inputs.workdir)/imaging_compress_pipeline.git/aggregate_and_plot.py -inputs: {} +inputs: + workdir: + type: string + inputBinding: + position: 1 # The first argument after the script will be the workdir outputs: [] diff --git a/aggregate_and_plot.py b/aggregate_and_plot.py index 1c59cd57c607aa14e83c41dc47119896fa9a1fd2..8eb79cb6573088f2a51b8991471d8c6474669f55 100644 --- a/aggregate_and_plot.py +++ b/aggregate_and_plot.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import argparse from lofar_quality.manipulate.common import join_inspect_ds from lofar_quality.inspect.inspect_dataset import plot_inspect_ds from pathlib import Path @@ -7,28 +8,38 @@ import logging FORMAT = '%(levelname)-15s %(asctime)s: %(message)s' logging.basicConfig(format=FORMAT, level="INFO") -if __name__ == "__main__": +def main(): + # Set up argument parser + parser = argparse.ArgumentParser(description="Process H5 files and output results to a specified directory.") + parser.add_argument('workdir', type=str, help='Path to the working directory') + + # Parse the arguments + args = parser.parse_args() + workdir = Path(args.workdir) # The first command-line argument + h5_paths = [] - with open("/mnt/workdir/inputs.txt", "r") as file: + with open(workdir / "inputs.txt", "r") as file: for line in file: - line = str(line.strip()) # remove white space + line = str(line.strip()) # Remove white space path = Path(line) if path.exists(): if path.suffix.lower() == ".h5": h5_paths.append(line) else: logging.info(f"File exists but is not an h5 file: {line}") - else: logging.info(f"Could not find the following input file: {line}") logging.info(f"Found {len(h5_paths)} h5 files") - h5_output_path = "/mnt/workdir/inspect.h5" + h5_output_path = workdir / "inspect.h5" join_inspect_ds(h5_paths, h5_output_path) - plot_paths = plot_inspect_ds(h5_output_path, "/mnt/workdir", min_elevation=15) + plot_paths = plot_inspect_ds(h5_output_path, workdir, min_elevation=15) - with open("/mnt/workdir/outputs.txt", "w") as file: - file.write(str(Path(h5_output_path).absolute()) + "\n") + with open(workdir / "outputs.txt", "w") as file: + file.write(str(h5_output_path.absolute()) + "\n") for file_path in plot_paths: path = Path(file_path).absolute() file.write(str(path) + "\n") + +if __name__ == "__main__": + main() diff --git a/aggregate_and_plot.sh b/aggregate_and_plot.sh index b1c74f864e4f7e27eb4618a8cd55ecf41a37b978..56a361679c91a5714e96bc7ef343df5126563105 100644 --- a/aggregate_and_plot.sh +++ b/aggregate_and_plot.sh @@ -1,13 +1,17 @@ #!/usr/bin/sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR +TOIL_VENV_DIR="${SCRIPT_DIR}/toil_venv" if [ -d "toil_venv" ]; then echo "Activating existing virtual environment." - source toil_venv/bin/activate + source "${TOIL_VENV_DIR}/bin/activate" else echo "Creating new virtual environment for using toil" - python3.9 -m venv toil_venv - source toil_venv/bin/activate + python3.9 -m venv $TOIL_VENV_DIR + source "${TOIL_VENV_DIR}/bin/activate" pip install --upgrade pip setuptools wheel pip install toil[cwl] singularity fi -export SINGULARITY_BIND="$PWD:/mnt/workdir,/project:/project" -toil-cwl-runner --singularity imaging_compress_pipeline.git/aggregate_and_plot.cwl +export SINGULARITY_BIND="/project:/project" +export TOIL_SLURM_ARGS="--exclusive" +toil-cwl-runner "${SCRIPT_DIR}/imaging_compress_pipeline.git/aggregate_and_plot.cwl" --workdir $SCRIPT_DIR --logDebug --singularity --batchSystem slurm --batchLogsDir ./logs --jobStore ./job_store