diff --git a/steps/preprocess.cwl b/steps/preprocess.cwl index dce2b6df7b184885e642359042eb5b10fde32c94..53b4b489c8a34f90f08b3e15d3ecfebfb840687e 100644 --- a/steps/preprocess.cwl +++ b/steps/preprocess.cwl @@ -90,4 +90,3 @@ outputs: requirements: InlineJavascriptRequirement: {} - \ No newline at end of file diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f78f067a6c969d8c9500fe1c8ef5288b1903566a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,27 @@ +# Testing the Pre-Processing Pipeline + +The `/tests` directory contains a number of sample inputs that can be used to test and verify the pipeline. + +## Sample MeasurementSets + +The sample inputs, defined in the JSON format, reference a number of relatively small raw LOFAR MeasurementSets (MSs), 3 GB each: +``` +L888536_SAP000_SB026_uv.MS +L888536_SAP000_SB027_uv.MS +L888536_SAP000_SB028_uv.MS +``` +These MSs are stored on the [DAS-6](https://www.cs.vu.nl/das6/) cluster in the following directory: +``` +/var/scratch/veldhuis/data/raw +``` +This folder also contains a tarball with all three MSs: `raw_lofar_data.tar`. If you don't have access to DAS-6 and would like to test the pipeline, but do not have access to raw LOFAR data, please contact Team Rapthor. + +## Running the tests + +To run these tests you need to have a CWL runner, such as `cwltool` or `toil`, installed on your system. + +For example, with `cwltool` you could run the top-level `pipeline.cwl` workflow as follows: +``` +cwltool --debug --preserve-entire-environment --outdir=pipeline-out $PREPROCESS_ROOT/workflows/pipeline.cwl $PREPROCESS_ROOT/tests/pipeline_input.json +``` +where `PREPROCESS_ROOT` refers to the location of the repository on your system. Please modify the command as desired, for an overview of possible arguments run `cwltool --help` or refer to their [Read the Docs pages](https://cwltool.readthedocs.io/). \ No newline at end of file diff --git a/tests/pipeline_input.json b/tests/pipeline_input.json new file mode 100644 index 0000000000000000000000000000000000000000..96a8b3741833906c93d737f497593b73f76e738c --- /dev/null +++ b/tests/pipeline_input.json @@ -0,0 +1,21 @@ +{ + "msin": [ + { + "class": "Directory", + "path": "/var/scratch/veldhuis/data/raw/L888536_SAP000_SB026_uv.MS" + }, + { + "class": "Directory", + "path": "/var/scratch/veldhuis/data/raw/L888536_SAP000_SB027_uv.MS" + }, + { + "class": "Directory", + "path": "/var/scratch/veldhuis/data/raw/L888536_SAP000_SB028_uv.MS" + } + ], + "demix_sources": [], + "demix_timestep": 10, + "demix_freqstep": 64, + "avg_timestep": 1, + "avg_freqstep": 4 +} diff --git a/tests/preprocess_input.json b/tests/preprocess_input.json index b7b85db3a8a85e32321146ab9340a1ba22f76bd8..19d5dfeb37e6b375dfd1fe606477c101d0f9d83d 100644 --- a/tests/preprocess_input.json +++ b/tests/preprocess_input.json @@ -1,7 +1,7 @@ { "msin": { "class": "Directory", - "path": "L2014581_SAP001_SB400_uv.MS" + "path": "/var/scratch/veldhuis/data/raw/L888536_SAP000_SB028_uv.MS" }, "demix_sources": [], "demix_timestep": 10, diff --git a/workflows/pipeline.cwl b/workflows/pipeline.cwl index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7d5f952fbea42f853b1d575449d0604384f3572c 100644 --- a/workflows/pipeline.cwl +++ b/workflows/pipeline.cwl @@ -0,0 +1,86 @@ +cwlVersion: v1.2 +class: Workflow +label: LOFAR Pre-Processing Pipeline +doc: | + This is the top-level workflow for the LOFAR Pre-Processing Pipeline. + The pipeline performs flagging, demixing, averaging, and Dysco + compression of raw LOFAR data. + +inputs: + - id: msin + type: Directory[] + doc: List of raw LOFAR MeasurementSets (MSs) + - id: rfistrategy + type: File? + default: + class: File + path: ../rfistrategies/lofar-default.lua + doc: AOFlagger RFI flagging strategy file + - id: demix_skymodel + type: File? + default: + class: File + path: ../skymodels/Ateam.skymodel + doc: Skymodel to use for calibration + - id: demix_sources + type: string[]? + default: [] + doc: Sources to subtract during demixing + - id: demix_timestep + type: int + doc: Number of time steps to average during demixing + - id: demix_freqstep + type: int + doc: Number of channels to average during demixing + - id: avg_timestep + type: int + doc: Number of time steps to average + - id: avg_freqstep + type: int + doc: Number of channels to average + +outputs: + - id: msout + type: Directory[] + outputSource: + - preprocess/msout + doc: List of pre-processed LOFAR MSs + +steps: + - id: preprocess + label: Perform pre-processing operations + doc: | + Perform pre-processing on a raw LOFAR MS using DP3 + scatter: + - msin + - msout_name + scatterMethod: dotproduct + run: ../steps/preprocess.cwl + in: + - id: msin + source: msin + - id: rfistrategy + source: rfistrategy + - id: demix_skymodel + source: demix_skymodel + - id: demix_sources + source: demix_sources + - id: demix_timestep + source: demix_timestep + - id: demix_freqstep + source: demix_freqstep + - id: avg_timestep + source: avg_timestep + - id: avg_freqstep + source: avg_freqstep + - id: msout_name + source: msin + # TODO: determine proper output scheme for preprocessed MSs. + # Maybe append to file name or in new directory? + valueFrom: $(self.basename + ".prepout") + out: [msout] + +requirements: + - class: InlineJavascriptRequirement + - class: ScatterFeatureRequirement + - class: StepInputExpressionRequirement