Merge branch 'facet_subtract' into 'master'

Facet subtract

See merge request !69

scripts/gather_model_images.sh

+#!/bin/bash
+
+# Ensure a model image folder is provided
+if [ -z "$1" ]; then
+    echo "Usage: $0 <model_image_folder>"
+    exit 1
+fi
+
+MODEL_IMAGE_FOLDER=$1
+
+# Patterns to check with model images
+PATTERNS=(
+    "*-model-fpb.fits"
+    "*-model-pb.fits"
+    "*-model.fits"
+)
+
+# Function to copy files if more than one match exists
+copy_files() {
+    local pattern="$1"
+    local files=()
+
+    # Filter files matching the pattern but excluding those with "MFS"
+    for file in "$MODEL_IMAGE_FOLDER"/$pattern; do
+        [[ "$file" == *"MFS"* ]] && continue  # Skip files with "MFS"
+        files+=("$file")
+    done
+
+    mkdir -p output_images
+
+    if [ "${#files[@]}" -gt 1 ]; then
+        cp "${files[@]}" output_images/
+        return 0
+    fi
+    return 1  # No valid files were copied
+}
+
+# Iterate through patterns and copy the first match
+for pattern in "${PATTERNS[@]}"; do
+    copy_files "$pattern" && exit 0
+done
+
+# If no files were copied, exit with error
+echo "ERROR: missing model images in folder $MODEL_IMAGE_FOLDER"
+exit 1

steps/dp3_parset.cwl

@@ -2,8 +2,7 @@ cwlVersion: v1.2
 class: CommandLineTool
 id: dp3_parset
 label: DP3 with parset
-doc: |
-    Run DP3 with a parset
+doc: Run DP3 with a parset
 
 baseCommand: DP3
 
@@ -22,7 +21,7 @@ outputs:
     type: Directory
     doc: Output measurement set
     outputBinding:
-      glob: "*.ms"
+      glob: "*.concat.ms"
   - id: logfile
     type: File[]
     outputBinding:

steps/gather_model_images.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: gatherdis2
+doc: Gather the correct WSClean model images from model_image_folder
+
+baseCommand:
+  - gather_model_images.sh
+
+inputs:
+  - id: model_image_folder
+    type: Directory
+    doc: Directory with model images
+    inputBinding:
+      position: 1
+
+outputs:
+  - id: filtered_model_image_folder
+    type: Directory
+    doc: Directory with filtered WSClean model images
+    outputBinding:
+      glob: output_images
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl

steps/get_facet_layout.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: get_facet_layout
+label: DS9 Facet Generator
+doc: Generates DS9 facet layout for direction-dependent facet imaging.
+
+baseCommand: python3
+
+inputs:
+  - id: msin
+    type: Directory
+    doc: MeasurementSet
+    inputBinding:
+      prefix: "--ms"
+      position: 2
+
+  - id: h5parm
+    type: File
+    doc: Multi-directional HDF5 file.
+    inputBinding:
+      prefix: "--h5"
+      position: 3
+
+  - id: imsize
+    type: int
+    doc: Image size in pixels (larger than image fov to help wsclean 1.2" imaging at boundaries).
+    default: 25000
+    inputBinding:
+      prefix: "--imsize"
+      position: 4
+
+  - id: pixelscale
+    type: float
+    doc: Pixel scale in arcseconds per pixel.
+    default: 0.4
+    inputBinding:
+      prefix: "--pixelscale"
+      position: 5
+
+  - id: regionfile
+    type: string
+    doc: Name of the output DS9 region file.
+    default: "facets.reg"
+    inputBinding:
+      prefix: "--DS9regionout"
+      position: 6
+
+  - id: facetselfcal
+    type: Directory
+    doc: facetselfcal directory.
+
+outputs:
+  - id: facet_regions
+    type: File
+    doc: The output DS9 region file.
+    outputBinding:
+      glob: $(inputs.regionfile)
+  - id: logfile
+    type: File[]
+    doc: log files from get_facet_layout
+    outputBinding:
+      glob: get_facet_layout*.log
+
+arguments:
+  - $( inputs.facetselfcal.path + '/submods/ds9facetgenerator.py' )
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.h5parm)
+      - entry: $(inputs.msin)
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl
+  - class: ResourceRequirement
+    coresMin: 1
+
+stdout: get_facet_layout.log
+stderr: get_facet_layout_err.log

steps/make_concat_parsets.cwl

@@ -16,6 +16,14 @@ inputs:
         position: 1
         separate: true
     doc: Input data in MeasurementSet format.
+  - id: dysco_bitrate
+    type: int?
+    doc: Number of bits per float used for columns containing visibilities.
+    default: 8
+    inputBinding:
+        prefix: "--bitrate"
+        position: 2
+        separate: true
   - id: lofar_helpers
     type: Directory
     doc: Path to lofar_helpers directory.

steps/predict_facet.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: predict_facet
+label: Predict with WSClean
+doc: Uses WSClean to predict sources within a facet and adds the predicted visibilities to the input data.
+
+
+baseCommand: python3
+
+inputs:
+  - id: subtracted_ms
+    type: Directory
+    doc: Input data in MeasurementSet format.
+    inputBinding:
+      prefix: "--mslist"
+      position: 1
+
+  - id: model_image_folder
+    type: Directory
+    doc: Folder containing 1.2" model images.
+    inputBinding:
+      prefix: "--model_image_folder"
+      position: 2
+
+  - id: polygon_region
+    type: File
+    doc: DS9 region file with facets for prediction.
+    inputBinding:
+      prefix: "--region"
+      position: 3
+
+  - id: h5parm
+    type: File
+    doc: HDF5 file containing the solutions for prediction.
+    inputBinding:
+      prefix: "--h5parm_predict"
+      position: 4
+
+  - id: lofar_helpers
+    type: Directory
+    doc: LOFAR helpers directory.
+
+  - id: polygon_info
+    type: File
+    doc: CSV file with polygon information (RA/DEC of calibrator and facet centers and averaging factor)
+
+  - id: copy_to_local_scratch
+    type: boolean?
+    default: false
+    inputBinding:
+      prefix: "--copy_to_local_scratch"
+      position: 5
+      separate: false
+    doc: Specific option for using --bypass-file-store on the Spider cluster to run predict and subtract on local scratch.
+
+  - id: ncpu
+    type: int?
+    doc: Number of cores to use during the subtract.
+    default: 16
+
+outputs:
+  - id: logfile
+    type: File[]
+    doc: Log files from current step.
+    outputBinding:
+      glob: predict_facet*.log
+  - id: facet_ms
+    type: Directory
+    doc: MeasurementSet after predicting back facet model visibilities
+    outputBinding:
+      glob: facet*.ms
+
+
+arguments:
+  - $( inputs.lofar_helpers.path + '/subtract/subtract_with_wsclean.py' )
+  - --applybeam
+  - --applycal
+  - --forwidefield
+  - --inverse
+  - --speedup_facet_subtract
+  - --cleanup_input_ms
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing: >
+      ${
+        // Set 'writable' on the "subtracted_ms" entry only if copy_to_local_scratch is true.
+        let stagedListing = [
+          { entry: inputs.subtracted_ms },
+          { entry: inputs.model_image_folder },
+          { entry: inputs.polygon_region },
+          { entry: inputs.h5parm },
+          { entry: inputs.polygon_info }
+        ];
+        if (!inputs.copy_to_local_scratch) {
+          stagedListing[0].writable = true;
+        }
+        return stagedListing;
+      }
+
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl
+  - class: ResourceRequirement
+    coresMin: $(inputs.ncpu)
+
+stdout: predict_facet.log
+stderr: predict_facet_err.log
\ No newline at end of file

steps/remove_flagged_stations.cwl

+cwlVersion: v1.2
+class: CommandLineTool
+id: remove_flagged_stations
+label: Remove fully flagged stations
+doc: Removes from the MeasurementSet all stations for which all data are flagged.
+
+baseCommand:
+  - python3
+
+inputs:
+    - id: ms
+      type: Directory
+      doc: MeasurementSet
+      inputBinding:
+        position: 3
+    - id: lofar_helpers
+      type: Directory
+      doc: LOFAR helpers directory.
+
+outputs:
+    - id: cleaned_ms
+      type: Directory
+      doc: MeasurementSet where fully flagged stations are removed.
+      outputBinding:
+        glob: $( 'flagged_' + inputs.ms.basename )
+    - id: logfile
+      type: File[]
+      doc: Log files from current step.
+      outputBinding:
+        glob: remove_flagged_stations*.log
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.ms)
+        writable: false
+
+arguments:
+  - $( inputs.lofar_helpers.path + '/ms_helpers/remove_flagged_stations.py' )
+  - --msout
+  - $( 'flagged_' + inputs.ms.basename )
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl
+  - class: ResourceRequirement
+    coresMin: 2
+
+stdout: remove_flagged_stations.log
+stderr: remove_flagged_stations_err.log
\ No newline at end of file

steps/split_polygons.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: split_polygons
+label: Split Polygon Facets
+doc: This step splits a multi-facet region file into individual facet region files.
+
+baseCommand: python3
+
+inputs:
+  - id: h5parm
+    type: File
+    doc: Multi-directional HDF5 file.
+    inputBinding:
+      prefix: "--h5"
+      position: 1
+
+  - id: facet_regions
+    type: File
+    doc: The DS9 region file that defines the facets.
+    inputBinding:
+      prefix: "--reg"
+      position: 2
+
+  - id: lofar_helpers
+    type: Directory
+    doc: The lofar_helpers directory.
+
+outputs:
+  - id: polygon_regions
+    type: File[]
+    doc: The facet regions.
+    outputBinding:
+      glob: "poly*.reg"
+  - id: polygon_info
+    type: File
+    doc: Polygon CSV file.
+    outputBinding:
+      glob: "*.csv"
+  - id: logfile
+    type: File[]
+    doc: Log files from current step.
+    outputBinding:
+      glob: split_polygons*.log
+
+arguments:
+  - $(inputs.lofar_helpers.path)/ds9_helpers/split_polygon_facets.py
+  - --extra_boundary=0.0
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.h5parm)
+      - entry: $(inputs.facet_regions)
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl
+  - class: ResourceRequirement
+    coresMin: 1
+
+stdout: split_polygons.log
+stderr: split_polygons_err.log

steps/subtract_fov_wsclean.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: subtract_with_wsclean
+label: Subtract with WSClean
+doc: This step uses WSClean to subtract visibilities corresponding to model images.
+
+baseCommand: python3
+
+inputs:
+  - id: msin
+    type: Directory
+    doc: MeasurementSet for source subtraction.
+    inputBinding:
+      prefix: "--mslist"
+      position: 1
+
+  - id: model_image_folder
+    type: Directory
+    doc: Directory containing 1.2" (or optionally other resolution) model images.
+    inputBinding:
+      prefix: "--model_image_folder"
+      position: 2
+
+  - id: facet_regions
+    type: File
+    doc: The DS9 region file that defines the facets for prediction.
+    inputBinding:
+      prefix: "--facets_predict"
+      position: 3
+
+  - id: h5parm
+    type: File
+    doc: The HDF5 solution file containing the solutions for prediction.
+    inputBinding:
+      prefix: "--h5parm_predict"
+      position: 4
+
+  - id: lofar_helpers
+    type: Directory
+    doc: LOFAR helpers directory.
+
+  - id: copy_to_local_scratch
+    type: boolean?
+    default: false
+    inputBinding:
+      prefix: "--copy_to_local_scratch"
+      position: 5
+      separate: false
+    doc: Specific option for using --bypass-file-store on the Spider cluster to run predict and subtract on local scratch.
+
+  - id: ncpu
+    type: int?
+    doc: Number of cores to use during the subtract.
+    default: 16
+
+outputs:
+  - id: logfile
+    type: File[]
+    doc: Log files from current step.
+    outputBinding:
+      glob: subtract_fov*.log
+  - id: subtracted_ms
+    type: Directory
+    doc: MS subtracted data
+    outputBinding:
+      glob: subfov_*.ms
+
+
+arguments:
+  - $( inputs.lofar_helpers.path + '/subtract/subtract_with_wsclean.py' )
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing: >
+      ${
+        // Set 'writable' on the "msin" entry only if copy_to_local_scratch is true.
+        let stagedListing = [
+          { entry: inputs.msin },
+          { entry: inputs.model_image_folder },
+          { entry: inputs.facet_regions },
+          { entry: inputs.h5parm }
+        ];
+        if (!inputs.copy_to_local_scratch) {
+          stagedListing[0].writable = true;
+        }
+        return stagedListing;
+      }
+
+hints:
+  - class: DockerRequirement
+    dockerPull: vlbi-cwl
+  - class: ResourceRequirement
+    coresMin: $(inputs.ncpu)
+
+stdout: subtract_fov.log
+stderr: subtract_fov_err.log
\ No newline at end of file

workflows/facet_subtract.cwl

+class: Workflow
+cwlVersion: v1.2
+id: facet_subtract
+label: Facet subtraction
+doc: Use WSClean to predict and subtract model data, to split all facets into separate MeasurementSets.
+
+inputs:
+    - id: msin
+      type: Directory[]
+      doc: Unaveraged MeasurementSets with coverage of the target directions.
+    - id: model_image_folder
+      type: Directory
+      doc: Folder with 1.2" model images.
+    - id: h5parm
+      type: File
+      doc: Merged h5parms
+    - id: lofar_helpers
+      type: Directory
+      doc: LOFAR helpers directory.
+    - id: facetselfcal
+      type: Directory
+      doc: facetselfcal directory.
+    - id: copy_to_local_scratch
+      type: boolean?
+      doc: Specific option for using --bypass-file-store on the Spider cluster to run predict and subtract on local scratch.
+      default: false
+    - id: ncpu
+      type: int?
+      doc: Number of cores to use during predict and subtract.
+      default: 16
+    - id: dysco_bitrate
+      type: int?
+      doc: Number of bits per float used for columns containing visibilities.
+      default: 8
+
+steps:
+    - id: get_facet_layout
+      label: Get DS9 facet layout
+      in:
+        - id: msin
+          source: msin
+          valueFrom: $(self[0])
+        - id: h5parm
+          source: h5parm
+        - id: facetselfcal
+          source: facetselfcal
+      out:
+        - id: facet_regions
+      run: ../steps/get_facet_layout.cwl
+
+    - id: get_model_images
+      label: Get WSClean model images
+      in:
+        - id: model_image_folder
+          source: model_image_folder
+      out:
+        - id: filtered_model_image_folder
+      run: ../steps/gather_model_images.cwl
+
+    - id: subtract_fov_wsclean
+      label: Subtract complete FoV
+      in:
+         - id: msin
+           source: msin
+         - id: h5parm
+           source: h5parm
+         - id: facet_regions
+           source: get_facet_layout/facet_regions
+         - id: model_image_folder
+           source: get_model_images/filtered_model_image_folder
+         - id: lofar_helpers
+           source: lofar_helpers
+         - id: copy_to_local_scratch
+           source: copy_to_local_scratch
+         - id: ncpu
+           source: ncpu
+      out:
+         - subtracted_ms
+      run: ../steps/subtract_fov_wsclean.cwl
+      scatter: msin
+
+    - id: split_polygons
+      label: Split polygon file
+      in:
+         - id: facet_regions
+           source: get_facet_layout/facet_regions
+         - id: h5parm
+           source: h5parm
+         - id: lofar_helpers
+           source: lofar_helpers
+      out:
+         - id: polygon_info
+         - id: polygon_regions
+      run: ../steps/split_polygons.cwl
+
+    - id: predict_facet
+      label: Predict a polygon back in empty MS
+      in:
+         - id: subtracted_ms
+           source: subtract_fov_wsclean/subtracted_ms
+         - id: polygon_region
+           source: split_polygons/polygon_regions
+         - id: h5parm
+           source: h5parm
+         - id: polygon_info
+           source: split_polygons/polygon_info
+         - id: model_image_folder
+           source: get_model_images/filtered_model_image_folder
+         - id: lofar_helpers
+           source: lofar_helpers
+         - id: copy_to_local_scratch
+           source: copy_to_local_scratch
+         - id: ncpu
+           source: ncpu
+      out:
+         - facet_ms
+      run: ../steps/predict_facet.cwl
+      scatter: [subtracted_ms, polygon_region]
+      scatterMethod: flat_crossproduct
+
+    - id: make_concat_parset
+      label: Make concat parsets
+      in:
+         - id: msin
+           source: predict_facet/facet_ms
+         - id: lofar_helpers
+           source: lofar_helpers
+         - id: dysco_bitrate
+           source: dysco_bitrate
+      out:
+         - id: concat_parsets
+      run: ../steps/make_concat_parsets.cwl
+
+    - id: concat_facets
+      label: Run DP3 parsets
+      in:
+        - id: parset
+          source: make_concat_parset/concat_parsets
+        - id: msin
+          source: predict_facet/facet_ms
+      out:
+        - id: msout
+      run: ../steps/dp3_parset.cwl
+      scatter: parset
+
+requirements:
+  - class: MultipleInputFeatureRequirement
+  - class: ScatterFeatureRequirement
+
+outputs:
+    - id: facet_ms
+      type: Directory[]
+      outputSource:
+        - concat_facets/msout
+        - predict_facet/facet_ms
+      pickValue: first_non_null
+    - id: polygon_info
+      type: File
+      outputSource: split_polygons/polygon_info
+    - id: polygon_regions
+      type: File[]
+      outputSource: split_polygons/polygon_regions