gsmcal.cwl

class: Workflow
cwlVersion: v1.2
id: gsmcal
label: gsmcal
inputs:
  - id: max_dp3_threads
    type: int?
  - id: msin
    type: Directory[]
  - id: filter_baselines
    type: string?
    default: '[CR]S*&'
  - id: target_skymodel
    type: 
      - File
      - Directory
  - id: do_smooth
    type: boolean?
    default: false
  - id: propagatesolutions
    type: boolean?
    default: true
  - id: avg_timeresolution_concat
    type: int?
    default: 8
  - id: avg_freqresolution_concat
    type: string?
    default: '97.64kHz'
  - id: min_unflagged_fraction
    type: float?
    default: 0.5
  - id: refant
    type: string?
    default: 'CS001HBA0'
  - id: rfistrategy
    type:
      - File?
      - string?
  - id: aoflag_reorder
    type: boolean?
    default: false
  - id: aoflag_chunksize
    type: int?
    default: 2000
  - id: aoflag_freqconcat
    type: boolean?
    default: true
  - id: gsmcal_step
    type: string?
    default: 'phase'
  - id: skymodel_source
    type: string?
    default: 'TGSS'
  - id: process_baselines_target
    type: string?
    default: '[CR]S*&'
  - id: insolutions
    type: File
  - id: selfcal
    type: boolean?
    default: $(true)
  - id: chunkduration
    type: float?
    default: 0.0
  - id: targetname
    type: string?
    default: 'pointing'
  - id: wsclean_tmpdir
    type: string?
    default: '/tmp'
  - id: total_bandwidth
    type: float?
    default: 50
  - id: midfreq
    type: float?
    default: 50e6
  - id: groupnames
    type: string[]
  - id: filenames
    type: File
  - id: selfcal_region
    type: File?
  - id: selfcal_strategy
    type: string?
    default: 'HBA'
outputs:
  - id: msout
    outputSource:
      # - calibrate_target/msout
      - selfcal_target/msout
      - selfcal_target_hba/msout
    pickValue: all_non_null
    linkMerge: merge_flattened
    type: Directory[]
  - id: outh5parm
    outputSource:
      - h5parm_collector/outh5parm
    type: File
  - id: outsolutions
    outputSource:
      - selfcal_target/outsolutions
      - insolutions
    pickValue: first_non_null
    type: File
  - id: bad_antennas
    outputSource:
      - identifybadantennas_join/filter_out
    type: string
  - id: outh5parm_logfile
    outputSource:
      - concat_logfiles_losoto/output
    type: File
  - id: Ateam_flags_join_out
    outputSource:
      - Ateam_flags_join/flagged_fraction_antenna
    type: File
  - id: inspection
    outputSource:
      - losoto_plot_P/output_plots
      - losoto_plot_P2/output_plots
      - losoto_plot_Pd/output_plots
      - losoto_plot_Pd2/output_plots
      - losoto_plot_tec/output_plots
      - plot_unflagged/output_imag
      - selfcal_target/inspection
    type: File[]
    linkMerge: merge_flattened
    pickValue: all_non_null
  - id: out_refant
    outputSource:
      - findRefAnt_join/refant
    type: string
  - id: logfiles
    outputSource:
      - concat_logfiles_identify/output
      - concat_logfiles_RefAnt/output
      - concat_logfiles_calib/output
      - concat_logfiles_dp3concat/output
      - concat_logfiles_blsmooth/output
      - concat_logfiles_unflagged/output
      - aoflag/logfile
      - selfcal_target/logfiles
    type: File[]
    linkMerge: merge_flattened
    pickValue: all_non_null
  - id: removed_bands
    outputSource:
      - check_unflagged_fraction/filenames
    type: string[]
steps:
  - id: identifybadantennas
    in:
      - id: msin
        source: msin
    out:
      - id: flaggedants
      - id: logfile
    run: ../../steps/identify_bad_antennas.cwl
    label: identifybadantennas
    scatter:
      - msin
  - id: identifybadantennas_join
    in:
      - id: flaggedants
        source:
          - identifybadantennas/flaggedants
      - id: filter
        source: filter_baselines
    out:
      - id: filter_out
      - id: logfile
    run: ../../steps/identify_bad_antennas_join.cwl
    label: identifybadantennas_join
  - id: findRefAnt_join
    in:
      - id: flagged_fraction_dict
        source:
          - concat/flagged_fraction_dict
      - id: filter_station
        source: refant
    out:
      - id: refant
      - id: logfile
    run: ../../steps/findRefAnt_join.cwl
    label: findRefAnt_join
  - id: Ateam_flags_join
    in:
      - id: flagged_fraction_dict
        source:
          - concat/flagged_fraction_dict
      - id: filter_station
        default: ''
      - id: state
        default: 'concat'
    out:
      - id: flagged_fraction_antenna
    run: ../../steps/findRefAnt_join.cwl
    label: Ateam_flags_join

  - id: concat_logfiles_dp3concat
    in:
      - id: file_list
        source:
          - concat/dp3concat.log
      - id: file_prefix
        default: dp3concat
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_dp3concat
  - id: concat_logfiles_blsmooth
    in:
      - id: file_list
        source:
          - calibrate_target/BLsmooth.log
        pickValue: all_non_null
      - id: file_prefix
        default: blsmooth_target
      - id: execute
        source: selfcal
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_blsmooth
    when: $(!inputs.execute)
  - id: concat_logfiles_calib
    in:
      - id: file_list
        source:
          - calibrate_target/gaincal.log
        pickValue: all_non_null
      - id: file_prefix
        default: gaincal
      - id: execute
        source: selfcal
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_calib
    when: $(!inputs.execute)
  - id: concat_logfiles_losoto
    in:
      - id: file_list
        linkMerge: merge_flattened
        source:
          - h5parm_collector/log
          - losoto_plot_tec/logfile
          - losoto_plot_P/logfile
          - losoto_plot_P2/logfile
          - losoto_plot_Pd/logfile
          - losoto_plot_Pd2/logfile
        pickValue: all_non_null
      - id: file_prefix
        default: losoto_gsmcal
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_losoto
  - id: concat_logfiles_unflagged
    in:
      - id: file_list
        linkMerge: merge_flattened
        source:
          - merge_array_files/output
          - plot_unflagged/logfile
      - id: file_prefix
        default: check_unflagged_fraction
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_unflagged
  - id: concat_logfiles_identify
    in:
      - id: file_list
        linkMerge: merge_flattened
        source:
          - identifybadantennas/logfile
          - identifybadantennas_join/logfile
      - id: file_prefix
        default: identifyBadAntennas
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_identify
  - id: concat_logfiles_RefAnt
    in:
      - id: file_list
        linkMerge: merge_flattened
        source:
          - findRefAnt_join/logfile
      - id: file_prefix
        default: findRefAnt
    out:
      - id: output
    run: ../../steps/concatenate_files.cwl
    label: concat_logfiles_RefAnt
  - id: concat
    in:
      - id: msin
        source: 
          - msin
      - id: group_id
        source: groupnames
      - id: groups_specification
        source: filenames
      - id: filter_baselines
        source: identifybadantennas_join/filter_out
      - id: avg_timeresolution_concat
        source: avg_timeresolution_concat
      - id: avg_freqresolution_concat
        source: avg_freqresolution_concat
      - id: chunkduration
        source: chunkduration
    out:
      - id: msout
      - id: flagged_fraction_dict
      - id: dp3concat.log
    run: ./concat.cwl
    label: concat
    scatter:
      - group_id

  - id: merge_array_concat
    in:
      - id: input
        source: concat/msout
    out:
      - id: output
    run: ../../steps/merge_array.cwl
    label: merge_array_concat

  - id: sort_times_into_freqGroups_full
    in:
      - id: msin
        source:
          - merge_array_concat/output
      - id: numbands
        valueFrom: $(-1)
      - id: DP3fill
        default: true
      - id: stepname
        default: .fullconcat.MS
      - id: truncateLastSBs
        default: false
    out:
      - id: filenames
      - id: groupnames
      - id: total_bandwidth
      - id: midfreq
      - id: minfreq
      - id: logfile
    run: ../../steps/sort_times_into_freqGroups.cwl
    label: sorttimesintofreqGroups_full
  
  - id: concat2
    in:
      - id: msin
        source: 
          - merge_array_concat/output
      - id: group_id
        source: sort_times_into_freqGroups_full/groupnames
      - id: groups_specification
        source: sort_times_into_freqGroups_full/filenames
      - id: filter_baselines
        source: identifybadantennas_join/filter_out
      - id: avg_timeresolution_concat
        source: avg_timeresolution_concat
      - id: avg_freqresolution_concat
        source: avg_freqresolution_concat
      - id: chunkduration
        source: chunkduration
      - id: execute
        source: selfcal
      - id: selfcal_strategy
        source: selfcal_strategy
    out:
      - id: msout
      - id: flagged_fraction_dict
      - id: dp3concat.log
    run: ./concat.cwl
    label: concat2
    scatter: group_id
    # HBA should do a full concat anyway, even without selfcal, so ignore execute.
    when: $(inputs.selfcal_strategy == 'HBA') 
  
  - id: merge_array_concat_full
    in:
      - id: input
        source: concat2/msout
    out:
      - id: output
    run: ../../steps/merge_array.cwl
    label: merge_array_concat_full

  - id: aoflag
    in:
      - id: msin
        source: merge_array_concat_full/output
      - id: verbose
        default: true
      - id: concatenate-frequency
        source: aoflag_freqconcat
      - id: strategy
        source: rfistrategy
      - id: reorder
        source: aoflag_reorder
      - id: chunk-size
        source: aoflag_chunksize
    out:
      - id: output_ms
      - id: logfile
    run: ../../steps/aoflag.cwl
    label: aoflag

  - id: check_unflagged_fraction
    in:
      - id: msin
        source: aoflag/output_ms
      - id: min_fraction
        source: min_unflagged_fraction
    out:
      - id: msout
      - id: frequency
      - id: unflagged_fraction
      - id: filenames
      - id: logfile
    run: ../../steps/check_unflagged_fraction.cwl
    label: check_unflagged_fraction
    scatter:
      - msin

  - id: merge_array
    in:
      - id: input
        source: check_unflagged_fraction/msout
    out:
      - id: output
    run: ../../steps/merge_array.cwl
    label: merge_array

  - id: merge_array_files
    in:
      - id: input
        source: check_unflagged_fraction/logfile
    out:
      - id: output
    run: ../../steps/merge_array_files.cwl
    label: merge_array_files

  - id: check_filtered_MS_array
    in:
      - id: input
        source: merge_array/output
    out:
      - id: output
    run: ../../steps/check_filtered_MS_array.cwl
    label: check_filtered_MS_array


  - id: calibrate_target
    in:
      - id: max_dp3_threads
        source: max_dp3_threads
      - id: msin
        source: check_filtered_MS_array/output
      - id: skymodel
        source: target_skymodel
      - id: do_smooth
        source: do_smooth
      - id: propagatesolutions
        source: propagatesolutions
      - id: gsmcal_step
        source: gsmcal_step
      - id: execute
        source: selfcal
      - id: selfcal_strategy
        source: selfcal_strategy
      - id: nchan
        valueFrom: $(1)
      - id: smoothnessconstraint
        valueFrom: $(5000000)
      - id: smoothnessreffrequency
        valueFrom: $(120000000)
    out:
      - id: msout
      - id: BLsmooth.log
      - id: gaincal.log
      - id: outh5parm
    run: ./calib_targ.cwl
    label: calibrate_target
    scatter:
      - msin
    when: $(!inputs.execute || inputs.selfcal_strategy == 'HBA')

  # START image target field
  - id: h5parm_collector_skysolve
    in:
      - id: h5parmFiles
        source:
          - calibrate_target/outh5parm
        pickValue: all_non_null
        linkMerge: merge_flattened
      - id: squeeze
        default: true
      - id: verbose
        default: true
      - id: clobber
        default: true
      - id: execute
        source: selfcal
      - id: selfcal_strategy
        source: selfcal_strategy
    out:
      - id: outh5parm
      - id: log
    run: ../../steps/H5ParmCollector.cwl
    label: h5parm_collector_skysolve
    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')

  - id: apply_skysolve
    in:
      - id: max_dp3_threads
        source: max_dp3_threads
      - id: msin
        source: calibrate_target/msout
      - id: msin_datacolumn
        default: DATA
      - id: parmdb
        source: h5parm_collector_skysolve/outh5parm
      - id: msout_datacolumn
        default: CORRECTED_DATA
      - id: storagemanager
        default: Dysco
      - id: databitrate
        default: 0
      - id: solset
        default: sol000
      - id: correction
        valueFrom: phase000
      - id: execute
        source: selfcal
      - id: selfcal_strategy
        source: selfcal_strategy
    out:
      - id: msout
      - id: logfile
    run: ../../steps/applycal.cwl
    scatter:
      - msin
    label: apply_skysolve
    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')

  - id: image_target
    in:
      - id: msin
        source: apply_skysolve/msout
      - id: image_scale
        default: 1.5asec
      - id: auto_threshold
        default: 0.5
      - id: image_size
        default: [2000,2000]
      - id: niter
        default: 285
      - id: nmiter
        default: 1
      - id: multiscale
        default: true
      - id: use-wgridder
        default: true
      - id: mgain
        default: 0.8
      - id: parallel-reordering
        default: 4
      - id: channels-out
        default: 6
      - id: join-channels
        default: True
      - id: fit-spectral-pol
        valueFrom: $(3)
      - id: weighting
        default: briggs -1.5
      - id: minuv-l
        default: 80
      - id: tempdir
        source: wsclean_tmpdir
      - id: image_name
        source: targetname
      - id: no_model_update
        default: false
      # This can be turned on if we do a separate predict after, not sure if that would be faster.
      # - id: baseline_averaging
      #   default: 0.4908738521234052
      - id: save_source_list
        default: true
      - id: execute
        source: selfcal
      - id: selfcal_strategy
        source: selfcal_strategy
    out:
      - id: msout
      - id: dirty_image
      - id: image
      - id: logfile
      - id: sourcelist
    run: ../../steps/wsclean.cwl
    label: image_target
    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
    # END image target field

  # START HBA selfcal
  - id: selfcal_target_hba
    in:
      - id: max_dp3_threads
        source: max_dp3_threads
      - id: msin
        source: image_target/msout
      # - id: skymodel
      #   source: image_target/sourcelist
      - id: do_smooth
        valueFrom: $(false)
      - id: propagatesolutions
        source: propagatesolutions
      - id: gsmcal_step
        source: gsmcal_step
      - id: execute
        source: selfcal
      - id: selfcal_strategy
        source: selfcal_strategy
      - id: model_column
        valueFrom: $(["MODEL_DATA"])
      - id: nchan
        valueFrom: $(1)
      - id: smoothnessconstraint
        valueFrom: $(5000000)
      - id: smoothnessreffrequency
        valueFrom: $(120000000)
    out:
      - id: msout
      - id: BLsmooth.log
      - id: gaincal.log
      - id: outh5parm
    run: ./calib_targ.cwl
    label: calibrate_target
    scatter:
      - msin
    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
  # END HBA selfcal

  - id: selfcal_target
    in:
      - id: max_dp3_threads
        source: max_dp3_threads
      - id: msin
        source: check_filtered_MS_array/output
      - id: skymodel
        source: target_skymodel
      - id: propagatesolutions
        source: propagatesolutions
      - id: do_smooth
        source: do_smooth
      - id: refant
        source: findRefAnt_join/refant
      - id: skymodel_source
        source: skymodel_source
      - id: gsmcal_step
        source: gsmcal_step
      - id: process_baselines_target
        source: process_baselines_target
      - id: bad_antennas
        source: identifybadantennas_join/filter_out
      - id: insolutions
        source: insolutions
      - id: targetname
        source: targetname
      - id: wsclean_tmpdir
        source: wsclean_tmpdir
      - id: total_bandwidth
        source: total_bandwidth
      - id: midfreq
        source: midfreq
      - id: selfcal_region
        source: selfcal_region
      - id: rfistrategy
        source: rfistrategy
      - id: aoflag_reorder
        source: aoflag_reorder
      - id: aoflag_chunksize
        source: aoflag_chunksize
      - id: aoflag_freqconcat
        source: aoflag_freqconcat
      - id: execute
        source: selfcal
    out:
      - id: msout
      - id: logfiles
      - id: inspection
      - id: outh5parm
      - id: outsolutions
    run: ./selfcal_targ.cwl
    label: selfcal_target
    when: $(inputs.execute && inputs.selfcal_strategy == 'LBA')

  - id: h5parm_collector
    in:
      - id: h5parmFiles
        source:
          # - calibrate_target/outh5parm
          - selfcal_target/outh5parm
          - selfcal_target_hba/outh5parm
        pickValue: all_non_null
        linkMerge: merge_flattened
      - id: squeeze
        default: true
      - id: verbose
        default: true
      - id: clobber
        default: true
      - id: execute
        source: selfcal
    out:
      - id: outh5parm
      - id: log
    run: ../../steps/H5ParmCollector.cwl
    label: H5parm_collector
  - id: plot_unflagged
    in:
      - id: frequencies
        source: check_unflagged_fraction/frequency
      - id: unflagged_fraction
        source: check_unflagged_fraction/unflagged_fraction
    out:
      - id: output_imag
      - id: logfile
    run: ../../steps/plot_unflagged.cwl
    label: plot_unflagged
  - id: losoto_plot_tec
    in:
      - id: input_h5parm
        source: h5parm_collector/outh5parm
      - id: soltab
        default: sol000/tec000
      - id: axesInPlot
        default:
          - time
      - id: axisInTable
        default: ant
      - id: plotFlag
        default: true
      - id: refAnt
        source: refant
      - id: minmax
        default:
          - -0.5
          - 0.5
      - id: prefix
        default: tec_
      - id: execute
        source: selfcal
    out:
      - id: output_plots
      - id: logfile
      - id: parset
    run: ../../steps/LoSoTo.Plot.cwl
    when: $(inputs.execute)
    label: losoto_plot_tec
  - id: losoto_plot_P
    in:
      - id: input_h5parm
        source: h5parm_collector/outh5parm
      - id: soltab
        default: sol000/phase000
      - id: axesInPlot
        default:
          - time
          - freq
      - id: axisInTable
        default: ant
      - id: minmax
        default:
          - -3.14
          - 3.14
      - id: plotFlag
        default: true
      - id: refAnt
        source: findRefAnt_join/refant
      - id: prefix
        default: ph_
      - id: selfcal_strategy
        source: selfcal_strategy
      - id: execute
        source: selfcal
        valueFrom: '$(self ? (false || inputs.selfcal_strategy == "HBA") : true)'
    out:
      - id: output_plots
      - id: logfile
      - id: parset
    run: ../../steps/LoSoTo.Plot.cwl
    label: losoto_plot_P
    when: $(inputs.execute)
  - id: losoto_plot_P2
    in:
      - id: input_h5parm
        source: h5parm_collector/outh5parm
      - id: soltab
        default: sol000/phase000
      - id: axesInPlot
        default:
          - time
      - id: axisInTable
        default: ant
      - id: axisInCol
        default: pol
      - id: minmax
        default:
          - -3.14
          - 3.14
      - id: plotFlag
        default: true
      - id: refAnt
        source: findRefAnt_join/refant
      - id: prefix
        default: ph_
      - id: execute
        source: selfcal
        valueFrom: '$(self ? (false || inputs.selfcal_strategy == "HBA") : true)'
    out:
      - id: output_plots
      - id: logfile
      - id: parset
    run: ../../steps/LoSoTo.Plot.cwl
    label: losoto_plot_P2
    when: $(inputs.execute)
  - id: losoto_plot_Pd
    in:
      - id: input_h5parm
        source: h5parm_collector/outh5parm
      - id: soltab
        default: sol000/phase000
      - id: axesInPlot
        default:
          - time
          - freq
      - id: axisInTable
        default: ant
      - id: axisDiff
        default: pol
      - id: minmax
        default:
          - -3.14
          - 3.14
      - id: plotFlag
        default: true
      - id: refAnt
        source: findRefAnt_join/refant
      - id: prefix
        default: ph_poldif
      - id: execute
        source: selfcal
        valueFrom: '$(self ? (false || inputs.selfcal_strategy == "HBA") : true)'
    out:
      - id: output_plots
      - id: logfile
      - id: parset
    run: ../../steps/LoSoTo.Plot.cwl
    label: losoto_plot_Pd
    when: $(inputs.execute)
  - id: losoto_plot_Pd2
    in:
      - id: input_h5parm
        source: h5parm_collector/outh5parm
      - id: soltab
        default: sol000/phase000
      - id: axesInPlot
        default:
          - time
      - id: axisInTable
        default: ant
      - id: axisDiff
        default: pol
      - id: minmax
        default:
          - -3.14
          - 3.14
      - id: plotFlag
        default: true
      - id: refAnt
        source: findRefAnt_join/refant
      - id: prefix
        default: ph_poldif_
      - id: execute
        source: selfcal
        valueFrom: '$(self ? (false || inputs.selfcal_strategy == "HBA") : true)'
    out:
      - id: output_plots
      - id: logfile
      - id: parset
    run: ../../steps/LoSoTo.Plot.cwl
    label: losoto_plot_Pd2
    when: $(inputs.execute)
requirements:
  - class: SubworkflowFeatureRequirement
  - class: ScatterFeatureRequirement
  - class: MultipleInputFeatureRequirement
  - class: StepInputExpressionRequirement
  - class: InlineJavascriptRequirement