add major steps for target pipeline

lofar-cwl/steps/applytarget.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: applytarget
+baseCommand:
+  - DPPP
+inputs:
+  - id: msin
+    type: Directory
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - id: msout_name
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout=msout_name
+      separate: false
+    doc: Name of output MS
+  - default: DATA
+    id: msin_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msin.datacolumn=
+      separate: false
+    doc: Input data Column
+  - id: parmdb
+    type: File
+    inputBinding:
+      position: 0
+      prefix: applycal.parmdb=
+      separate: false
+    doc: >-
+      Path of parmdb in which the parameters are stored. This can also be an
+      H5Parm file, in that case the filename has to end in '.h5'
+  - id: msout_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout.datacolumn=
+      separate: false
+    doc: Output data column
+  - default: gain
+    id: correction
+    type: string
+    inputBinding:
+      position: 0
+      prefix: applycal.correction=
+      separate: false
+    doc: >
+      Type of correction to perform. When using H5Parm, this is for now the name
+      of the soltab; the type will be deduced from the metadata in that soltab,
+      except for full Jones, in which case correction should be 'fulljones'.
+  - id: storagemanager
+    type: string
+    default: ""
+    inputBinding:
+      prefix: msout.storagemanager=
+  - id: databitrate
+    type: int?
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
+       separate: false
+  - id: updateweights
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: applycal.updateweights=
+      separate: false
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msin.basename)
+  - id: logfile
+    type: File
+    outputBinding:
+      glob: 'applycal_$(inputs.correction).log'
+stdout: applycal_$(inputs.correction).log
+stderr: applycal_$(inputs.correction)_err.log
+arguments:
+  - 'steps=[applycal]'
+requirements:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: InlineJavascriptRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: 'lofareosc/prefactor:HBAcalibrator'

lofar-cwl/steps/dpppconcat.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: dpppconcat
+baseCommand:
+  - DPPP
+inputs:
+  - id: msin
+    type: Directory[]
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - id: msout_name
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout=
+      separate: false
+  - default: DATA
+    id: msin_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msin.datacolumn=
+      separate: false
+    doc: Input data Column
+  - default: DATA
+    id: msout_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout.datacolumn=
+      separate: false
+  - default: null
+    id: filter_baselines
+    type: string
+    inputBinding:
+      position: 0
+      prefix: filter.baseline=
+      separate: false
+      valueFrom: '$(self)'
+  - default: false
+    id: filter_remove
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: filter.remove=True
+  - id: writefullresflag
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.writefullresflag=True
+  - id: overwrite
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.overwrite=True
+  - id: storagemanager
+    type: string
+    default: ""
+    inputBinding:
+       prefix: msout.storagemanager=
+       separate: false
+  - id: databitrate
+    type: int?
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
+       separate: false
+  - id: missingdata
+    type: boolean
+    inputBinding:
+       prefix: msout.missingdata=True
+       separate: false
+  - id: baseline
+    type: string?
+    default: '*'
+    inputBinding:
+       prefix: msin.baseline=
+       separate: false
+  - id: avg_timeresolution
+    type: int?
+    default: 1
+    inputBinding:
+       prefix: avg.timeresolution=
+       separate: false
+  - id: avg_freqresolution
+    type: string?
+    default: '12.205kHz'
+    inputBinding:
+       prefix: avg.freqresolution=
+       separate: false
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msout_name)
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'concat*.log'
+arguments:
+  - 'steps=[filter,avg]'
+  - 'msout.orderms=False'
+requirements:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: InlineJavascriptRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: concat.log
+stderr: concat_err.log
\ No newline at end of file

lofar-cwl/steps/gaincal.cwl

@@ -97,6 +97,14 @@ inputs:
       prefix: msout=
       separate: false
     doc: Output Measurement Set
+  - default: true
+    id: propagatesolutions
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: propagatesolutions=True
+      separate: false
+    doc: Output Measurement Set
 outputs:
   - id: msout
     doc: Output Measurement Set
@@ -109,8 +117,14 @@ outputs:
     outputBinding:
       glob: $(inputs.output_name_h5parm)
     format: 'lofar:#H5Parm'
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'gaincal*.log'
+stdout: gaincal.log
+stderr: gaincal_err.log
 arguments:
-  - 'steps=[gaincal]'
+  - 'steps=[filter,gaincal]'
 hints:
   - class: DockerRequirement
     dockerPull: 'lofareosc/prefactor:HBAcalibrator'

steps/add_missing_stations.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: add_missing_stations
+baseCommand:
+  - python3
+  - /usr/local/bin/add_missing_stations.py
+inputs:
+  - default: output.h5
+    id: h5parm
+    type: File
+    inputBinding:
+      position: 1
+    doc: H5parm to which this action should be performed.
+  - id: refh5parm
+    type: File
+    inputBinding:
+      position: 0
+      prefix: '--refh5parm'
+    doc: External H5parm from which the full list of antennas is used from.
+  - id: solset
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--solset'
+    doc: Input calibration solutions
+  - id: refsolset
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--refsolset'
+    doc: Input calibration solutions of the reference h5parm file
+  - id: soltab_in
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--soltab_in'
+    doc: Input solution table
+  - id: soltab_out
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--soltab_out'
+    doc: Output solution table (has to be different from input solution table)
+  - id: filter
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--filter'
+    doc: Filter these antenna string from the processing
+  - id: bad_antennas
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--bad_antennas'
+    doc: Antenna string to be processed
+outputs:
+  - id: outh5parm
+    type: File
+    outputBinding:
+      glob: $(inputs.h5parm.basename)
+  - id: log
+    type: File[]
+    outputBinding:
+      glob: 'add_missing_stations*.log'
+
+stdout: add_missing_stations.log
+stderr: add_missing_stations_err.log
+label: add_missing_stations
+hints:
+  - class: DockerRequirement
+    dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.h5parm)
+        writable: true

steps/filter_ms_group.cwl

@@ -19,7 +19,7 @@ inputs:
       position: 2
 outputs:
   - id: output
-    type: 'Directory[]?'
+    type: Directory[]
     outputBinding:
       loadContents: true
       glob: selected_ms.json

steps/find_skymodel_target.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: find_skymodel_target_py
+baseCommand:
+  - python3
+inputs:
+    - id: msin
+      type: Directory[]
+      doc: MS containing the target
+      inputBinding:
+        position: 0
+    - id: SkymodelPath
+      type: File?
+      doc: File containing or putting the skymodel
+    - id: Radius
+      type: float?
+      doc: Radius of the skymodel
+      default: 5.0
+    - id: Source
+      type: string?
+      doc: Source of the skymodel
+      default: 'TGSS'
+    - id: DoDownload
+      type: boolean?
+      doc: Download a new skymodel if given path is empty
+      default: true
+
+label: find_skymodel_target.py
+arguments:
+  - '-c'
+  - |
+    import sys
+    import shutil
+    null = None
+
+    from download_skymodel_target import main as download_skymodel_target
+
+    mss = sys.argv[1:]
+
+    SkymodelPath = $(inputs.skymodels)
+    if SkymodelPath is None:
+        SkymodelPath = "target.skymodel"
+    else:
+        SkymodelPath = SkymodelPath["path"]
+
+    Radius = $(inputs.Radius)
+    Source = $(inputs.Source)
+    DoDownload = "$(inputs.DoDownload)"
+    
+    output = {}
+    if DoDownload != 'null':
+        output = download_skymodel_target(mss, SkymodelPath, Radius, DoDownload="True", Source)
+    else:
+        output = download_skymodel_target(mss, SkymodelPath, Radius, DoDownload="False", Source)
+
+    if SkymodelPath != "target.skymodel":
+       shutil.copyfile(SkymodelPath, "target.skymodel)
+   
+outputs:
+  - id: skymodel
+    type: File?
+    outputBinding:
+      glob: target.skymodel
+  - id: logfile
+    type: File?
+    outputBinding:
+      glob: find_skymodel_target.log    
+        
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  DockerRequirement:
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: find_skymodel_target.log
\ No newline at end of file

steps/h5parmcat.cwl

@@ -58,6 +58,13 @@ inputs:
       position: 0
       prefix: '-c'
     doc: overwrite output
+  - default: false
+    id: history
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: '-H'
+    doc: Keep history of the export soltabs
 outputs:
   - id: outh5parm
     doc: Output h5parm
@@ -68,7 +75,7 @@ outputs:
   - id: log
     type: File[]
     outputBinding:
-      glob: 'parm_collector_output*.log'
+      glob: 'h5parm_collector_output*.log'
 label: h5parm_cat
 arguments:
   - position: 0

steps/sort_times_into_freqGroups.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: sort_times_into_freqGroups
+baseCommand:
+  - python3
+inputs:
+    - id: msin
+      type: Directory[]
+      doc: MS to sort
+      inputBinding:
+        position: 0
+    - id: numbands
+      type: int?
+      default: 10
+      doc: Number of how many files should be grouped together in frequency.
+    - id: NDPPPfill
+      type: boolean?
+      default: True
+      doc: Add dummy file-names for missing frequencies, so that NDPPP can fill the data with flagged dummy data.
+    - id: stepname
+      type: string?
+      default: null
+      doc: Add this step-name into the file-names of the output files.
+    - id: firstSB
+      type: int?
+      default: null
+      doc: If set, then reference the grouping of files to this station-subband. As if a file with this station-subband would be included in the input files.
+    - id: mergeLastGroup
+      type: boolean?
+      default: False
+      doc: Add dummy file-names for missing frequencies, so that NDPPP can fill the data with flagged dummy data.
+    - id: truncateLastSBs
+      type: boolean?
+      default: True
+      doc: Add dummy file-names for missing frequencies, so that NDPPP can fill the data with flagged dummy data.
+
+label: sort_times_into_freqGroups.py
+arguments:
+  - '-c'
+  - |
+    import sys
+    import json
+    from sort_times_into_freqGroups_CWL import main as sort_times_into_freqGroups
+    
+    mss = sys.argv[1:]
+    numbands = int($(inputs.numbands))
+    stepname = $(inputs.stepname)
+    mergeLastGroup = $(inputs.mergeLastGroup)
+    truncateLastSBs = $(inputs.truncateLastSBs)
+    firstSB = int($(inputs.firstSB))
+
+    output = sort_times_into_freqGroups(mss, numbands, NDPPPfill, stepname, mergeLastGroup, truncateLastSBs, firstSB)
+
+    filenames  = output['filenames']
+    groupnames = output['groupnames']
+    
+    #cwl_output = {}
+    #filelists  = []
+    #for groupname in groupnames:
+    #    filelist = []
+    #    for filename in filenames[groupname]:
+    #        filelist.append({'class': 'Dictionary', 'path': filename})
+    #    filelists.append(filelist)
+        
+    cwl_output['groupnames'] = groupnames
+
+    with open('./filenames.json', 'w') as fp:
+        json.dump(filenames, fp)
+        
+    with open('./out.json', 'w') as fp:
+        json.dump(cwl_output, fp)
+        
+    return(0)
+    
+outputs:
+  - id: filenames
+    type: File
+    outputBinding:
+        glob: 'filenames.json'
+  - id: groupnames
+    type: string[]
+    outputBinding:
+        loadContents: true
+        glob: 'out.json'
+        outputEval: $(JSON.parse(self[0].contents).groupnames)
+  - id: logfile
+    type: File?
+    outputBinding:
+      glob: sort_times_into_freqGroups.log    
+        
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: sort_times_into_freqGroups.log
\ No newline at end of file

subworkflow/concat_calibrate.cwl

+class: Workflow
+cwlVersion: v1.0
+id: concat_calibrate
+label: concat_calibrate
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+inputs:
+  - id: msin
+    type: Directory[]
+    'sbg:x': -1000
+    'sbg:y': -100
+  - id: group_id
+    type: string
+    'sbg:x': -1000
+    'sbg:y': 0
+  - id: groups_specification
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 100
+  - id: filter_baselines
+    type: string
+    'sbg:x': -1000
+    'sbg:y': 200
+  - id: avg_timeresolution_concat
+    type: int
+    'sbg:x': -1000
+    'sbg:y': 300
+  - id: avg_freqresolution_concat
+    type: string
+    'sbg:x': -1000
+    'sbg:y': 400
+  - id: skymodel
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 500
+  - id: do_smooth
+    type: boolean
+    'sbg:x': -1000
+    'sbg:y': 600
+  - id: propagatesolutions
+    type: boolean
+    'sbg:x': -1000
+    'sbg:y': 700
+outputs:
+  - id: msout
+    outputSource:
+      - dpppconcat/msout
+    type: Directory
+    'sbg:x': 1000
+    'sbg:y': 0
+  - id: dpppconcat.log
+    outputSource:
+      - concat_logfiles_dpppconcat/output
+    type: File
+    'sbg:x': 1000
+    'sbg:y': -300
+  - id: BLsmooth.log
+    outputSource:
+      - BLsmooth/logfile
+    type: File
+    'sbg:x': 1000
+    'sbg:y': -100
+  - id: gaincal.log
+    outputSource:
+      - concat_logfiles_gaincal/output
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 200
+  - id: outh5parm
+    outputSource:
+      - calib_targ/h5parm
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 400
+
+steps:
+  - id: filter_ms_group
+    in:
+      - id: group_id
+        source: group_id
+      - id: groups_specification
+        source: groups_specification
+      - id: measurement_sets
+        source: msin
+    out:
+      - id: output
+    run: ../steps/filter_ms_group.cwl
+    label: filter_ms_group
+    'sbg:x': -500
+    'sbg:y': 0
+  - id: dpppconcat
+    in:
+      - id: msin
+        source: filter_ms_group/output
+      - id: msout_name
+        source: group_id
+      - id: msin_datacolumn
+        default: DATA
+      - id: msout_datacolumn
+        default: DATA
+      - id: baseline
+        source: filter_baselines
+      - id: filter_baselines
+        source: filter_baselines
+      - id: filter_remove
+        default: true
+      - id: overwrite
+        default: false
+      - id: missingdata
+        default: True
+      - id: storagemanager
+        default: "Dysco"
+      - id: databitrate
+        default: 0
+      - id: avg_timeresolution
+        source: avg_timeresolution_concat
+      - id: avg_freqresolution
+        source: avg_freqresolution_concat
+    out:
+      - id: msout
+      - id: logfile
+    run: ../lofar-cwl/steps/dpppconcat.cwl
+    label: dpppconcat
+    'sbg:x': -200
+    'sbg:y': 0
+  - id: BLsmooth
+    in:
+      - id: msin
+        source: dpppconcat/msout
+      - id: do_smooth
+        source: do_smooth
+      - id: in_column_name
+        default: DATA
+      - id: restore
+        default: true
+    out:
+      - id: msout
+      - id: logfile
+    run: ../steps/blsmooth.cwl
+    label: BLsmooth
+    'sbg:x': 100
+    'sbg:y': 0
+  - id: calib_targ
+    in:
+      - id: msin
+        source: BLsmooth/msout
+      - id: msin_datacolumn
+        default: SMOOTHED_DATA
+      - id: sourcedb
+        source: skymodel
+      - id: parmdb
+        default: 'instrument.h5'
+      - id: caltype
+        default: 'phaseonly'
+      - id: maxiter
+        default: 50
+      - id: solint
+        default: 1
+      - id: nchan
+        default: 0
+      - id: tolerance
+        default: 1e-3
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: usebeammodel
+        default: true
+      - id: usechannelfreq
+        default: true
+      - id: beammode
+        default: 'array_factor'
+      - id: onebeamperpatch
+        default: false
+    out:
+      - id: msout
+      - id: h5parm
+      - id: logfile
+    run: ../lofar-cwl/steps/gaincal.cwl
+    'sbg:x': 400
+    'sbg:y': 0
+  - id: concat_logfiles_dpppconcat
+    in:
+      - id: file_list
+        source:
+          - dpppconcat/logfile
+      - id: file_prefix
+        default: dpppconcat
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_dpppconcat
+    'sbg:x': 600
+    'sbg:y': 200
+  - id: concat_logfiles_gaincal
+    in:
+      - id: file_list
+        source:
+          - calib_targ/logfile
+      - id: file_prefix
+        default: gaincal
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_gaincal
+    'sbg:x': 600
+    'sbg:y': 400
+requirements: []
\ No newline at end of file

workflows/HBA_target.cwl

@@ -174,8 +174,8 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1600
   - id: avg_timeresolution
-    type: float?
-    default: 4.
+    type: int?
+    default: 4
     'sbg:x': -1000
     'sbg:y': 1700
   - id: avg_freqresolution
@@ -184,8 +184,8 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1800
   - id: avg_timeresolution_concat
-    type: float?
-    default: 8.
+    type: int?
+    default: 8
     'sbg:x': -1000
     'sbg:y': 1900
   - id: avg_freqresolution_concat
@@ -199,10 +199,8 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 2100
   - id: reference_stationSB
-    type: 
-      - int?
-      - string?
-    default: ''
+    type: int?
+    default: null
     'sbg:x': -1000
     'sbg:y': 2200
   - id: ionex_server

workflows/prefactor_target.cwl

@@ -170,7 +170,7 @@ inputs:
     'sbg:y': 1500
   - id: skymodel_source
     type: string?
-    default: TGSS
+    default: "TGSS"
     'sbg:x': -1000
     'sbg:y': 1600
   - id: avg_timeresolution
@@ -184,13 +184,13 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1800
   - id: avg_timeresolution_concat
-    type: float?
+    type: int?
     default: 8
     'sbg:x': -1000
     'sbg:y': 1900
   - id: avg_freqresolution_concat
     type: string?
-    default: 97.64kHz
+    default: '97.64kHz'
     'sbg:x': -1000
     'sbg:y': 2000
   - id: num_SBs_per_group
@@ -199,45 +199,45 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 2100
   - id: reference_stationSB
-    type: string?
-    default: ''
+    type: int?
+    default: null
     'sbg:x': -1000
-    'sbg:y': 1800
+    'sbg:y': 2200
   - id: ionex_server
     type: string?
     default: 'ftp://ftp.aiub.unibe.ch/CODE/'
     'sbg:x': -1000
-    'sbg:y': 1900
+    'sbg:y': 2300
   - id: ionex_prefix
     type: string?
     default: CODG
     'sbg:x': -1000
-    'sbg:y': 2000
+    'sbg:y': 2400
   - id: proxy_server
     type: string?
     default: null
     'sbg:x': -1000
-    'sbg:y': 2100
+    'sbg:y': 2500
   - id: proxy_port
     type: string?
     default: null
     'sbg:x': -1000
-    'sbg:y': 2200
+    'sbg:y': 2600
   - id: proxy_type
     type: string?
     default: null
     'sbg:x': -1000
-    'sbg:y': 2300
+    'sbg:y': 2700
   - id: proxy_user
     type: string?
     default: null
     'sbg:x': -1000
-    'sbg:y': 2400
+    'sbg:y': 2800
   - id: proxy_pass
     type: string?
     default: null
     'sbg:x': -1000
-    'sbg:y': 2500
+    'sbg:y': 2900
 outputs:
   - id: inspection
     outputSource:
@@ -248,20 +248,21 @@ outputs:
     'sbg:y': -600
   - id: solutions
     outputSource:
-      - prep/outh5parm
+      - finalize/solutions
     type: File
     'sbg:x': 2000
     'sbg:y': -500
   - id: msout
     outputSource:
-      - concat/msout
+      - finalize/msout
     type: 'Directory[]'
     'sbg:x': 2000
     'sbg:y': 0
   - id: logfiles
     outputSource:
       - prep/logfiles
-      - concat/logfiles
+      - gsmcal/logfiles
+      - finalize/logfiles
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 2000
@@ -345,19 +346,66 @@ steps:
     label: prep
     'sbg:x': 0
     'sbg:y': 0
-  - id: concat
+  - id: gsmcal
     in:
       - id: msin
         source:
           - prep/msout
       - id: filter_baselines
         source: process_baselines_target
+      - id: num_SBs_per_group
+        source: num_SBs_per_group
+      - id: reference_stationSB
+        source: reference_stationSB
+      - id: use_target
+        source: use_target
+      - id: target_skymodel
+        source: target_skymodel
+      - id: skymodel_source
+        source: skymodel_source
+      - id: do_smooth
+        source: do_smooth
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: avg_timeresolution_concat
+        source: avg_timeresolution_concat
+      - id: avg_freqresolution_concat
+        source: avg_freqresolution_concat
     out:
       - id: msout
+      - id: outh5parm
+      - id: bad_antennas
+      - id: outh5parm_logfile
       - id: logfiles
-    run: ./prefactor_target/concat.cwl
-    label: concat
+    run: ./prefactor_target/gsmcal.cwl
+    label: gsmcal
     'sbg:x': 500
     'sbg:y': 0
+  - id: finalize
+    in:
+      - id: msin
+        source: gsmcal/msout
+      - id: input_h5parm
+        source: gsmcal/outh5parm
+      - id: inh5parm_logfile
+        source: gsmcal/outh5parm_logfile
+      - id: gsmcal_step
+        source: gsmcal_step
+      - id: process_baselines_target
+        source: process_baselines_target
+      - id: bad_antennas
+        source: gsmcal/bad_antennas
+      - id: insolutions
+        source: prep/outh5parm
+      - id: compression_bitrate
+        source: compression_bitrate
+    out:
+      - id: msout
+      - id: solutions
+      - id: logfiles
+    run: ./prefactor_target/finalize.cwl
+    label: finalize
+    'sbg:x': 800
+    'sbg:y': 0
 requirements:
   - class: SubworkflowFeatureRequirement

workflows/prefactor_target/concat.cwl

-class: Workflow
-cwlVersion: v1.0
-id: concat
-label: concat
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
-inputs:
-  - id: msin
-    type: 'Directory[]'
-    'sbg:x': -1000
-    'sbg:y': -900
-  - id: filter_baselines
-    type: string
-    'sbg:x': -1000
-    'sbg:y': -800
-outputs:
-  - id: msout
-    outputSource:
-      - msin
-    type: 'Directory[]'
-    'sbg:x': 1000
-    'sbg:y': 0
-  - id: logfiles
-    linkMerge: merge_flattened
-    outputSource:
-      - identifybadantennas/logfile
-    type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 800
-steps:
-  - id: identifybadantennas
-    in:
-      - id: msin
-        source:
-          - msin
-      - id: filter
-        source: filter_baselines
-    out:
-      - id: filter_out
-      - id: logfile
-    run: ../../steps/identify_bad_antennas.cwl
-    label: identifybadantennas
-    'sbg:x': 0
-    'sbg:y': -300
-requirements:
-  - class: SubworkflowFeatureRequirement
-  - class: ScatterFeatureRequirement

workflows/prefactor_target/finalize.cwl

+class: Workflow
+cwlVersion: v1.0
+id: finalize
+label: finalize
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+inputs:
+  - id: msin
+    type: 'Directory[]'
+    'sbg:x': -1000
+    'sbg:y': -400
+  - id: input_h5parm
+    type: File
+    'sbg:x': -1000
+    'sbg:y': -300
+  - id: inh5parm_logfile
+    type: 'File[]'
+    'sbg:x': -1000
+    'sbg:y': -200
+  - id: gsmcal_step
+    type: string
+    'sbg:x': -1000
+    'sbg:y': -200
+  - id: process_baselines_target
+    type: string
+    'sbg:x': -1000
+    'sbg:y': -100
+  - id: bad_antennas
+    type: string
+    'sbg:x': -1000
+    'sbg:y': 0
+  - id: insolutions
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 100
+  - id: compression_bitrate
+    type: int
+    'sbg:x': -1000
+    'sbg:y': 200
+
+outputs:
+  - id: msout
+    outputSource:
+      - apply_gsmcal/msout
+    type: 'Directory[]'
+    'sbg:x': 1000
+    'sbg:y': 0
+  - id: solutions
+    outputSource:
+      - write_solutions/outh5parm
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 200
+  - id: logfiles
+    outputSource:
+      - concat_logfiles_applygsm/output
+      - concat_logfiles_solutions/output
+    type: 'File[]'
+    linkMerge: merge_flattened
+    'sbg:x': 1000
+    'sbg:y': 800
+steps:
+  - id: add_missing_stations
+    in:
+      - id: h5parm
+        source: input_h5parm
+      - id: solset
+        default: 'sol000'
+      - id: refsolset
+        default: 'target'
+      - id: refh5parm
+        source: insolutions
+      - id: soltab_in
+        source: gsmcal_step
+      - id: soltab_out
+        default: 'TGSSphase'
+      - id: filter
+        source: process_baselines_target
+      - id: bad_antennas
+        source: bad_antennas
+    out:
+      - id: outh5parm
+      - id: log
+    run: ../../steps/add_missing_stations.cwl
+    label: add_missing_stations
+    'sbg:x': -200
+    'sbg:y': -300
+
+  - id: apply_gsmcal
+    in:
+      - id: msin
+        source: msin
+      - id: msin_datacolumn
+        default: DATA
+      - id: parmdb
+        source: write_solutions/outh5parm
+      - id: msout_datacolumn
+        default: DATA
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        source: compression_bitrate
+      - id: correction
+        default: 'TGSSphase'
+      - id: solset
+        default: 'target'
+      - id: msout_name
+        source: msin
+        valueFrom: $(self.basename+'_prep.ms')
+    out:
+      - id: msout
+      - id: logfile
+    run: ../../lofar-cwl/steps/applytarget.cwl
+    label: apply_gsmcal
+    scatter:
+      - msin
+    'sbg:x': 400
+    'sbg:y': 0
+  - id: write_solutions
+    in:
+      - id: h5parmFile
+        source: add_missing_stations/outh5parm
+      - id: outsolset
+        default: target
+      - id: insoltab
+        default: 'TGSSphase'
+      - id: input_file
+        source: insolutions
+      - id: squeeze
+        default: true
+      - id: verbose
+        default: true
+      - id: history
+        default: true
+    out:
+      - id: outh5parm
+      - id: log
+    run: ../../steps/h5parmcat.cwl
+    label: write_solutions
+    'sbg:x': 200
+    'sbg:y': 500
+  - id: concat_logfiles_applygsm
+    in:
+      - id: file_list
+        source:
+          - apply_gsmcal/logfile
+      - id: file_prefix
+        default: apply_gsmcal
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_applygsm
+    'sbg:x': 750
+    'sbg:y': 500
+  - id: concat_logfiles_solutions
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - inh5parm_logfile
+          - add_missing_stations/log
+          - write_solutions/log
+      - id: file_prefix
+        default: losoto_gsmcal
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_solutions
+    'sbg:x': 500
+    'sbg:y': 500
+requirements:
+  - class: SubworkflowFeatureRequirement
+  - class: ScatterFeatureRequirement

workflows/prefactor_target/gsmcal.cwl

 class: Workflow
 cwlVersion: v1.0
-id: prep
-label: prep
+id: gsmcal
+label: gsmcal
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
   - id: msin
     type: 'Directory[]'
     'sbg:x': -1000
-    'sbg:y': -900
+    'sbg:y': -400
   - id: filter_baselines
     type: string
     'sbg:x': -1000
-    'sbg:y': -800
-  - id: raw_data
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -700
-  - id: propagatesolutions
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -600
-  - id: flagunconverged
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -500
-  - id: demix
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -400
-  - id: max_dppp_threads
-    type: int?
-    default: 10
-    'sbg:x': -1000
     'sbg:y': -300
-  - id: memoryperc
+  - id: num_SBs_per_group
     type: int
     'sbg:x': -1000
     'sbg:y': -200
-  - id: flag_baselines
-    type: 'string[]'
+  - id: reference_stationSB
+    type: int
+    default: null
     'sbg:x': -1000
     'sbg:y': -100
-  - id: avg_timeresolution
-    type: float
+  - id: use_target
+    type: boolean
     'sbg:x': -1000
     'sbg:y': 0
-  - id: avg_freqresolution
-    type: string
+  - id: target_skymodel
+    type: File?
     'sbg:x': -1000
     'sbg:y': 100
-  - id: process_baselines_cal
+  - id: skymodel_source
     type: string
     'sbg:x': -1000
     'sbg:y': 200
-  - id: demix_timestep
-    type: int
+  - id: do_smooth
+    type: boolean
     'sbg:x': -1000
     'sbg:y': 300
-  - id: demix_freqstep
-    type: int
+  - id: propagatesolutions
+    type: boolean
     'sbg:x': -1000
     'sbg:y': 400
-  - id: demix_target
-    type: string
+  - id: avg_timeresolution_concat
+    type: int
     'sbg:x': -1000
     'sbg:y': 500
-  - id: demix_sources
-    type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': 600
-  - id: min_length
-    type: int?
-    'sbg:x': -1000
-    'sbg:y': 700
-  - id: overhead
-    type: float?
-    'sbg:x': -1000
-    'sbg:y': 800
-  - id: min_separation
-    type: int?
-    'sbg:x': -1000
-    'sbg:y': 900
-  - id: do_smooth
-    type: boolean?
-    default: false
-    'sbg:x': -1000
-    'sbg:y': 1000
-  - id: max_separation_arcmin
-    type: float?
-    'sbg:x': -1000
-    'sbg:y': 1100
-  - id: calibrator_path_skymodel
-    type: Directory?
-    'sbg:x': -1000
-    'sbg:y': 1200
-  - id: A-Team_skymodel
-    type: File?
-    'sbg:x': -1000
-    'sbg:y': 1300
-  - id: elevation
+  - id: avg_freqresolution_concat
     type: string
-    default: 0deg..20deg
     'sbg:x': -1000
-    'sbg:y': 1400
-  - id: amplmin
-    type: float
-    default: 1.e-30
-    'sbg:x': -1000
-    'sbg:y': 1500
+    'sbg:y': 600
 outputs:
-  - id: outh5parm
-    outputSource:
-      - h5parm_collector/outh5parm
-    type: File
-    'sbg:x': 1000
-    'sbg:y': -500
-  - id: check_Ateam_separation.png
-    outputSource:
-      - check_ateam_separation/output_imag
-    type: File
-    'sbg:x': 1000
-    'sbg:y': -300
   - id: msout
     outputSource:
-      - predict_calibrate/msout
+      - concat_calibrate/msout
     type: 'Directory[]'
     'sbg:x': 1000
     'sbg:y': 0
-  - id: parset
+  - id: outh5parm
     outputSource:
-      - ndppp_prep_cal/parset
-    type: 'File[]'
+      - h5parm_collector/outh5parm
+    type: File
     'sbg:x': 1000
     'sbg:y': 200
-  - id: calibrator_name
+  - id: bad_antennas
     outputSource:
-      - find_skymodel_cal/model_name
+      - identifybadantennas/filter_out
     type: string
     'sbg:x': 1000
     'sbg:y': 400
@@ -145,224 +74,169 @@ outputs:
       - h5parm_collector/log
     type: File[]
     'sbg:x': 1000
-    'sbg:y': 650
+    'sbg:y': 600
   - id: logfiles
-    linkMerge: merge_flattened
     outputSource:
+      - identifybadantennas/logfile
+      - sort_times_into_freqGroups/logfile
+      - find_skymodel_target/logfile
+      - make_sourcedb_target/log
       - concat_logfiles_calib/output
-      - concat_logfiles_predict/output
+      - concat_logfiles_dpppconcat/output
       - concat_logfiles_blsmooth/output
-      - make_sourcedb/log
-      - find_skymodel_cal/logfile
-      - concat_logfiles_prep_cal/output
-      - make_sourcedb_ateam/log
-      - check_ateam_separation/logfile
     type: 'File[]'
+    linkMerge: merge_flattened
     'sbg:x': 1000
     'sbg:y': 800
+    
 steps:
-  - id: ndppp_prep_cal
+  - id: identifybadantennas
     in:
-      - id: baselines_to_flag
-        default: []
-        source:
-          - flag_baselines
-      - id: elevation_to_flag
-        source: elevation
-      - id: min_amplitude_to_flag
-        source: amplmin
-      - id: memoryperc
-        default: 20
-        source: memoryperc
-      - id: raw_data
-        default: false
-        source: raw_data
-      - id: demix
-        default: false
-        source: demix
       - id: msin
-        linkMerge: merge_flattened
         source:
           - msin
-      - id: msin_baseline
-        default: '*'
+      - id: filter
         source: filter_baselines
-      - id: skymodel
-        source: make_sourcedb_ateam/sourcedb
-      - id: timeresolution
-        default: 1
-        source: avg_timeresolution
-      - id: freqresolution
-        default: 12.21kHz
-        source: avg_freqresolution
-      - id: demix_timestep
-        default: 1
-        source: demix_timestep
-      - id: demix_freqstep
-        default: 1
-        source: demix_freqstep
-      - id: demix_baseline
-        default: '*'
-        source: process_baselines_cal
-      - id: target_source
-        source: demix_target
-      - id: ntimechunk
-        default: 10
-        source: max_dppp_threads
-      - id: subtract_sources
-        source:
-          - demix_sources
     out:
-      - id: msout
+      - id: filter_out
       - id: logfile
-      - id: parset
-    run: ../../subworkflow/ndppp_prep_cal.cwl
-    label: ndppp_prep_cal
-    scatter:
-      - msin
-    'sbg:x': 0
-    'sbg:y': 0
-  - id: predict_calibrate
+    run: ../../steps/identify_bad_antennas.cwl
+    label: identifybadantennas
+    'sbg:x': -200
+    'sbg:y': -300
+  - id: sort_times_into_freqGroups
     in:
       - id: msin
-        linkMerge: merge_flattened
         source:
-          - ndppp_prep_cal/msout
-      - id: do_smooth
-        source: do_smooth
-      - id: sourcedb
-        source: make_sourcedb/sourcedb
-      - id: flagunconverged
-        source: flagunconverged
-      - id: propagatesolutions
-        source: propagatesolutions
-    out:
-      - id: msout
-      - id: BLsmooth.log
-      - id: predict_cal.log
-      - id: calib_cal.log
-      - id: outh5parm
-    run: ../../subworkflow/predict_calibrate.cwl
-    label: predict_calibrate
-    scatter:
           - msin
-    'sbg:x': 500
-    'sbg:y': 0
-  - id: check_ateam_separation
-    in:
-      - id: ms
-        source:
-          - msin
-      - id: min_separation
-        source: min_separation
+      - id: numbands
+        source: num_SBs_per_group
+      - id: NDPPPfill
+        default: true
+      - id: stepname
+        default: dpppconcat
+      - id: firstSB
+        source: reference_stationSB
+      - id: truncateLastSBs
+        default: false
     out:
-      - id: output_imag
+      - id: filenames
+      - id: groupnames
       - id: logfile
-    run: ../../lofar-cwl/steps/check_ateam_separation.cwl
-    label: check_Ateam_separation
-    'sbg:x': 0
-    'sbg:y': -300
-  - id: make_sourcedb_ateam
-    in:
-      - id: sky_model
-        source: A-Team_skymodel
-      - id: logname
-        default: make_sourcedb_ateam.log
-    out:
-      - id: sourcedb
-      - id: log
-    run: ../../lofar-cwl/steps/makesourcedb.cwl
-    label: make_sourcedb_ateam
-    'sbg:x': -500
-    'sbg:y': 300
-  - id: find_skymodel_cal
+    run: ../../steps/sort_times_into_freqGroups.cwl
+    label: sorttimesintofreqGroups
+    'sbg:x': -200
+    'sbg:y': 0
+  - id: find_skymodel_target
     in:
       - id: msin
         source:
           - msin
-      - id: skymodels
-        source: calibrator_path_skymodel
-      - id: max_separation_arcmin
-        source: max_separation_arcmin
+      - id: SkymodelPath
+        source: target_skymodel
+      - id: Radius
+        default: 5
+      - id: Source
+        source: skymodel_source
+      - id: DoDownload
+        source: use_target
     out:
-      - id: output_models
-      - id: model_name
+      - id: skymodel
       - id: logfile
-    run: ../../steps/find_skymodel_cal.cwl
-    label: find_skymodel_cal
-    'sbg:x': -500
-    'sbg:y': 500
-  - id: make_sourcedb
+    run: ../../steps/find_skymodel_target.cwl
+    label: find_skymodel_target
+    'sbg:x': -200
+    'sbg:y': -500
+  - id: make_sourcedb_target
     in:
       - id: sky_model
-        source: find_skymodel_cal/output_models
-      - id: output_file_name
-        default: calibrator.sourcedb
+        source: find_skymodel_target/skymodel
+      - id: logname
+        default: make_sourcedb_target.log
     out:
       - id: sourcedb
       - id: log
     run: ../../lofar-cwl/steps/makesourcedb.cwl
-    label: make_sourcedb
+    label: make_sourcedb_target
     'sbg:x': 0
-    'sbg:y': 500
-  - id: concat_logfiles_prep_cal
+    'sbg:y': -500
+  - id: concat_logfiles_dpppconcat
     in:
       - id: file_list
-        linkMerge: merge_flattened
-        source: ndppp_prep_cal/logfile
+        source:
+          - concat_calibrate/dpppconcat.log
       - id: file_prefix
-        default: ndppp_prep_cal
+        default: dpppconcat
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_prep_cal
-    'sbg:x': 250
-    'sbg:y': 300
+    label: concat_logfiles_dpppconcat
+    'sbg:x': 750
+    'sbg:y': 100
   - id: concat_logfiles_blsmooth
     in:
       - id: file_list
         source:
-          - predict_calibrate/BLsmooth.log
+          - concat_calibrate/BLsmooth.log
       - id: file_prefix
-        default: blsmooth_prep
+        default: blsmooth_target
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_blsmooth
     'sbg:x': 750
     'sbg:y': 300
-  - id: concat_logfiles_predict
+  - id: concat_logfiles_calib
     in:
       - id: file_list
-        linkMerge: merge_flattened
         source:
-          - predict_calibrate/predict_cal.log
+          - concat_calibrate/gaincal.log
       - id: file_prefix
-        default: predict_cal
+        default: gaincal
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_predict
+    label: concat_logfiles_calib
     'sbg:x': 750
     'sbg:y': 500
-  - id: concat_logfiles_calib
+  - id: concat_calibrate
     in:
-      - id: file_list
+      - id: msin
         source:
-          - predict_calibrate/calib_cal.log
-      - id: file_prefix
-        default: calib_cal_prep
+          - msin
+      - id: group_id
+        source: sort_times_into_freqGroups/groupnames
+      - id: groups_specification
+        source: sort_times_into_freqGroups/filenames
+      - id: filter_baselines
+        source: identifybadantennas/filter_out
+      - id: skymodel
+        source: make_sourcedb_target/sourcedb
+      - id: do_smooth
+        source: do_smooth
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: avg_timeresolution_concat
+        source: avg_timeresolution_concat
+      - id: avg_freqresolution_concat
+        source: avg_freqresolution_concat
     out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_calib
-    'sbg:x': 750
-    'sbg:y': 800
+      - id: msout
+      - id: dpppconcat.log
+      - id: BLsmooth.log
+      - id: gaincal.log
+      - id: outh5parm
+    run: ../../subworkflow/concat_calibrate.cwl
+    label: concat_calibrate
+    scatter:
+      - group_id
+    'sbg:x': 200
+    'sbg:y': 0
   - id: h5parm_collector
     in:
       - id: h5parmFiles
         source:
-          - predict_calibrate/outh5parm
+          - concat_calibrate/outh5parm
       - id: squeeze
         default: true
       - id: verbose

workflows/prefactor_target/prep_gsmcal.cwl

-class: Workflow
-cwlVersion: v1.0
-id: prep
-label: prep
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
-inputs:
-  - id: msin
-    type: 'Directory[]'
-    'sbg:x': -1000
-    'sbg:y': -900
-  - id: filter_baselines
-    type: string
-    'sbg:x': -1000
-    'sbg:y': -800
-  - id: raw_data
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -700
-  - id: propagatesolutions
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -600
-  - id: flagunconverged
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -500
-  - id: demix
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -400
-  - id: max_dppp_threads
-    type: int?
-    default: 10
-    'sbg:x': -1000
-    'sbg:y': -300
-  - id: memoryperc
-    type: int
-    'sbg:x': -1000
-    'sbg:y': -200
-  - id: flag_baselines
-    type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': -100
-  - id: avg_timeresolution
-    type: float
-    'sbg:x': -1000
-    'sbg:y': 0
-  - id: avg_freqresolution
-    type: string
-    'sbg:x': -1000
-    'sbg:y': 100
-  - id: process_baselines_cal
-    type: string
-    'sbg:x': -1000
-    'sbg:y': 200
-  - id: demix_timestep
-    type: int
-    'sbg:x': -1000
-    'sbg:y': 300
-  - id: demix_freqstep
-    type: int
-    'sbg:x': -1000
-    'sbg:y': 400
-  - id: demix_target
-    type: string
-    'sbg:x': -1000
-    'sbg:y': 500
-  - id: demix_sources
-    type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': 600
-  - id: min_length
-    type: int?
-    'sbg:x': -1000
-    'sbg:y': 700
-  - id: overhead
-    type: float?
-    'sbg:x': -1000
-    'sbg:y': 800
-  - id: min_separation
-    type: int?
-    'sbg:x': -1000
-    'sbg:y': 900
-  - id: do_smooth
-    type: boolean?
-    default: false
-    'sbg:x': -1000
-    'sbg:y': 1000
-  - id: max_separation_arcmin
-    type: float?
-    'sbg:x': -1000
-    'sbg:y': 1100
-  - id: calibrator_path_skymodel
-    type: Directory?
-    'sbg:x': -1000
-    'sbg:y': 1200
-  - id: A-Team_skymodel
-    type: File?
-    'sbg:x': -1000
-    'sbg:y': 1300
-  - id: elevation
-    type: string
-    default: 0deg..20deg
-    'sbg:x': -1000
-    'sbg:y': 1400
-  - id: amplmin
-    type: float
-    default: 1.e-30
-    'sbg:x': -1000
-    'sbg:y': 1500
-outputs:
-  - id: outh5parm
-    outputSource:
-      - h5parm_collector/outh5parm
-    type: File
-    'sbg:x': 1000
-    'sbg:y': -500
-  - id: check_Ateam_separation.png
-    outputSource:
-      - check_ateam_separation/output_imag
-    type: File
-    'sbg:x': 1000
-    'sbg:y': -300
-  - id: msout
-    outputSource:
-      - predict_calibrate/msout
-    type: 'Directory[]'
-    'sbg:x': 1000
-    'sbg:y': 0
-  - id: parset
-    outputSource:
-      - ndppp_prep_cal/parset
-    type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 200
-  - id: calibrator_name
-    outputSource:
-      - find_skymodel_cal/model_name
-    type: string
-    'sbg:x': 1000
-    'sbg:y': 400
-  - id: outh5parm_logfile
-    outputSource:
-      - h5parm_collector/log
-    type: File[]
-    'sbg:x': 1000
-    'sbg:y': 650
-  - id: logfiles
-    linkMerge: merge_flattened
-    outputSource:
-      - concat_logfiles_calib/output
-      - concat_logfiles_predict/output
-      - concat_logfiles_blsmooth/output
-      - make_sourcedb/log
-      - find_skymodel_cal/logfile
-      - concat_logfiles_prep_cal/output
-      - make_sourcedb_ateam/log
-      - check_ateam_separation/logfile
-    type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 800
-steps:
-  - id: ndppp_prep_cal
-    in:
-      - id: baselines_to_flag
-        default: []
-        source:
-          - flag_baselines
-      - id: elevation_to_flag
-        source: elevation
-      - id: min_amplitude_to_flag
-        source: amplmin
-      - id: memoryperc
-        default: 20
-        source: memoryperc
-      - id: raw_data
-        default: false
-        source: raw_data
-      - id: demix
-        default: false
-        source: demix
-      - id: msin
-        linkMerge: merge_flattened
-        source:
-          - msin
-      - id: msin_baseline
-        default: '*'
-        source: filter_baselines
-      - id: skymodel
-        source: make_sourcedb_ateam/sourcedb
-      - id: timeresolution
-        default: 1
-        source: avg_timeresolution
-      - id: freqresolution
-        default: 12.21kHz
-        source: avg_freqresolution
-      - id: demix_timestep
-        default: 1
-        source: demix_timestep
-      - id: demix_freqstep
-        default: 1
-        source: demix_freqstep
-      - id: demix_baseline
-        default: '*'
-        source: process_baselines_cal
-      - id: target_source
-        source: demix_target
-      - id: ntimechunk
-        default: 10
-        source: max_dppp_threads
-      - id: subtract_sources
-        source:
-          - demix_sources
-    out:
-      - id: msout
-      - id: logfile
-      - id: parset
-    run: ../../subworkflow/ndppp_prep_cal.cwl
-    label: ndppp_prep_cal
-    scatter:
-      - msin
-    'sbg:x': 0
-    'sbg:y': 0
-  - id: predict_calibrate
-    in:
-      - id: msin
-        linkMerge: merge_flattened
-        source:
-          - ndppp_prep_cal/msout
-      - id: do_smooth
-        source: do_smooth
-      - id: sourcedb
-        source: make_sourcedb/sourcedb
-      - id: flagunconverged
-        source: flagunconverged
-      - id: propagatesolutions
-        source: propagatesolutions
-    out:
-      - id: msout
-      - id: BLsmooth.log
-      - id: predict_cal.log
-      - id: calib_cal.log
-      - id: outh5parm
-    run: ../../subworkflow/predict_calibrate.cwl
-    label: predict_calibrate
-    scatter:
-      - msin
-    'sbg:x': 500
-    'sbg:y': 0
-  - id: check_ateam_separation
-    in:
-      - id: ms
-        source:
-          - msin
-      - id: min_separation
-        source: min_separation
-    out:
-      - id: output_imag
-      - id: logfile
-    run: ../../lofar-cwl/steps/check_ateam_separation.cwl
-    label: check_Ateam_separation
-    'sbg:x': 0
-    'sbg:y': -300
-  - id: make_sourcedb_ateam
-    in:
-      - id: sky_model
-        source: A-Team_skymodel
-      - id: logname
-        default: make_sourcedb_ateam.log
-    out:
-      - id: sourcedb
-      - id: log
-    run: ../../lofar-cwl/steps/makesourcedb.cwl
-    label: make_sourcedb_ateam
-    'sbg:x': -500
-    'sbg:y': 300
-  - id: find_skymodel_cal
-    in:
-      - id: msin
-        source:
-          - msin
-      - id: skymodels
-        source: calibrator_path_skymodel
-      - id: max_separation_arcmin
-        source: max_separation_arcmin
-    out:
-      - id: output_models
-      - id: model_name
-      - id: logfile
-    run: ../../steps/find_skymodel_cal.cwl
-    label: find_skymodel_cal
-    'sbg:x': -500
-    'sbg:y': 500
-  - id: make_sourcedb
-    in:
-      - id: sky_model
-        source: find_skymodel_cal/output_models
-      - id: output_file_name
-        default: calibrator.sourcedb
-    out:
-      - id: sourcedb
-      - id: log
-    run: ../../lofar-cwl/steps/makesourcedb.cwl
-    label: make_sourcedb
-    'sbg:x': 0
-    'sbg:y': 500
-  - id: concat_logfiles_prep_cal
-    in:
-      - id: file_list
-        linkMerge: merge_flattened
-        source: ndppp_prep_cal/logfile
-      - id: file_prefix
-        default: ndppp_prep_cal
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_prep_cal
-    'sbg:x': 250
-    'sbg:y': 300
-  - id: concat_logfiles_blsmooth
-    in:
-      - id: file_list
-        source:
-          - predict_calibrate/BLsmooth.log
-      - id: file_prefix
-        default: blsmooth_prep
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_blsmooth
-    'sbg:x': 750
-    'sbg:y': 300
-  - id: concat_logfiles_predict
-    in:
-      - id: file_list
-        linkMerge: merge_flattened
-        source:
-          - predict_calibrate/predict_cal.log
-      - id: file_prefix
-        default: predict_cal
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_predict
-    'sbg:x': 750
-    'sbg:y': 500
-  - id: concat_logfiles_calib
-    in:
-      - id: file_list
-        source:
-          - predict_calibrate/calib_cal.log
-      - id: file_prefix
-        default: calib_cal_prep
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_calib
-    'sbg:x': 750
-    'sbg:y': 800
-  - id: h5parm_collector
-    in:
-      - id: h5parmFiles
-        source:
-          - predict_calibrate/outh5parm
-      - id: squeeze
-        default: true
-      - id: verbose
-        default: true
-      - id: clobber
-        default: true
-    out:
-      - id: outh5parm
-      - id: log
-    run: ../../lofar-cwl/steps/H5ParmCollector.cwl
-    label: H5parm_collector
-    'sbg:x': 750
-    'sbg:y': -300
-requirements:
-  - class: SubworkflowFeatureRequirement
-  - class: ScatterFeatureRequirement