add target pipeline

steps/compare_station_list.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: compare_station_list
+baseCommand:
+  - python3
+inputs:
+    - id: msin
+      type: Directory[]
+      doc: MS to compare with
+      inputBinding:
+        position: 0
+    - id: h5parmdb
+      type: File
+      doc: H5parm database to compare with
+    - id: solset_name
+      type: string?
+      doc: Name of the H5parm solset
+      default: 'sol000'
+    - id: filter
+      type: string?
+      default: '*&'
+      doc: Filter these baselines for the comparison
+
+label: compareStationList.py
+arguments:
+  - '-c'
+  - |
+    import sys
+    import json
+    from compareStationList import main as compareStationList
+    
+    mss = sys.argv[1:]
+    h5parmdb = $(inputs.h5parmdb)['path']
+    solset_name = '$(inputs.solset_name)'
+    filter = '$(inputs.filter)'
+
+    output = compareStationList(mss, h5parmdb, solset_name, filter)
+
+    filter_out = output['filter']
+    cwl_output = {"filter_out": filter_out}
+
+    with open('./out.json', 'w') as fp:
+        json.dump(cwl_output, fp)
+outputs:
+  - id: filter_out
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: 'out.json'
+        outputEval: $(JSON.parse(self[0].contents).filter_out)
+  - id: logfile
+    type: File?
+    outputBinding:
+      glob: compareStationList.log    
+        
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  DockerRequirement:
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: compareStationList.log
\ No newline at end of file

steps/createRMh5parm.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+id: createRMh5parm
+label: createRMh5parm
+baseCommand:
+  - python3
+  - /usr/local/bin/createRMh5parm.py
+inputs:
+  - id: msin
+    type:
+      - Directory
+      - type: array
+        items: Directory
+    inputBinding:
+      position: 1
+    doc: Input measurement set
+  - id: h5parm
+    type: File
+    inputBinding:
+      position: 2
+    doc: Input h5parm solutions file
+  - id: ionex_server
+    type: string
+    doc: IONEX Server path
+    default: 'ftp://ftp.aiub.unibe.ch/CODE/'
+    inputBinding:
+      prefix: '--server'
+      position: 0
+  - id: ionex_prefix
+    type: string
+    doc: IONEX Prefix
+    default: 'CODG'
+    inputBinding:
+      prefix: '--prefix'
+      position: 0
+  - id: ionex_path
+    type: string
+    doc: IONEX path
+    default: './'
+    inputBinding:
+      prefix: '--path'
+      position: 0
+  - id: solset
+    type: string
+    doc: solset in which IONEX solution are put
+    default: 'sol000'
+    inputBinding:
+      prefix: '--solsetName'
+      position: 0
+  - id: timestep
+    type: float
+    doc: timestep in seconds
+    default: 300.
+    inputBinding:
+      prefix: '-t'
+      position: 0
+  - id: smart_interpol
+    type: float
+    doc: float parameter describing how much of Earth rotation is taken in to account in interpolation of the IONEX files. 1.0 means time interpolation assumes ionosphere rotates in opposite direction of the Earth
+    default: 0.
+    inputBinding:
+      prefix: '-e'
+      position: 0
+  - id: proxyserver
+    type: string
+    doc: Specify proxy server if necessary
+    default: null
+    inputBinding:
+      prefix: '--proxyserver'
+      position: 0
+  - id: proxyport
+    type: int
+    doc: Specify proxy port if necessary
+    default: null
+    inputBinding:
+      prefix: '--proxyport'
+      position: 0
+  - id: proxytype
+    type: string
+    doc: Specify proxy type if necessary
+    default: null
+    inputBinding:
+      prefix: '--proxytype'
+      position: 0
+  - id: proxyuser
+    type: string
+    doc: Specify proxy user name if necessary
+    default: null
+    inputBinding:
+      prefix: '--proxyuser'
+      position: 0
+  - id: proxypass
+    type: string
+    doc: Specify proxy server password if necessary
+    default: null
+    inputBinding:
+      prefix: '--proxypass'
+      position: 0
+outputs:
+  - id: h5parmout
+    doc: h5parm output
+    type: File
+    format: 'lofar:#H5Parm'
+    outputBinding:
+      glob: $(inputs.h5parm.basename)
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'createh5parm*.log'
+hints:
+ - class: DockerRequirement
+   dockerPull: lofareosc/prefactor:HBAcalibrator
+ - class: InitialWorkDirRequirement
+   listing:
+     - entry: $(inputs.h5parm)
+       writable: true
+   
+stdout: createh5parm.log
+stderr: createh5parm_err.log
+$namespaces:
+  lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+  - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

steps/h5parmcat.cwl

@@ -66,9 +66,9 @@ outputs:
       glob: $(inputs.input_file.basename)
     format: lofar:#H5Parm
   - id: log
-    type: File
+    type: File[]
     outputBinding:
-      glob: parm_collector_output.log
+      glob: 'parm_collector_output*.log'
 label: h5parm_cat
 arguments:
   - position: 0
@@ -84,5 +84,6 @@ hints:
   - class: DockerRequirement
     dockerPull: lofareosc/prefactor:HBAcalibrator
 stdout: parm_collector_output.log
+stderr: parm_collector_output_err.log
 $schema:
   - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'

workflows/HBA_target.cwl

@@ -179,8 +179,10 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1700
   - id: reference_stationSB
-    type: int?
-    default: None
+    type: 
+      - int?
+      - string?
+    default: ''
     'sbg:x': -1000
     'sbg:y': 1800
   - id: ionex_server
@@ -195,27 +197,27 @@ inputs:
     'sbg:y': 2000
   - id: proxy_server
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2100
   - id: proxy_port
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2200
   - id: proxy_type
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2300
   - id: proxy_user
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2400
   - id: proxy_pass
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2500
 outputs:
@@ -285,8 +287,6 @@ steps:
         source: overhead
       - id: min_separation
         source: min_separation
-      - id: calibrator_path_skymodel
-        source: calibrator_path_skymodel
       - id: A-Team_skymodel
         source: A-Team_skymodel
       - id: target_skymodel
@@ -324,7 +324,6 @@ steps:
     out:
       - id: logfiles
       - id: msout
-      - id: parset
       - id: solutions
       - id: inspection
     run: ./prefactor_target.cwl

workflows/prefactor_target.cwl

@@ -179,8 +179,10 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1700
   - id: reference_stationSB
-    type: int?
-    default: None
+    type: 
+      - int?
+      - string?
+    default: ''
     'sbg:x': -1000
     'sbg:y': 1800
   - id: ionex_server
@@ -195,35 +197,34 @@ inputs:
     'sbg:y': 2000
   - id: proxy_server
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2100
   - id: proxy_port
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2200
   - id: proxy_type
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2300
   - id: proxy_user
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2400
   - id: proxy_pass
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 2500
 outputs:
   - id: inspection
     linkMerge: merge_flattened
     outputSource:
-      - prep/check_Ateam_separation.png
-      - prep/RMextract.png
+      - prep/inspection
     type: File[]
     'sbg:x': 2000
     'sbg:y': -600
@@ -266,8 +267,8 @@ steps:
         source: demix
       - id: apply_steps
         source: apply_steps
-      - id: clipAteam
-        source: clipAteam
+      - id: clipATeam
+        source: clipATeam
       - id: max_dppp_threads
         source: max_dppp_threads
       - id: memoryperc
@@ -311,9 +312,8 @@ steps:
     out:
       - id: outh5parm
       - id: logfiles
-      - id: check_Ateam_separation.png
+      - id: inspection
       - id: msout
-      - id: target_name
     run: ./prefactor_target/prep.cwl
     label: prep
     'sbg:x': 0

workflows/prefactor_target/prep.cwl

@@ -10,7 +10,7 @@ inputs:
     'sbg:x': -1000
     'sbg:y': -1700
   - id: cal_solutions
-    type: 'File'
+    type: File
     'sbg:x': -1000
     'sbg:y': -1600
   - id: flag_baselines
@@ -106,7 +106,7 @@ inputs:
     'sbg:y': 200
   - id: avg_timeresolution
     type: float?
-    default: 4.
+    default: 4
     'sbg:x': -1000
     'sbg:y': 300
   - id: avg_freqresolution
@@ -121,147 +121,67 @@ inputs:
     'sbg:y': 500
   - id: ionex_prefix
     type: string?
-    default: 'CODG'
+    default: CODG
     'sbg:x': -1000
     'sbg:y': 600
   - id: proxy_server
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 700
   - id: proxy_port
-    type: string?
-    default: ''
+    type: int?
+    default: null
     'sbg:x': -1000
     'sbg:y': 800
   - id: proxy_type
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 900
   - id: proxy_user
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 1000
   - id: proxy_pass
     type: string?
-    default: ''
+    default: null
     'sbg:x': -1000
     'sbg:y': 1100
 outputs:
   - id: outh5parm
     outputSource:
-      - h5parm_collector/outh5parm
+      - createRMh5parm/h5parmout
     type: File
     'sbg:x': 1000
     'sbg:y': -500
-  - id: check_Ateam_separation.png
+  - id: inspection
     outputSource:
       - check_ateam_separation/output_imag
-    type: File
+      - losoto_plot_RM/output_plots
+    type: 'File[]?'
+    linkMerge: merge_flattened
     'sbg:x': 1000
     'sbg:y': -300
   - id: msout
     outputSource:
-      - predict_calibrate/msout
+      - msin
     type: 'Directory[]'
     'sbg:x': 1000
     'sbg:y': 0
-  - id: parset
-    outputSource:
-      - ndppp_prep_cal/parset
-    type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 200
-  - id: calibrator_name
-    outputSource:
-      - find_skymodel_cal/model_name
-    type: string
-    'sbg:x': 1000
-    'sbg:y': 400
-  - id: outh5parm_logfile
-    outputSource:
-      - h5parm_collector/log
-    type: File[]
-    'sbg:x': 1000
-    'sbg:y': 650
   - id: logfiles
-    linkMerge: merge_flattened
     outputSource:
-      - concat_logfiles_calib/output
-      - concat_logfiles_predict/output
-      - concat_logfiles_blsmooth/output
-      - make_sourcedb/log
-      - find_skymodel_cal/logfile
-      - concat_logfiles_prep_cal/output
       - make_sourcedb_ateam/log
       - check_ateam_separation/logfile
+      - createRMh5parm/logfile
+      - losoto_plot_RM/logfile
+      - compare_station_list/logfile
     type: 'File[]'
+    linkMerge: merge_flattened
     'sbg:x': 1000
     'sbg:y': 800
 steps:
-  - id: ndppp_prep_target
-    in:
-      - id: baselines_to_flag
-        default: []
-        source:
-          - flag_baselines
-      - id: elevation_to_flag
-        source: elevation
-      - id: min_amplitude_to_flag
-        source: amplmin
-      - id: memoryperc
-        default: 20
-        source: memoryperc
-      - id: raw_data
-        default: false
-        source: raw_data
-      - id: demix
-        default: false
-        source: demix
-      - id: msin
-        linkMerge: merge_flattened
-        source:
-          - msin
-      - id: msin_baseline
-        default: '*'
-        source: filter_baselines
-      - id: skymodel
-        source: make_sourcedb_ateam/sourcedb
-      - id: timeresolution
-        default: 1
-        source: avg_timeresolution
-      - id: freqresolution
-        default: 12.21kHz
-        source: avg_freqresolution
-      - id: demix_timestep
-        default: 1
-        source: demix_timestep
-      - id: demix_freqstep
-        default: 1
-        source: demix_freqstep
-      - id: demix_baseline
-        default: '*'
-        source: process_baselines_cal
-      - id: target_source
-        source: demix_target
-      - id: ntimechunk
-        default: 10
-        source: max_dppp_threads
-      - id: subtract_sources
-        source:
-          - demix_sources
-    out:
-      - id: msout
-      - id: logfile
-      - id: parset
-    run: ../../subworkflow/ndppp_prep_cal.cwl
-    label: ndppp_prep_target
-    scatter:
-      - msin
-    'sbg:x': 0
-    'sbg:y': 0
   - id: check_ateam_separation
     in:
       - id: ms
@@ -274,8 +194,26 @@ steps:
       - id: logfile
     run: ../../lofar-cwl/steps/check_ateam_separation.cwl
     label: check_Ateam_separation
-    'sbg:x': 0
+    'sbg:x': -500
     'sbg:y': -300
+  - id: compare_station_list
+    in:
+      - id: msin
+        source:
+          - msin
+      - id: h5parmdb
+        source: cal_solutions
+      - id: solset_name
+        default: 'calibrator'
+      - id: filter
+        source: filter_baselines
+    out:
+      - id: filter_out
+      - id: logfile
+    run: ../../steps/compare_station_list.cwl
+    label: compare_station_list
+    'sbg:x': -500
+    'sbg:y': -500
   - id: make_sourcedb_ateam
     in:
       - id: sky_model
@@ -289,19 +227,55 @@ steps:
     label: make_sourcedb_ateam
     'sbg:x': -500
     'sbg:y': 300
-  - id: concat_logfiles_prep_target
+  - id: createRMh5parm
     in:
-      - id: file_list
-        linkMerge: merge_flattened
-        source: ndppp_prep_cal/logfile
-      - id: file_prefix
-        default: ndppp_prep_cal
+      - id: msin
+        source:
+          - msin
+      - id: h5parm
+        source: cal_solutions
+      - id: ionex_server
+        source: ionex_server
+      - id: ionex_prefix
+        source: ionex_prefix
+      - id: solset
+        default: target
+      - id: proxyserver
+        source: proxy_server
+      - id: proxyport
+        source: proxy_port
+      - id: proxytype
+        source: proxy_type
+      - id: proxyuser
+        source: proxy_user
+      - id: proxypass
+        source: proxy_pass
     out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_prep_target
-    'sbg:x': 250
-    'sbg:y': 300
-requirements:
-  - class: SubworkflowFeatureRequirement
-  - class: ScatterFeatureRequirement
+      - id: h5parmout
+      - id: logfile
+    run: ../../steps/createRMh5parm.cwl
+    label: createRMh5parm
+    'sbg:x': -500
+    'sbg:y': 0
+  - id: losoto_plot_RM
+    in:
+      - id: input_h5parm
+        source: createRMh5parm/h5parmout
+      - id: soltab
+        default: target/RMextract
+      - id: axesInPlot
+        default:
+          - time
+      - id: axisInTable
+        default: ant
+      - id: prefix
+        default: RMextract
+    out:
+      - id: output_plots
+      - id: logfile
+      - id: parset
+    run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
+    label: losoto_plot_RM
+    'sbg:x': -300
+    'sbg:y': 0
+requirements: []