add identify_bad_antennas

Former-commit-id: f53880fa

lofar-cwl/steps/Ateamclipper.cwl

@@ -20,11 +20,15 @@ outputs:
     doc: Output MS
     type: Directory
     outputBinding:
-      glob: $(inputs.msin)
+      glob: $(inputs.msin.basename)
   - id: logfile
     type: 'File[]'
     outputBinding:
       glob: Ateamclipper.log
+  - id: output
+    type: File
+    outputBinding:
+      glob: Ateamclipper.txt
 label: Ateamclipper
 hints:
   - class: InitialWorkDirRequirement

lofar-cwl/steps/filter_predict.cwl

@@ -42,7 +42,7 @@ inputs:
       prefix: predict.sources=
       separate: false
       itemSeparator: ','
-      valueFrom: '[$(self)]'
+      valueFrom: "[$(self.join(','))]"
   - default: false
     id: usebeammodel
     type: boolean
@@ -99,7 +99,7 @@ outputs:
   - id: logfile
     type: File[]
     outputBinding:
-      glob: 'predict_cal*.log'
+      glob: 'filter_predict*.log'
 arguments:
   - 'steps=[filter,predict]'
   - predict.beammode=array_factor

steps/identify_bad_antennas.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: identify_bad_antennas
+baseCommand:
+  - python3
+inputs:
+    - id: msin
+      type: Directory[]
+      doc: MS to compare with
+      inputBinding:
+        position: 0
+    - id: filter
+      type: string?
+      default: '*&'
+      doc: Filter these baselines for the comparison
+
+label: identifyBadAntennas.py
+arguments:
+  - '-c'
+  - |
+    import sys
+    import json
+    from identifyBadAntennas import main as identifyBadAntennas
+    
+    mss = sys.argv[1:]
+    filter = '$(inputs.filter)'
+
+    output = identifyBadAntennas(mss, filter)
+
+    filter_out = output['filter']
+    cwl_output = {"filter_out": filter_out}
+
+    with open('./out.json', 'w') as fp:
+        json.dump(cwl_output, fp)
+outputs:
+  - id: filter_out
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: 'out.json'
+        outputEval: $(JSON.parse(self[0].contents).filter_out)
+  - id: logfile
+    type: File?
+    outputBinding:
+      glob: identifyBadAntennas.log    
+        
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: identifyBadAntennas.log
\ No newline at end of file

steps/plot_Ateamclipper.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: plot_Ateamclipper
+baseCommand:
+  - python3
+  - /usr/local/bin/plot_Ateamclipper.py
+inputs:
+  - id: clipper_output
+    type: File
+    inputBinding:
+      position: 1
+  - id: outputimage
+    type: string?
+    default: Ateamclipper.png
+    inputBinding:
+      position: 2
+outputs:
+  - id: output_imag
+    doc: Output image
+    type: File
+    outputBinding:
+      glob: $(inputs.outputimage)
+label: plot_Ateamclipper
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator

subworkflow/ndppp_prep_targ.cwl

@@ -126,6 +126,12 @@ outputs:
     type: Directory
     'sbg:x': 5502.318359375
     'sbg:y': -279.49835205078125
+  - id: clipper_output
+    outputSource:
+      - Ateamclipper/output
+    type: File
+    'sbg:x': 5502.318359375
+    'sbg:y': -500
   - id: parset
     outputSource:
       - dp3_execute/parset
@@ -435,10 +441,10 @@ steps:
         source: process_baselines_target
       - id: sources
         default: 
-          - 'VirA_4_patch'
-          - 'CygAGG'
-          - 'CasA_4_patch'
-          - 'TauAGG'
+          - VirA_4_patch
+          - CygAGG
+          - CasA_4_patch
+          - TauAGG
       - id: usebeammodel
         default: true
       - id: onebeamperpatch
@@ -456,6 +462,7 @@ steps:
     out:
       - id: msout
       - id: logfile
+      - id: output
     run: ../lofar-cwl/steps/Ateamclipper.cwl
     'sbg:x': 4700
     'sbg:y': -344.5472106933594

workflows/prefactor_target.cwl

@@ -10,7 +10,7 @@ inputs:
     'sbg:x': -1000
     'sbg:y': -1700
   - id: cal_solutions
-    type: 'File'
+    type: File
     'sbg:x': -1000
     'sbg:y': -1600
   - id: refant
@@ -122,7 +122,7 @@ inputs:
     'sbg:y': 500
   - id: gsmcal_step
     type: string?
-    default: 'phase'
+    default: phase
     'sbg:x': -1000
     'sbg:y': 600
   - id: updateweights
@@ -170,12 +170,12 @@ inputs:
     'sbg:y': 1500
   - id: skymodel_source
     type: string?
-    default: 'TGSS'
+    default: TGSS
     'sbg:x': -1000
     'sbg:y': 1600
   - id: avg_timeresolution
     type: float?
-    default: 4.
+    default: 4
     'sbg:x': -1000
     'sbg:y': 1700
   - id: avg_freqresolution
@@ -185,7 +185,7 @@ inputs:
     'sbg:y': 1800
   - id: avg_timeresolution_concat
     type: float?
-    default: 8.
+    default: 8
     'sbg:x': -1000
     'sbg:y': 1900
   - id: avg_freqresolution_concat
@@ -199,9 +199,7 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 2100
   - id: reference_stationSB
-    type: 
-      - int?
-      - string?
+    type: string?
     default: ''
     'sbg:x': -1000
     'sbg:y': 1800
@@ -212,7 +210,7 @@ inputs:
     'sbg:y': 1900
   - id: ionex_prefix
     type: string?
-    default: 'CODG'
+    default: CODG
     'sbg:x': -1000
     'sbg:y': 2000
   - id: proxy_server
@@ -242,10 +240,10 @@ inputs:
     'sbg:y': 2500
 outputs:
   - id: inspection
-    linkMerge: merge_flattened
     outputSource:
       - prep/inspection
-    type: File[]
+    type: 'File[]'
+    linkMerge: merge_flattened
     'sbg:x': 2000
     'sbg:y': -600
   - id: solutions
@@ -256,13 +254,14 @@ outputs:
     'sbg:y': -500
   - id: msout
     outputSource:
-      - prep/msout
+      - concat/msout
     type: 'Directory[]'
     'sbg:x': 2000
     'sbg:y': 0
   - id: logfiles
     outputSource:
       - prep/logfiles
+      - concat/logfiles
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 2000
@@ -275,14 +274,26 @@ steps:
           - msin
       - id: cal_solutions
         source: cal_solutions
+      - id: flag_baselines
+        source:
+          - flag_baselines
+      - id: process_baselines_target
+        source: process_baselines_target
       - id: filter_baselines
         source: filter_baselines
       - id: raw_data
         source: raw_data
       - id: propagatesolutions
         source: propagatesolutions
-      - id: updateweights
-        source: updateweights
+      - id: demix_sources
+        source:
+          - demix_sources
+      - id: demix_target
+        source: demix_target
+      - id: demix_freqstep
+        source: demix_freqstep
+      - id: demix_timestep
+        source: demix_timestep
       - id: demix
         source: demix
       - id: apply_tec
@@ -297,32 +308,20 @@ steps:
         source: apply_beam
       - id: clipATeam
         source: clipATeam
+      - id: updateweights
+        source: updateweights
       - id: max_dppp_threads
         source: max_dppp_threads
       - id: memoryperc
         source: memoryperc
-      - id: flag_baselines
-        source:
-          - flag_baselines
-      - id: avg_timeresolution
-        source: avg_timeresolution
-      - id: avg_freqresolution
-        source: avg_freqresolution
-      - id: process_baselines_target
-        source: process_baselines_target
-      - id: demix_timestep
-        source: demix_timestep
-      - id: demix_freqstep
-        source: demix_freqstep
-      - id: demix_target
-        source: demix_target
-      - id: demix_sources
-        source:
-          - demix_sources
       - id: min_separation
         source: min_separation
       - id: A-Team_skymodel
         source: A-Team_skymodel
+      - id: avg_timeresolution
+        source: avg_timeresolution
+      - id: avg_freqresolution
+        source: avg_freqresolution
       - id: ionex_server
         source: ionex_server
       - id: ionex_prefix
@@ -339,12 +338,26 @@ steps:
         source: proxy_pass
     out:
       - id: outh5parm
-      - id: logfiles
       - id: inspection
       - id: msout
+      - id: logfiles
     run: ./prefactor_target/prep.cwl
     label: prep
     'sbg:x': 0
     'sbg:y': 0
+  - id: concat
+    in:
+      - id: msin
+        source:
+          - prep/msout
+      - id: filter_baselines
+        source: process_baselines_target
+    out:
+      - id: msout
+      - id: logfiles
+    run: ./prefactor_target/concat.cwl
+    label: concat
+    'sbg:x': 500
+    'sbg:y': 0
 requirements:
   - class: SubworkflowFeatureRequirement

workflows/prefactor_target/concat.cwl

 class: Workflow
 cwlVersion: v1.0
-id: prep
-label: prep
+id: concat
+label: concat
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
@@ -13,368 +13,34 @@ inputs:
     type: string
     'sbg:x': -1000
     'sbg:y': -800
-  - id: raw_data
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -700
-  - id: propagatesolutions
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -600
-  - id: flagunconverged
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -500
-  - id: demix
-    type: boolean
-    'sbg:x': -1000
-    'sbg:y': -400
-  - id: max_dppp_threads
-    type: int?
-    default: 10
-    'sbg:x': -1000
-    'sbg:y': -300
-  - id: memoryperc
-    type: int
-    'sbg:x': -1000
-    'sbg:y': -200
-  - id: flag_baselines
-    type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': -100
-  - id: avg_timeresolution
-    type: float
-    'sbg:x': -1000
-    'sbg:y': 0
-  - id: avg_freqresolution
-    type: string
-    'sbg:x': -1000
-    'sbg:y': 100
-  - id: process_baselines_cal
-    type: string
-    'sbg:x': -1000
-    'sbg:y': 200
-  - id: demix_timestep
-    type: int
-    'sbg:x': -1000
-    'sbg:y': 300
-  - id: demix_freqstep
-    type: int
-    'sbg:x': -1000
-    'sbg:y': 400
-  - id: demix_target
-    type: string
-    'sbg:x': -1000
-    'sbg:y': 500
-  - id: demix_sources
-    type: 'string[]'
-    'sbg:x': -1000
-    'sbg:y': 600
-  - id: min_length
-    type: int?
-    'sbg:x': -1000
-    'sbg:y': 700
-  - id: overhead
-    type: float?
-    'sbg:x': -1000
-    'sbg:y': 800
-  - id: min_separation
-    type: int?
-    'sbg:x': -1000
-    'sbg:y': 900
-  - id: do_smooth
-    type: boolean?
-    default: false
-    'sbg:x': -1000
-    'sbg:y': 1000
-  - id: max_separation_arcmin
-    type: float?
-    'sbg:x': -1000
-    'sbg:y': 1100
-  - id: calibrator_path_skymodel
-    type: Directory?
-    'sbg:x': -1000
-    'sbg:y': 1200
-  - id: A-Team_skymodel
-    type: File?
-    'sbg:x': -1000
-    'sbg:y': 1300
-  - id: elevation
-    type: string
-    default: 0deg..20deg
-    'sbg:x': -1000
-    'sbg:y': 1400
-  - id: amplmin
-    type: float
-    default: 1.e-30
-    'sbg:x': -1000
-    'sbg:y': 1500
 outputs:
-  - id: outh5parm
-    outputSource:
-      - h5parm_collector/outh5parm
-    type: File
-    'sbg:x': 1000
-    'sbg:y': -500
-  - id: check_Ateam_separation.png
-    outputSource:
-      - check_ateam_separation/output_imag
-    type: File
-    'sbg:x': 1000
-    'sbg:y': -300
   - id: msout
     outputSource:
-      - predict_calibrate/msout
+      - msin
     type: 'Directory[]'
     'sbg:x': 1000
     'sbg:y': 0
-  - id: parset
-    outputSource:
-      - ndppp_prep_cal/parset
-    type: 'File[]'
-    'sbg:x': 1000
-    'sbg:y': 200
-  - id: calibrator_name
-    outputSource:
-      - find_skymodel_cal/model_name
-    type: string
-    'sbg:x': 1000
-    'sbg:y': 400
-  - id: outh5parm_logfile
-    outputSource:
-      - h5parm_collector/log
-    type: File[]
-    'sbg:x': 1000
-    'sbg:y': 650
   - id: logfiles
     linkMerge: merge_flattened
     outputSource:
-      - concat_logfiles_calib/output
-      - concat_logfiles_predict/output
-      - concat_logfiles_blsmooth/output
-      - make_sourcedb/log
-      - find_skymodel_cal/logfile
-      - concat_logfiles_prep_cal/output
-      - make_sourcedb_ateam/log
-      - check_ateam_separation/logfile
+      - identifybadantennas/logfile
     type: 'File[]'
     'sbg:x': 1000
     'sbg:y': 800
 steps:
-  - id: ndppp_prep_cal
+  - id: identifybadantennas
     in:
-      - id: baselines_to_flag
-        default: []
-        source:
-          - flag_baselines
-      - id: elevation_to_flag
-        source: elevation
-      - id: min_amplitude_to_flag
-        source: amplmin
-      - id: memoryperc
-        default: 20
-        source: memoryperc
-      - id: raw_data
-        default: false
-        source: raw_data
-      - id: demix
-        default: false
-        source: demix
       - id: msin
-        linkMerge: merge_flattened
         source:
           - msin
-      - id: msin_baseline
-        default: '*'
+      - id: filter
         source: filter_baselines
-      - id: skymodel
-        source: make_sourcedb_ateam/sourcedb
-      - id: timeresolution
-        default: 1
-        source: avg_timeresolution
-      - id: freqresolution
-        default: 12.21kHz
-        source: avg_freqresolution
-      - id: demix_timestep
-        default: 1
-        source: demix_timestep
-      - id: demix_freqstep
-        default: 1
-        source: demix_freqstep
-      - id: demix_baseline
-        default: '*'
-        source: process_baselines_cal
-      - id: target_source
-        source: demix_target
-      - id: ntimechunk
-        default: 10
-        source: max_dppp_threads
-      - id: subtract_sources
-        source:
-          - demix_sources
     out:
-      - id: msout
-      - id: logfile
-      - id: parset
-    run: ../../subworkflow/ndppp_prep_cal.cwl
-    label: ndppp_prep_cal
-    scatter:
-      - msin
-    'sbg:x': 0
-    'sbg:y': 0
-  - id: predict_calibrate
-    in:
-      - id: msin
-        linkMerge: merge_flattened
-        source:
-          - ndppp_prep_cal/msout
-      - id: do_smooth
-        source: do_smooth
-      - id: sourcedb
-        source: make_sourcedb/sourcedb
-      - id: flagunconverged
-        source: flagunconverged
-      - id: propagatesolutions
-        source: propagatesolutions
-    out:
-      - id: msout
-      - id: BLsmooth.log
-      - id: predict_cal.log
-      - id: calib_cal.log
-      - id: outh5parm
-    run: ../../subworkflow/predict_calibrate.cwl
-    label: predict_calibrate
-    scatter:
-      - msin
-    'sbg:x': 500
-    'sbg:y': 0
-  - id: check_ateam_separation
-    in:
-      - id: ms
-        source:
-          - msin
-      - id: min_separation
-        source: min_separation
-    out:
-      - id: output_imag
-      - id: logfile
-    run: ../../lofar-cwl/steps/check_ateam_separation.cwl
-    label: check_Ateam_separation
-    'sbg:x': 0
-    'sbg:y': -300
-  - id: make_sourcedb_ateam
-    in:
-      - id: sky_model
-        source: A-Team_skymodel
-      - id: logname
-        default: make_sourcedb_ateam.log
-    out:
-      - id: sourcedb
-      - id: log
-    run: ../../lofar-cwl/steps/makesourcedb.cwl
-    label: make_sourcedb_ateam
-    'sbg:x': -500
-    'sbg:y': 300
-  - id: find_skymodel_cal
-    in:
-      - id: msin
-        source:
-          - msin
-      - id: skymodels
-        source: calibrator_path_skymodel
-      - id: max_separation_arcmin
-        source: max_separation_arcmin
-    out:
-      - id: output_models
-      - id: model_name
+      - id: filter_out
       - id: logfile
-    run: ../../steps/find_skymodel_cal.cwl
-    label: find_skymodel_cal
-    'sbg:x': -500
-    'sbg:y': 500
-  - id: make_sourcedb
-    in:
-      - id: sky_model
-        source: find_skymodel_cal/output_models
-      - id: output_file_name
-        default: calibrator.sourcedb
-    out:
-      - id: sourcedb
-      - id: log
-    run: ../../lofar-cwl/steps/makesourcedb.cwl
-    label: make_sourcedb
+    run: ../../steps/identify_bad_antennas.cwl
+    label: identifybadantennas
     'sbg:x': 0
-    'sbg:y': 500
-  - id: concat_logfiles_prep_cal
-    in:
-      - id: file_list
-        linkMerge: merge_flattened
-        source: ndppp_prep_cal/logfile
-      - id: file_prefix
-        default: ndppp_prep_cal
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_prep_cal
-    'sbg:x': 250
-    'sbg:y': 300
-  - id: concat_logfiles_blsmooth
-    in:
-      - id: file_list
-        source:
-          - predict_calibrate/BLsmooth.log
-      - id: file_prefix
-        default: blsmooth_prep
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_blsmooth
-    'sbg:x': 750
-    'sbg:y': 300
-  - id: concat_logfiles_predict
-    in:
-      - id: file_list
-        linkMerge: merge_flattened
-        source:
-          - predict_calibrate/predict_cal.log
-      - id: file_prefix
-        default: predict_cal
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_predict
-    'sbg:x': 750
-    'sbg:y': 500
-  - id: concat_logfiles_calib
-    in:
-      - id: file_list
-        source:
-          - predict_calibrate/calib_cal.log
-      - id: file_prefix
-        default: calib_cal_prep
-    out:
-      - id: output
-    run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_calib
-    'sbg:x': 750
-    'sbg:y': 800
-  - id: h5parm_collector
-    in:
-      - id: h5parmFiles
-        source:
-          - predict_calibrate/outh5parm
-      - id: squeeze
-        default: true
-      - id: verbose
-        default: true
-      - id: clobber
-        default: true
-    out:
-      - id: outh5parm
-      - id: log
-    run: ../../lofar-cwl/steps/H5ParmCollector.cwl
-    label: H5parm_collector
-    'sbg:x': 750
     'sbg:y': -300
 requirements:
   - class: SubworkflowFeatureRequirement

workflows/prefactor_target/prep.cwl

@@ -190,6 +190,7 @@ outputs:
     outputSource:
       - check_ateam_separation/output_imag
       - losoto_plot_RM/output_plots
+      - plot_Ateamclipper/output_imag
     type: 'File[]?'
     linkMerge: merge_flattened
     'sbg:x': 1000
@@ -386,6 +387,7 @@ steps:
       - id: prep_logfile
       - id: predict_logfile
       - id: clipper_logfile
+      - id: clipper_output
       - id: parset
     run: ../../subworkflow/ndppp_prep_targ.cwl
     label: ndppp_prep_target
@@ -393,6 +395,31 @@ steps:
       - msin
     'sbg:x': 0
     'sbg:y': 0
+  - id: plot_Ateamclipper
+    in:
+      - id: clipper_output
+        source: concat_logfiles_clipper_output/output
+    out:
+      - id: output_imag
+    run: ../../steps/plot_Ateamclipper.cwl
+    label: concat_logfiles_clipper_output
+    'sbg:x': 600
+    'sbg:y': 200
+  - id: concat_logfiles_clipper_output
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source: dppp_prep_target/clipper_output
+      - id: file_prefix
+        default: Ateamclipper
+      - id: file_suffix
+        default: txt
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_clipper_output
+    'sbg:x': 400
+    'sbg:y': 200
   - id: concat_logfiles_prep_targ
     in:
       - id: file_list
@@ -404,7 +431,7 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_prep_target
-    'sbg:x': 250
+    'sbg:x': 400
     'sbg:y': 300
   - id: concat_logfiles_predict_targ
     in:
@@ -417,7 +444,7 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_predict_targ
-    'sbg:x': 250
+    'sbg:x': 400
     'sbg:y': 400
   - id: concat_logfiles_clipper_targ
     in:
@@ -430,7 +457,7 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_clipper_targ
-    'sbg:x': 250
+    'sbg:x': 400
     'sbg:y': 500
 requirements:
   - class: ScatterFeatureRequirement
\ No newline at end of file