add additional count statements for debug DPPP

Former-commit-id: 3c5b3467 [formerly 8183f1d5]
Former-commit-id: e1b419c7
Former-commit-id: 9efe4beb

lofar-cwl/steps/DP3.ParsetGenerator.cwl

@@ -47,7 +47,7 @@ arguments:
             except KeyError as e:
                 print('invalid input for step ', step, ':', e)
                 sys.exit(1)
-        last_step_line = 'steps=[%s]' % ', '.join(steps_id_list)
+        last_step_line = 'steps=[%s,count]' % ', '.join(steps_id_list)
         print(last_step_line)
 inputs:
   - id: steps

lofar-cwl/steps/applybeam.cwl

@@ -87,7 +87,7 @@ outputs:
 stdout: applycal_$(inputs.type).log
 stderr: applycal_$(inputs.type)_err.log
 arguments:
-  - 'steps=[applybeam]'
+  - 'steps=[applybeam,count]'
   - msout=.
 requirements:
   - class: InitialWorkDirRequirement

lofar-cwl/steps/applytarget.cwl

@@ -93,7 +93,7 @@ outputs:
 stdout: applycal_$(inputs.correction).log
 stderr: applycal_$(inputs.correction)_err.log
 arguments:
-  - 'steps=[applycal]'
+  - 'steps=[applycal,count]'
 requirements:
   - class: InitialWorkDirRequirement
     listing:

lofar-cwl/steps/calib_rot_diag.cwl

@@ -88,7 +88,7 @@ outputs:
       glob: $(inputs.output_name_h5parm)
     format: 'lofar:#H5Parm'
 arguments:
-  - 'steps=[ddecal]'
+  - 'steps=[ddecal,count]'
   - ddecal.mode=rotation+diagonal
   - ddecal.uvlambdamin=300
   - ddecal.maxiter=50

lofar-cwl/steps/ddecal.cwl

@@ -17,7 +17,7 @@ hints:
     dockerPull: lofareosc/prefactor:HBAcalibrator
 
 arguments:
-  - steps=[ddecal]
+  - steps=[ddecal,count]
   - ddecal.uvlambdamin=300
   - ddecal.maxiter=50
   - ddecal.nchan=1

lofar-cwl/steps/dpppconcat.cwl

@@ -126,7 +126,7 @@ outputs:
     outputBinding:
       glob: concat*.log
 arguments:
-  - 'steps=[filter,avg]'
+  - 'steps=[filter,avg,count]'
   - msout.orderms=False
   - avg.type=average
 requirements:

lofar-cwl/steps/filter_predict.cwl

@@ -101,7 +101,7 @@ outputs:
     outputBinding:
       glob: 'filter_predict*.log'
 arguments:
-  - 'steps=[filter,predict]'
+  - 'steps=[filter,predict,count]'
   - predict.beammode=array_factor
   - predict.usechannelfreq=False
   - msout=.

lofar-cwl/steps/gaincal.cwl

@@ -167,7 +167,7 @@ outputs:
 stdout: gaincal.log
 stderr: gaincal_err.log
 arguments:
-  - 'steps=[filter,gaincal]'
+  - 'steps=[filter,gaincal,count]'
 hints:
   - class: DockerRequirement
     dockerPull: 'lofareosc/prefactor:HBAcalibrator'

lofar-cwl/steps/predict.cwl

@@ -81,7 +81,7 @@ outputs:
     outputBinding:
       glob: 'predict_cal*.log'
 arguments:
-  - 'steps=[predict]'
+  - 'steps=[predict,count]'
   - predict.beammode=array_factor
   - predict.usechannelfreq=False
   - msout=.

subworkflow/apply_calibrate_bp.cwl

@@ -197,7 +197,7 @@ steps:
       - id: flagunconverged
         source: flagunconverged
       - id: mode
-        default: diagonal
+        default: scalarphase
     out:
       - id: h5parm
       - id: logfile

subworkflow/clocktec.cwl

@@ -34,10 +34,6 @@ outputs:
     'sbg:y': 0
   - id: logfiles
     outputSource:
-      - duplicateAbkp/log
-      - flag/log
-      - flagextend/log
-      - merge/log
       - duplicatePbkp/log
       - losoto_clocktec/log
       - losoto_residual/log
@@ -55,94 +51,10 @@ outputs:
     'sbg:x': 1500
     'sbg:y': 200
 steps:
-  - id: duplicateAbkp
-    in:
-      - id: input_h5parm
-        source: input_h5parm
-      - id: soltab
-        default: sol000/amplitude000
-      - id: soltabOut
-        default: amplitudeOrig
-    out:
-      - id: output_h5parm
-      - id: log
-    run: ../lofar-cwl/steps/LoSoTo.Duplicate.cwl
-    'sbg:x': -500
-    'sbg:y': 0
-  - id: flag
-    in:
-      - id: input_h5parm
-        source: duplicateAbkp/output_h5parm
-      - id: soltab
-        default: sol000/amplitude000
-      - id: axesToFlag
-        default:
-          - time
-          - freq
-      - id: order
-        default:
-          - 100
-          - 40
-      - id: maxCycles
-        default: 1
-      - id: maxRms
-        default: 5
-      - id: replace
-        default: false
-      - id: preflagzeros
-        default: false
-      - id: mode
-        default: smooth
-    out:
-      - id: output_h5parm
-      - id: log
-    run: ../lofar-cwl/steps/LoSoTo.Flag.cwl
-    'sbg:x': -300
-    'sbg:y': 0
-  - id: flagextend
-    in:
-      - id: input_h5parm
-        source: flag/output_h5parm
-      - id: soltab
-        default: sol000/amplitude000
-      - id: axesToExt
-        default:
-          - time
-          - freq
-      - id: size
-        default:
-          - 200
-          - 80
-      - id: percent
-        default: 50
-      - id: maxCycles
-        default: 2
-    out:
-      - id: output_h5parm
-      - id: log
-    run: ../lofar-cwl/steps/LoSoTo.Flagextend.cwl
-    'sbg:x': -100
-    'sbg:y': 0
-  - id: merge
-    in:
-      - id: input_h5parm
-        source: flagextend/output_h5parm
-      - id: soltab
-        default: sol000/phase000
-      - id: mode
-        default: copy
-      - id: soltabImport
-        default: amplitude000
-    out:
-      - id: output_h5parm
-      - id: log
-    run: ../lofar-cwl/steps/LoSoTo.Reweight.cwl
-    'sbg:x': 100
-    'sbg:y': 0
   - id: duplicatePbkp
     in:
       - id: input_h5parm
-        source: merge/output_h5parm
+        source: input_h5parm
       - id: soltab
         default: sol000/phase000
       - id: soltabOut
@@ -170,7 +82,7 @@ steps:
       - id: FlagBadChannels
         default: false
       - id: CombinePol
-        default: true
+        default: false
       - id: Fit3rdOrder
         source: fit3rdorder
       - id: Circular
@@ -182,26 +94,10 @@ steps:
     run: ../lofar-cwl/steps/LoSoTo.ClockTec.cwl
     'sbg:x': 500
     'sbg:y': 0
-  - id: losoto_residual
-    in:
-      - id: input_h5parm
-        source: losoto_clocktec/output_h5parm
-      - id: soltab
-        default: sol000/phase000
-      - id: soltabsToSub
-        source:
-          - fit3rdorder
-        valueFrom: '$(self ? ["tec", "clock", "tec3rd"] : ["tec", "clock"])'
-    out:
-      - id: output_h5parm
-      - id: log
-    run: ../lofar-cwl/steps/LoSoTo.Residual.cwl
-    'sbg:x': 700
-    'sbg:y': 0
   - id: duplicateCbkp
     in:
       - id: input_h5parm
-        source: losoto_residual/output_h5parm
+        source: losoto_clocktec/output_h5parm
       - id: soltab
         default: sol000/clock
       - id: soltabOut
@@ -210,7 +106,7 @@ steps:
       - id: output_h5parm
       - id: log
     run: ../lofar-cwl/steps/LoSoTo.Duplicate.cwl
-    'sbg:x': 900
+    'sbg:x': 700
     'sbg:y': 0
   - id: smooth
     in:
@@ -233,6 +129,22 @@ steps:
       - id: output_h5parm
       - id: logfile
     run: ../lofar-cwl/steps/LoSoTo.Smooth.cwl
+    'sbg:x': 900
+    'sbg:y': 0
+  - id: losoto_residual
+    in:
+      - id: input_h5parm
+        source: smooth/output_h5parm
+      - id: soltab
+        default: sol000/phase000
+      - id: soltabsToSub
+        source:
+          - fit3rdorder
+        valueFrom: '$(self ? ["tec", "clock", "tec3rd"] : ["tec", "clock"])'
+    out:
+      - id: output_h5parm
+      - id: log
+    run: ../lofar-cwl/steps/LoSoTo.Residual.cwl
     'sbg:x': 1100
     'sbg:y': 0
   - id: losoto_flagstation

workflows/HBA_calibrator.cwl

@@ -97,7 +97,7 @@ inputs:
     'sbg:y': -200
   - id: trusted_sources
     type: string
-    default: '3C48,3C147,3C196,3C295'
+    default: '3C48,3C147,3C196,3C295,3C380'
     'sbg:x': -1000
     'sbg:y': -100
   - id: demix_sources
@@ -294,8 +294,6 @@ steps:
         source: bandpass_freqresolution
     out:
       - id: logfiles
-      - id: msout
-      - id: parset
       - id: solutions
       - id: inspection
     run: ./prefactor_calibrator.cwl

workflows/prefactor_calibrator/ion.cwl

@@ -44,15 +44,12 @@ inputs:
 outputs:
   - id: inspection
     outputSource:
-      - losoto_plot_A1/output_plots
-      - losoto_plot_A2/output_plots
       - losoto_plot_P3/output_plots
       - losoto_plot_Pd/output_plots
       - losoto_plot_tec/output_plots
       - losoto_plot_tec3rd/output_plots
       - losoto_plot_clock/output_plots
       - losoto_plot_Pr/output_plots
-      - losoto_plot_Pr3/output_plots
     type: 'File[]?'
     linkMerge: merge_flattened
     'sbg:x': 1000
@@ -89,54 +86,6 @@ steps:
     label: clocktec
     'sbg:x': -500
     'sbg:y': 0
-  - id: losoto_plot_A1
-    in:
-      - id: input_h5parm
-        source: clocktec/output_h5parm
-      - id: soltab
-        default: sol000/amplitudeOrig
-      - id: axesInPlot
-        default:
-          - time
-          - freq
-      - id: axisInTable
-        default: ant
-      - id: plotFlag
-        default: true
-      - id: prefix
-        default: ion_ampBFlag_
-    out:
-      - id: output_plots
-      - id: logfile
-      - id: parset
-    run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
-    label: losoto_plot_A1
-    'sbg:x': 0
-    'sbg:y': -400
-  - id: losoto_plot_A2
-    in:
-      - id: input_h5parm
-        source: clocktec/output_h5parm
-      - id: soltab
-        default: sol000/amplitude000
-      - id: axesInPlot
-        default:
-          - time
-          - freq
-      - id: axisInTable
-        default: ant
-      - id: plotFlag
-        default: true
-      - id: prefix
-        default: ion_ampAFlag_
-    out:
-      - id: output_plots
-      - id: logfile
-      - id: parset
-    run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
-    label: losoto_plot_A2
-    'sbg:x': 0
-    'sbg:y': -300
   - id: losoto_plot_P3
     in:
       - id: input_h5parm
@@ -158,7 +107,7 @@ steps:
       - id: refAnt
         source: refant
       - id: prefix
-        default: ion_ph_
+        default: ion_ph
     out:
       - id: output_plots
       - id: logfile
@@ -299,7 +248,7 @@ steps:
       - id: refAnt
         source: refant
       - id: prefix
-        default: ion_ph-res_poldif
+        default: ion_ph-res
     out:
       - id: output_plots
       - id: logfile
@@ -308,36 +257,6 @@ steps:
     label: losoto_plot_Pr
     'sbg:x': 0
     'sbg:y': 300
-  - id: losoto_plot_Pr3
-    in:
-      - id: input_h5parm
-        source: clocktec/output_h5parm
-      - id: soltab
-        default: sol000/phase000
-      - id: axesInPlot
-        default:
-          - time
-          - freq
-      - id: axisInTable
-        default: ant
-      - id: minmax
-        default:
-          - -3.14
-          - 3.14
-      - id: plotFlag
-        default: true
-      - id: refAnt
-        source: refant
-      - id: prefix
-        default: ion_ph-res_
-    out:
-      - id: output_plots
-      - id: logfile
-      - id: parset
-    run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
-    label: losoto_plot_Pr3
-    'sbg:x': 0
-    'sbg:y': 400
   - id: concat_logfiles_ion
     in:
       - id: file_list
@@ -345,15 +264,12 @@ steps:
         source:
           - inh5parm_logfile
           - clocktec/logfiles
-          - losoto_plot_A1/logfile
-          - losoto_plot_A2/logfile
           - losoto_plot_P3/logfile
           - losoto_plot_Pd/logfile
           - losoto_plot_tec/logfile
           - losoto_plot_tec3rd/logfile
           - losoto_plot_clock/logfile
           - losoto_plot_Pr/logfile
-          - losoto_plot_Pr3/logfile
           - write_solutions/log
           - h5parm_pointingname/log
       - id: file_prefix