Debug delay solve

facetselfcal_config.txt

@@ -12,6 +12,5 @@ antennaconstraint_list     = ['alldutch',None,None]
 docircular                 = True
 forwidefield               = True
 stop                       = 10
-no_beamcor                 = True
+beamcor                    = auto
 phaseupstations            = core
-channelsout                = 5

steps/delay_cal_model.cwl

@@ -20,6 +20,10 @@ inputs:
         separate: true
 
 outputs:
+    - id: skymodel
+      type: File
+      outputBinding:
+        glob: $(inputs.msin.basename)/skymodel
     - id: msout
       type: Directory
       outputBinding:

steps/delay_solve.cwl

@@ -11,6 +11,13 @@ inputs:
     - id: msin
       type: Directory
       doc: Delay calibrator measurement set.
+    - id: skymodel
+      type: File
+      doc: The skymodel to be used in the first cycle in the self-calibration.
+      inputBinding:
+        prefix: --skymodel=
+        separate: false
+        shellQuote: false
     - id: configfile
       type: File
       doc: Configuration options for self-calibration.
@@ -19,13 +26,13 @@ inputs:
       doc: External self-calibration script.
     - id: h5merger
       type: Directory
-      doc: External LOFAR helper scripts for mergin h5 files.
+      doc: External LOFAR helper scripts for merging h5 files.
 
 outputs:
     - id: h5parm
-      type: File[]
+      type: File
       outputBinding:
-        glob: merged_selfcal*.h5
+        glob: merged_addCS_selfcalcyle009_linear*.h5
     - id: images
       type: File[]
       outputBinding:
@@ -53,12 +60,11 @@ requirements:
             
           msin = inputs['msin']['basename']
           configfile = inputs['configfile']['path']
-          skymodel = inputs['msin']['path'] + "/skymodel"
+          skymodel = inputs['skymodel']['path']
           selfcal = inputs['selfcal']['path']
           h5merge = inputs['h5merger']['path']
 
-          print(f'{msin}\n{skymodel}\n{selfcal}\n{h5merge}\n{configfile}')
-          subprocess.run(f'python3 {selfcal}/facetselfcal.py {msin} --helperscriptspath {selfcal} --helperscriptspathh5merge {h5merge}', shell = True)
+          subprocess.run(f'python3 {selfcal}/facetselfcal.py {msin} --helperscriptspath {selfcal} --helperscriptspathh5merge {h5merge} --skymodel {skymodel}', shell = True)
           #.format(os.path.join(helperscriptspath,'facetselfcal.py'), msin ) )
 
 hints:

steps/dp3_apply_delay.cwl

@@ -6,13 +6,11 @@ label: dp3_apply_delay
 baseCommand: DP3
 
 arguments:
-    - steps=[applyphase,applyamp,count]
-    - applyphase.type=applycal
-    - applyphase.parmdb=$(inputs.h5parm.path)
-    - applyphase.solset=$(inputs.solset)
-    - applyamp.type=applycal
-    - applyamp.parmdb=$(inputs.h5parm.path)
-    - applyamp.solset=$(inputs.solset)
+    - steps=[applysols,count]
+    - applysols.type=applycal
+    - applysols.soltab=[amplitude000,phase000]
+    - applysols.correction=fulljones
+    - applysols.updateweights=false
 
 inputs:
     - id: msin
@@ -25,10 +23,18 @@ inputs:
     - id: h5parm
       type: File
       doc: Delay calibrator solution set.
+      inputBinding:
+        position: 0
+        prefix: applysols.parmdb=
+        separate: false
     - id: solset
       type: string?
-      default: sol001
+      default: sol000
       doc: solution set in the h5parm to be used.
+      inputBinding:
+        position: 0
+        prefix: applysols.solset=
+        separate: false
     - id: max_dp3_threads
       type: int?
       default: 5
@@ -66,30 +72,6 @@ inputs:
       inputBinding:
         prefix: msout.writefullresflag=
         separate: false
-    - id: phase_correction
-      type: string?
-      default: phase000
-      inputBinding:
-        prefix: applyphase.correction=
-        separate: false
-    - id: phase_weights
-      type: boolean?
-      default: False
-      inputBinding:
-        prefix: applyphase.updateweights=
-        separate: false
-    - id: amplitude_correction
-      type: string?
-      default: amplitude000
-      inputBinding:
-        prefix: applyamp.correction=
-        separate: false
-    - id: amplitude_weights
-      type: boolean?
-      default: False
-      inputBinding:
-        prefix: applyamp.updateweights=
-        separate: false
     - id: save2json
       default: true
       type: boolean?
@@ -113,11 +95,10 @@ outputs:
       outputBinding:
         glob: dp3_apply_delay*.log
     - id: flagged_fraction_dict
-      type: string
+      type: File
       outputBinding:
           loadContents: true
           glob: $(inputs.jsonfilename)
-          outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
 
 hints:
   - class: DockerRequirement
@@ -130,6 +111,8 @@ requirements:
         writable: true
   - class: InplaceUpdateRequirement
     inplaceUpdate: true
+  - class: ResourceRequirement
+    coresMin: 6
 
 stdout: dp3_apply_delay.log
 stderr: dp3_apply_delay_err.log

workflows/concatenate-flag.cwl

@@ -6,7 +6,7 @@ label: sort-concat-flag
 inputs:
   - id: msin
     type: Directory[]
-  - id: ddf-solset
+  - id: ddf_solset
     type: File?
     doc: The solution tables generated by the DDF pipeline in an HDF5 format.
   - id: numbands
@@ -42,8 +42,8 @@ steps:
       - id: msin
         source:
           - msin
-      - id: ddf-solset
-        source: ddf-solset
+      - id: ddf_solset
+        source: ddf_solset
       - id: group_id
         source: sort_concatenate/groupnames
       - id: groups_specification

workflows/delay-calibration.cwl

@@ -13,7 +13,7 @@ inputs:
     - id: solset
       type: File
       doc: The solution set from the prefactor pipeline.
-    - id: ddf-solset
+    - id: ddf_solset
       type: File?
       doc: The solution tables generated by the DDF pipeline in an HDF5 format.
     - id: filter_baselines
@@ -80,8 +80,8 @@ steps:
           source: reference_stationSB
         - id: max_dp3_threads
           source: max_dp3_threads
-        - id: ddf-solset
-          source: ddf-solset
+        - id: ddf_solset
+          source: ddf_solset
       out:
         - id: logdir
         - id: concat_flags
@@ -114,12 +114,11 @@ steps:
         - id: msout
         - id: solutions
         - id: pictures
-        - id: phaseup_flags
         - id: logdir
-#        - id: summary_file
+        - id: summary_file
       run: ./phaseup-concat.cwl
       label: phaseup
-    
+
     - id: store_logs
       in:
         - id: files
@@ -137,6 +136,10 @@ steps:
 
 outputs:
   - id: msout
+    outputSource: phaseup/msout
+    type: Directory
+
+  - id: msouts
     outputSource: sort-concatenate-flag/msout
     type: Directory[]
 
@@ -154,8 +157,8 @@ outputs:
 
   - id: solutions
     outputSource: phaseup/solutions
-    type: File[]
+    type: File
 
-#  - id: summary_file
-#    outputSource: phaseup-concat.cwl
-#    type: File
+  - id: summary_file
+    outputSource: phaseup/summary_file
+    type: File

workflows/phaseup-concat.cwl

@@ -177,6 +177,7 @@ steps:
       - id: delay_calibrator
         source: delay_calibrator
     out:
+      - id: skymodel
       - id: msout
       - id: logfile
     run: ../steps/delay_cal_model.cwl
@@ -185,6 +186,8 @@ steps:
     in:
       - id: msin
         source: delay_cal_model/msout
+      - id: skymodel
+        source: delay_cal_model/skymodel
       - id: configfile
         source: configfile
       - id: selfcal
@@ -198,43 +201,58 @@ steps:
     run: ../steps/delay_solve.cwl
     label: delay_solve
 
-#  - id: summary
-#    in:
-#      - id: flagFiles
-#        source:
-#          - flags
-#          - prep_target_flags_join/flagged_fraction_antenna
-#        linkMerge: merge_flattened
-#      - id: pipeline
-#        source: pipeline
-#      - id: run_type
-#        source: run_type
-#      - id: filter
-#        source: filter_baselines
-#      - id: bad_antennas
-#        source:
-#          - bad_antennas
-#          - compare_stations_filter
-#        valueFrom: $(self.join(''))
-#      - id: Ateam_separation_file
-#        source: check_Ateam_separation.json
-#      - id: solutions
-#        source: delay_solve/h5parm
-#      - id: clip_sources
-#        source: clip_sources
-#        valueFrom: "$(self.join(','))"
-#      - id: removed_bands
-#        source: removed_bands
-#        valueFrom: "$(self.join(','))"
-#      - id: min_unflagged_fraction
-#        source: min_unflagged_fraction
-#      - id: refant
-#        source: refant
-#    out:
-#      - id: summary_file
-#      - id: logfile
-#    run: ../steps/summary.cwl
-#    label: summary
+  - id: apply_delay
+    in:
+      - id: msin
+        source: phaseup_concatenate/msout
+        valueFrom: $(self[0])
+      - id: h5parm
+        source: delay_solve/h5parm
+    out:
+      - id: msout
+      - id: logfile
+      - id: flagged_fraction_dict
+    run: ../steps/dp3_apply_delay.cwl
+    label: apply_delay
+
+  - id: summary
+    in:
+      - id: flagFiles
+        source:
+          - flags
+          - phaseup_flags_join/flagged_fraction_antenna
+          - apply_delay/flagged_fraction_dict
+        linkMerge: merge_flattened
+      - id: pipeline
+        source: pipeline
+      - id: run_type
+        source: run_type
+      - id: filter
+        source: filter_baselines
+      - id: bad_antennas
+        source:
+          - bad_antennas
+          - compare_stations_filter
+        valueFrom: $(self.join(''))
+      - id: Ateam_separation_file
+        source: check_Ateam_separation.json
+      - id: solutions
+        source: delay_solve/h5parm
+      - id: clip_sources
+        source: clip_sources
+        valueFrom: "$(self.join(','))"
+      - id: removed_bands
+        source: removed_bands
+        valueFrom: "$(self.join(','))"
+      - id: min_unflagged_fraction
+        source: min_unflagged_fraction
+      - id: refant
+        source: refant
+    out:
+      - id: summary_file
+      - id: logfile
+    run: ../steps/summary.cwl
+    label: summary
 
   - id: save_logfiles
     in:
@@ -247,7 +265,8 @@ steps:
           - concat_logfiles_phaseup/output
           - delay_cal_model/logfile
           - delay_solve/logfile
-#          - summary/logfile
+          - apply_delay/logfile
+          - summary/logfile
       - id: sub_directory_name
         default: phaseup
     out:
@@ -257,24 +276,20 @@ steps:
 
 outputs:
   - id: msout
-    outputSource: delay_cal_model/msout
     type: Directory
+    outputSource: apply_delay/msout
   - id: solutions
-    outputSource: delay_solve/h5parm
-    type: File[]
-  - id: phaseup_flags
     type: File
-    outputSource: phaseup_flags_join/flagged_fraction_antenna
+    outputSource: delay_solve/h5parm
   - id: logdir
     outputSource: save_logfiles/dir
     type: Directory
   - id: pictures
     type: File[]
     outputSource: delay_solve/images
-#  - id: summary_file
-#    type: File
-#    outputSource: summary/summary_file
-
+  - id: summary_file
+    type: File
+    outputSource: summary/summary_file
 
 requirements:
   - class: SubworkflowFeatureRequirement

workflows/subworkflows/concatenation.cwl

@@ -6,7 +6,7 @@ label: concatenation
 inputs:
   - id: msin
     type: Directory[]
-  - id: ddf-solset
+  - id: ddf_solset
     type: File?
     doc: The solution tables generated by the DDF pipeline in an HDF5 format.
   - id: group_id
@@ -70,12 +70,12 @@ steps:
         source: 
             - AOflagging/msout
         pickValue: first_non_null
-      - id: ddf-solset
-        source: ddf-solset
+      - id: ddf_solset
+        source: ddf_solset
     out:
       - id: msout
       - id: logfile
-    when: $(inputs.ddf-solset != null)
+    when: $(inputs.ddf_solset != null)
     run: ../../steps/dp3_applycal.cwl
     label: dp3_applycal
   - id: concat_logfiles_aoflagging
@@ -124,3 +124,6 @@ outputs:
   - id: concat_flag_statistics
     type: string
     outputSource: dp3_concat/flagged_statistics
+
+requirements:
+    - class: InlineJavascriptRequirement