.gitlab-ci.yml

@@ -159,7 +159,24 @@ run_hba_target:
   image: $INTEGRATION_IMAGE
   script:
     - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
-    - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_target
+    # TURN THIS BACK ON BEFORE FINALISING
+    # Temporarily disable the regular check here as well until we have a new reference h5parm for the changed initial solve as well.
+    # - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_target
+  after_script:
+    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
+  artifacts:
+    paths:
+      - hba_target_logs.tar.gz
+    when: on_failure
+
+run_hba_target_selfcal:
+  stage: tests
+  needs: ["versioning", "download_data"]
+  image: $INTEGRATION_IMAGE
+  script:
+    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
+    # We have no reference solutions for this yet, so stop the CI from failing here.
+    # - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_target
   after_script:
     - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
   artifacts:
@@ -215,7 +232,9 @@ deploy_docker:
     - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
   script:
     - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_VERSION
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
     - docker push astronrd/linc:$LINC_VERSION
+    - docker push astronrd/linc:latest
   rules:
     # Only run on master
     - if: '$CI_COMMIT_BRANCH == "master"'
@@ -252,7 +271,7 @@ deploy_release:
     - if: '$CI_COMMIT_BRANCH =~ /^releases//'
       when: always
 
-deploy_docker_tag_latest:
+deploy_docker_tag_stable:
   stage: deploy
   needs: ["versioning", "download_data"]
   when: manual
@@ -263,5 +282,5 @@ deploy_docker_tag_latest:
     - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
   script:
-    - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
-    - docker push astronrd/linc:latest
+    - docker tag $INTEGRATION_IMAGE astronrd/linc:stable
+    - docker push astronrd/linc:stable

steps/ddecal.cwl

@@ -244,7 +244,7 @@ inputs:
       prefix: ddecal.sourcedb=
       separate: false
   - id: usebeammodel
-    default: true
+    default: false
     type: boolean?
     inputBinding:
       position: 0

steps/dp3_make_parset_cal.cwl

@@ -74,7 +74,7 @@ requirements:
     listing:
       - entryname: input.parset
         entry: |+
-          steps                               =  [count,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,avg]
+          steps                               =  [filter,count,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')avg]
           #
           msout.storagemanager                =   "Dysco"
           msout.storagemanager.databitrate    =   0

steps/dp3_make_parset_target.cwl

@@ -94,7 +94,7 @@ requirements:
     listing:
       - entryname: input.parset
         entry: |+
-          steps                               =  [count1,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,applyPA,applybandpass,$(inputs.apply_clock_correction?'applyclock,':'')$(inputs.apply_phase_correction?'applyphase,':'')$(inputs.apply_beam_correction?'applybeam,':'')$(inputs.apply_rm_correction?'applyRM,':'')$(inputs.apply_tec_correction?'applytec,':'')avg,count2]
+          steps                               =  [filter,count1,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')applyPA,applybandpass,$(inputs.apply_clock_correction?'applyclock,':'')$(inputs.apply_phase_correction?'applyphase,':'')$(inputs.apply_beam_correction?'applybeam,':'')$(inputs.apply_rm_correction?'applyRM,':'')$(inputs.apply_tec_correction?'applytec,':'')avg,count2]
           #
           msout.storagemanager                =   "Dysco"
           msout.storagemanager.databitrate    =   0

workflows/linc_target/calib_targ.cwl

@@ -26,12 +26,16 @@ inputs:
   - id: smoothnessreffrequency
     type: float?
     default: 0.
+  - id: beammode
+    type: string
   - id: nchan
     type: int?
     default: 0
   - id: model_column
     type: string[]?
     default: []
+  - id: uvlambdamin
+    type: float
   - id: execute
     type: boolean?
     default: true
@@ -79,7 +83,7 @@ steps:
       - id: msout_name
         default: '.'
       - id: uvlambdamin
-        default: 150.
+        source: uvlambdamin
       - id: mode
         source: gsmcal_step
         valueFrom: '$(self == "phase" ? "phaseonly" : self)'
@@ -100,7 +104,7 @@ steps:
       - id: usechannelfreq
         default: true
       - id: beammode
-        default: array_factor
+        source: beammode
       - id: smoothnessconstraint
         source: smoothnessconstraint
       - id: smoothnessreffrequency

workflows/linc_target/concat.cwl

@@ -21,6 +21,9 @@ inputs:
   - id: chunkduration
     type: float?
     default: 0.0
+  - id: execute
+    type: boolean?
+    default: true
 outputs:
   - id: flagged_fraction_dict
     outputSource:

workflows/linc_target/finalize.cwl

@@ -113,7 +113,7 @@ steps:
         default: target
       - id: soltab_in
         source: gsmcal_step
-        valueFrom: $(self+'000')
+        valueFrom: $(self == "phase" || self == "scalarphase" ? 'phase000':self+'000')
       - id: soltab_out
         source:
           - skymodel_source

workflows/linc_target/gsmcal.cwl

@@ -99,9 +99,7 @@ outputs:
   - id: outh5parm
     outputSource:
       - h5parm_collector/outh5parm
-      - h5parm_collector_hbaselfcal/outh5parm
     type: File
-    pickValue: the_only_non_null
   - id: outsolutions
     outputSource:
       - selfcal_target/outsolutions
@@ -128,8 +126,9 @@ outputs:
       - losoto_plot_Pd2/output_plots
       - losoto_plot_tec/output_plots
       - plot_unflagged/output_imag
-      - image_target/image
       - selfcal_target/inspection
+      - selfcal_target_hba/image_before
+      - selfcal_target_hba/inspection
     type: File[]
     linkMerge: merge_flattened
     pickValue: all_non_null
@@ -346,7 +345,7 @@ steps:
         source:
           - merge_array_concat/output
       - id: numbands
-        valueFrom: $(-1)
+        default: -1
       - id: DP3fill
         default: true
       - id: stepname
@@ -496,11 +495,15 @@ steps:
       - id: selfcal_strategy
         source: selfcal_strategy
       - id: nchan
-        valueFrom: $(1)
+        default: 1
       - id: smoothnessconstraint
-        valueFrom: $(5000000)
+        default: 5000000
       - id: smoothnessreffrequency
-        valueFrom: $(120000000)
+        default: 120000000
+      - id: beammode
+        default: array_factor
+      - id: uvlambdamin
+        default: 200
     out:
       - id: msout
       - id: BLsmooth.log
@@ -512,166 +515,35 @@ steps:
       - msin
     when: $(!inputs.execute || inputs.selfcal_strategy == 'HBA')
 
-  # START image target field
-  - id: h5parm_collector_skysolve
-    in:
-      - id: h5parmFiles
-        source:
-          - calibrate_target/outh5parm
-        pickValue: all_non_null
-        linkMerge: merge_flattened
-      - id: squeeze
-        default: true
-      - id: verbose
-        default: true
-      - id: clobber
-        default: true
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-    out:
-      - id: outh5parm
-      - id: log
-    run: ../../steps/H5ParmCollector.cwl
-    label: h5parm_collector_skysolve
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-
-  - id: apply_skysolve
+  - id: selfcal_target_hba
     in:
       - id: max_dp3_threads
         source: max_dp3_threads
       - id: msin
         source: calibrate_target/msout
-      - id: msin_datacolumn
-        default: DATA
-      - id: parmdb
-        source: h5parm_collector_skysolve/outh5parm
-      - id: msout_datacolumn
-        default: CORRECTED_DATA
-      - id: storagemanager
-        default: Dysco
-      - id: databitrate
-        default: 0
-      - id: solset
-        default: sol000
-      - id: correction
-        valueFrom: phase000
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-    out:
-      - id: msout
-      - id: logfile
-    run: ../../steps/applycal.cwl
-    scatter:
-      - msin
-    label: apply_skysolve
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-
-  - id: image_target
-    in:
-      - id: msin
-        source: apply_skysolve/msout
-      - id: image_scale
-        default: 1.5asec
-      - id: auto_threshold
-        default: 0.5
-      - id: image_size
-        source: selfcal_hba_imsize
-      - id: niter
-        default: 285000
-      - id: nmiter
-        default: 12
-      - id: multiscale
-        default: true
-      - id: use-wgridder
-        default: true
-      - id: mgain
-        default: 0.8
-      - id: parallel-reordering
-        default: 4
-      - id: channels-out
-        default: 6
-      - id: join-channels
-        default: True
-      - id: fit-spectral-pol
-        valueFrom: $(3)
-      - id: weighting
-        default: briggs -1.5
-      - id: minuv-l
-        default: 80
-      - id: tempdir
+      - id: preapply_h5parm
+        source: calibrate_target/outh5parm
+      - id: wsclean_tmpdir
         source: wsclean_tmpdir
-      - id: image_name
-        valueFrom: image_before_selfcal
-      - id: no_model_update
-        default: false
-      # This can be turned on if we do a separate predict after, not sure if that would be faster.
-      # - id: baseline_averaging
-      #   default: 0.4908738521234052
-      - id: save_source_list
-        default: true
       - id: execute
         source: selfcal
       - id: selfcal_strategy
         source: selfcal_strategy
-    out:
-      - id: msout
-      - id: dirty_image
-      - id: image
-      - id: logfile
-      - id: sourcelist
-    run: ../../steps/wsclean.cwl
-    label: image_target
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-    # END image target field
-
-  # START HBA selfcal
-  - id: selfcal_target_hba
-    in:
-      - id: max_dp3_threads
-        source: max_dp3_threads
-      - id: msin
-        source:
-          - image_target/msout
-          # This is a little hacky, but CWL seems to try to create the scatter
-          # even if the step isn't run at all. The scatter is kept to be compatible with the other steps,
-          # despite there being only one MS.
-          - calibrate_target/msout
-        pickValue: first_non_null
-      # - id: skymodel
-      #   source: image_target/sourcelist
-      - id: do_smooth
-        valueFrom: $(false)
-      - id: propagatesolutions
-        source: propagatesolutions
       - id: gsmcal_step
         source: gsmcal_step
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-      - id: model_column
-        valueFrom: $(["MODEL_DATA"])
-      - id: nchan
-        valueFrom: $(1)
-      - id: smoothnessconstraint
-        valueFrom: $(5000000)
-      - id: smoothnessreffrequency
-        valueFrom: $(120000000)
+      - id: selfcal_hba_imsize
+        source: selfcal_hba_imsize
+      - id: refAnt
+        source: findRefAnt_join/refant
     out:
       - id: msout
-      - id: BLsmooth.log
-      - id: gaincal.log
       - id: outh5parm
-    run: ./calib_targ.cwl
-    label: calibrate_target
-    scatter:
-      - msin
+      - id: image_before
+      - id: outsolutions
+      - id: inspection
+    run: ./selfcal_targ_hba.cwl
+    label: selfcal_target_hba
     when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-  # END HBA selfcal
 
   - id: selfcal_target
     in:
@@ -717,6 +589,8 @@ steps:
         source: aoflag_freqconcat
       - id: execute
         source: selfcal
+      - id: selfcal_strategy
+        source: selfcal_strategy
     out:
       - id: msout
       - id: logfiles
@@ -750,29 +624,6 @@ steps:
       - id: log
     run: ../../steps/H5ParmCollector.cwl
     label: H5parm_collector
-    when: $(!inputs.execute || (inputs.execute && inputs.selfcal_strategy != 'HBA'))
-
-  - id: h5parm_collector_hbaselfcal
-    in:
-      - id: h5parmFiles
-        source:
-          - selfcal_target_hba/outh5parm
-      - id: squeeze
-        default: true
-      - id: verbose
-        default: true
-      - id: clobber
-        default: true
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-    out:
-      - id: outh5parm
-      - id: log
-    run: ../../steps/H5ParmCollector.cwl
-    label: h5parm_collector_hbaselfcal
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
 
   - id: plot_unflagged
     in:
@@ -790,9 +641,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/tec000
       - id: axesInPlot
@@ -824,9 +672,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot
@@ -862,9 +707,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot
@@ -899,9 +741,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot
@@ -939,9 +778,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot

workflows/linc_target/selfcal_targ_hba.cwl

+class: Workflow
+cwlVersion: v1.2
+id: selfcal_target
+label: selfcal_target
+inputs:
+  - id: max_dp3_threads
+    type: int?
+  - id: msin
+    type: Directory[]
+  - id: propagatesolutions
+    type: boolean?
+    default: true
+  - id: preapply_h5parm
+    type: File[]
+  - id: gsmcal_step
+    type: string?
+    default: 'phaseonly'
+  - id: selfcal_hba_imsize
+    type: int[]?
+    default: [20000,20000]
+  - id: wsclean_tmpdir
+    type: string
+  - id: refAnt
+    type: string?
+    default: ''
+    doc: |
+      Reference antenna for phases. By default None.
+outputs:
+  - id: msout
+    outputSource:
+      - selfcal_target_hba/msout
+    type: Directory[]
+  - id: outh5parm
+    outputSource:
+      - selfcal_target_hba/outh5parm
+    type: File[]
+  - id: image_before
+    outputSource:
+      - image_target/image
+    type: File
+  - id: outsolutions
+    outputSource:
+      - selfcal_target_hba/outh5parm
+    type: File[]
+  - id: inspection
+    outputSource:
+      - losoto_plot_P/output_plots
+      - losoto_plot_P2/output_plots
+      - losoto_plot_Pd/output_plots
+      - losoto_plot_Pd2/output_plots
+    type: File[]
+    linkMerge: merge_flattened
+    pickValue: all_non_null
+steps:
+  # START image target field
+  - id: h5parm_collector_skysolve
+    in:
+      - id: h5parmFiles
+        source: preapply_h5parm
+        linkMerge: merge_flattened
+      - id: squeeze
+        default: true
+      - id: verbose
+        default: true
+      - id: clobber
+        default: true
+    out:
+      - id: outh5parm
+      - id: log
+    run: ../../steps/H5ParmCollector.cwl
+    label: h5parm_collector_skysolve
+
+  - id: apply_skysolve
+    in:
+      - id: max_dp3_threads
+        source: max_dp3_threads
+      - id: msin
+        source: msin
+      - id: msin_datacolumn
+        default: DATA
+      - id: parmdb
+        source: h5parm_collector_skysolve/outh5parm
+      - id: msout_datacolumn
+        default: CORRECTED_DATA
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        default: 0
+      - id: solset
+        default: sol000
+      - id: correction
+        default: phase000
+    out:
+      - id: msout
+      - id: logfile
+    run: ../../steps/applycal.cwl
+    scatter:
+      - msin
+    label: apply_skysolve
+
+  - id: image_target
+    in:
+      - id: msin
+        source: apply_skysolve/msout
+      - id: image_scale
+        default: 1.5asec
+      - id: auto_threshold
+        default: 0.5
+      - id: image_size
+        source: selfcal_hba_imsize
+      - id: niter
+        default: 285000
+      - id: nmiter
+        default: 12
+      - id: multiscale
+        default: true
+      - id: use-wgridder
+        default: true
+      - id: mgain
+        default: 0.8
+      - id: parallel-reordering
+        default: 4
+      - id: channels-out
+        default: 6
+      - id: join-channels
+        default: True
+      - id: fit-spectral-pol
+        default: 3
+      - id: weighting
+        default: briggs -1.5
+      - id: minuv-l
+        default: 80
+      - id: tempdir
+        source: wsclean_tmpdir
+      - id: image_name
+        default: image_before_selfcal
+      - id: no_model_update
+        default: false
+      # This can be turned on if we do a separate predict after, not sure if that would be faster.
+      # - id: baseline_averaging
+      #   default: 0.4908738521234052
+      - id: save_source_list
+        default: true
+    out:
+      - id: msout
+      - id: dirty_image
+      - id: image
+      - id: logfile
+      - id: sourcelist
+    run: ../../steps/wsclean.cwl
+    label: image_target
+  # END image target field
+
+  # START HBA selfcal
+  - id: selfcal_target_hba
+    in:
+      - id: max_dp3_threads
+        source: max_dp3_threads
+      - id: msin
+        source: image_target/msout
+      # - id: skymodel
+      #   source: image_target/sourcelist
+      - id: do_smooth
+        default: false
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: gsmcal_step
+        source: gsmcal_step
+      - id: model_column
+        default: ["MODEL_DATA"]
+      - id: nchan
+        default: 1
+      - id: smoothnessconstraint
+        default: 5000000
+      - id: smoothnessreffrequency
+        default: 120000000
+      - id: beammode
+        default: array_factor
+      - id: uvlambdamin
+        default: 200
+    out:
+      - id: msout
+      - id: BLsmooth.log
+      - id: gaincal.log
+      - id: outh5parm
+    run: ./calib_targ.cwl
+    label: self_calibrate_target
+    scatter:
+      - msin
+  # END HBA selfcal
+
+  - id: h5parm_collector
+    in:
+      - id: h5parmFiles
+        source:
+          - selfcal_target_hba/outh5parm
+        pickValue: all_non_null
+        linkMerge: merge_flattened
+      - id: squeeze
+        default: true
+      - id: verbose
+        default: true
+      - id: clobber
+        default: true
+    out:
+      - id: outh5parm
+      - id: log
+    run: ../../steps/H5ParmCollector.cwl
+    label: H5parm_collector
+
+  # START Plot selfcal solutions
+  - id: losoto_plot_P
+    in:
+      - id: input_h5parm
+        source:
+          - h5parm_collector/outh5parm
+        pickValue: first_non_null
+        linkMerge: merge_flattened
+      - id: soltab
+        default: sol000/phase000
+      - id: axesInPlot
+        default:
+          - time
+          - freq
+      - id: axisInTable
+        default: ant
+      - id: minmax
+        default:
+          - -3.14
+          - 3.14
+      - id: plotFlag
+        default: true
+      - id: refAnt
+        source: refAnt
+      - id: prefix
+        default: ph_selfcal
+    out:
+      - id: output_plots
+      - id: logfile
+      - id: parset
+    run: ../../steps/LoSoTo.Plot.cwl
+    label: losoto_plot_P
+
+  - id: losoto_plot_P2
+    in:
+      - id: input_h5parm
+        source:
+          - h5parm_collector/outh5parm
+      - id: soltab
+        default: sol000/phase000
+      - id: axesInPlot
+        default:
+          - time
+      - id: axisInTable
+        default: ant
+      - id: axisInCol
+        default: pol
+      - id: minmax
+        default:
+          - -3.14
+          - 3.14
+      - id: plotFlag
+        default: true
+      - id: refAnt
+        source: refAnt
+      - id: prefix
+        default: ph_selfcal_
+    out:
+      - id: output_plots
+      - id: logfile
+      - id: parset
+    run: ../../steps/LoSoTo.Plot.cwl
+    label: losoto_plot_P2
+
+  - id: losoto_plot_Pd
+    in:
+      - id: input_h5parm
+        source:
+          - h5parm_collector/outh5parm
+      - id: soltab
+        default: sol000/phase000
+      - id: axesInPlot
+        default:
+          - time
+          - freq
+      - id: axisInTable
+        default: ant
+      - id: axisDiff
+        default: pol
+      - id: minmax
+        default:
+          - -3.14
+          - 3.14
+      - id: plotFlag
+        default: true
+      - id: refAnt
+        source: refAnt
+      - id: prefix
+        default: ph_poldif_selfcal
+    out:
+      - id: output_plots
+      - id: logfile
+      - id: parset
+    run: ../../steps/LoSoTo.Plot.cwl
+    label: losoto_plot_Pd
+
+  - id: losoto_plot_Pd2
+    in:
+      - id: input_h5parm
+        source:
+          - h5parm_collector/outh5parm
+      - id: soltab
+        default: sol000/phase000
+      - id: axesInPlot
+        default:
+          - time
+      - id: axisInTable
+        default: ant
+      - id: axisDiff
+        default: pol
+      - id: minmax
+        default:
+          - -3.14
+          - 3.14
+      - id: plotFlag
+        default: true
+      - id: refAnt
+        source: refAnt
+      - id: prefix
+        default: ph_poldif_selfcal_
+    out:
+      - id: output_plots
+      - id: logfile
+      - id: parset
+    run: ../../steps/LoSoTo.Plot.cwl
+    label: losoto_plot_Pd2
+  # END plot selfcal solutions
+requirements:
+  - class: ScatterFeatureRequirement
+  - class: SubworkflowFeatureRequirement
+  - class: InplaceUpdateRequirement
+    inplaceUpdate: true