automized demixing selection (RAP-414)

.gitlab-ci.yml

@@ -121,7 +121,7 @@ run_hba_calibrator:
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
-    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --preserve-environment OPENBLAS_NUM_THREADS --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
   after_script:
     - find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
@@ -135,7 +135,7 @@ run_hba_target:
   needs: ["versioning", "download_data"]
   image: $INTEGRATION_IMAGE
   script:
-    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --preserve-environment OPENBLAS_NUM_THREADS --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
+    - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
   after_script:
     - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -

docs/source/calibrator.rst

@@ -258,10 +258,11 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
 *Demixing options* (only used if demix step is added to the ``prep_cal_strategy`` variable)
 
 - ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
-- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
 - ``demix_freqstep``: number of channels to average when demixing (default: 16)
 - ``demix_timestep`` : number of time slots to average when demixing (default: 10)
-- ``demix``: enable demixing (default: ``false``)
+- ``demix``: if ``true`` force demixing using all sources of ``demix_sources``, if ``false`` do not demix (default: ``null``, automatically determines sources to be demixed according to ``min_separation``)
+- ``lbfgs_historysize``: for the LBFGS solver: the history size, specified as a multiple of the parameter vector, to use to approximate the inverse Hessian (default: 10)
+- ``lbfgs_robustdof``: for the LBFGS solver: the degrees of freedom (DOF) given to the noise model (default: 200)
 
 *Further pipeline options*
 

docs/source/parset.rst

@@ -61,7 +61,6 @@ There are more parameters you may want to adjust that can be added to this input
         "do_transfer": false,
         "trusted_sources": "3C48,3C147,3C196,3C295,3C380",
         "demix_sources": ["CasA","CygA"],
-        "demix_target": "",
         "demix_freqstep": 16,
         "demix_timestep": 10,
         "demix": false,
@@ -78,7 +77,9 @@ There are more parameters you may want to adjust that can be added to this input
         "A-Team_skymodel": null,
         "avg_timeresolution": 4,
         "avg_freqresolution": "48.82kHz",
-        "bandpass_freqresolution": "195.3125kHz"
+        "bandpass_freqresolution": "195.3125kHz",
+        "lbfgs_historysize" : 10,
+        "lbfgs_robustdof" : 200
     }
 
 If you just want to alter one of the defaults it is sufficient to override it by specifing its new value the JSON input file::

docs/source/target.rst

@@ -280,10 +280,11 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
 *Demixing and clipping options*
 
 - ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
-- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
 - ``demix_freqstep``: number of channels to average when demixing (default: 16)
 - ``demix_timestep`` : number of time slots to average when demixing (default: 10)
-- ``demix``: enable demixing (default: ``false``)
+- ``demix``: if ``true`` force demixing using all sources of ``demix_sources``, if ``false`` do not demix (default: ``null``, automatically determines sources to be demixed according to ``min_separation``)
+- ``lbfgs_historysize``: for the LBFGS solver: the history size, specified as a multiple of the parameter vector, to use to approximate the inverse Hessian (default: 10)
+- ``lbfgs_robustdof``: for the LBFGS solver: the degrees of freedom (DOF) given to the noise model (default: 200)
 - ``clip_sources``: list of the skymodel patches to be used for Ateamclipping (default: ``[VirA_4_patch,CygAGG,CasA_4_patch,TauAGG]``)
 
 *Further pipeline options*

scripts/make_summary.py

@@ -91,8 +91,7 @@ def main(flagFiles = None, pipeline = 'prefactor', run_type = 'calibrator', filt
 			demix_list = []
 			for i in range(len(json_output['metrics'][pipeline]['close_sources'])):
 				Ateam_name = json_output['metrics'][pipeline]['close_sources'][i]['source']
-				if demix:
-					if Ateam_name in demix_sources:
+				if Ateam_name in demix_sources and demix:
 					json_output['metrics'][pipeline]['close_sources'][i]['mitigation'] = 'demix'
 					demix_list.append(Ateam_name)
 				elif Ateam_name in clip_sources:

steps/H5ParmCollector.cwl

@@ -12,7 +12,7 @@ baseCommand:
 inputs:
   - id: h5parmFiles
     type:
-      - 'File[]'
+      - File[]
       - File
     format: lofar:#H5Parm
     inputBinding:

steps/average.cwl

@@ -79,7 +79,7 @@ outputs:
     outputBinding:
       glob: $(inputs.msout_name)
   - id: logfile
-    type: 'File[]'
+    type: File[]
     outputBinding:
       glob: average*.log
 arguments:

steps/check_demix.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: check_demix
+baseCommand:
+  - python3
+  - check_demix.py
+inputs:
+  - id: skymodel
+    type: File?
+  - id: demix_sources
+    type: string[]?
+    default:
+      - VirA_4_patch
+      - CygAGG
+      - CasA_4_patch
+      - TauAGG
+  - id: clip_sources
+    type: string[]?
+    default:
+      - VirA_4_patch
+      - CygAGG
+      - CasA_4_patch
+      - TauAGG
+  - id: demix
+    type: boolean?
+    inputBinding:
+        valueFrom: '$(self ? 1 : 0)'
+  - id: Ateam_separation_file
+    type: File
+
+outputs:
+  - id: out_demix
+    type: boolean
+    outputBinding:
+      loadContents: true
+      glob: 'out.json'
+      outputEval: '$(JSON.parse(self[0].contents).demix == 0 ? false : true)'
+  - id: out_demix_sources
+    type: string[]
+    outputBinding:
+      loadContents: true
+      glob: 'out.json'
+      outputEval: '$(JSON.parse(self[0].contents).demix_sources)'
+  - id: out_clip_sources
+    type: string[]
+    outputBinding:
+      loadContents: true
+      glob: 'out.json'
+      outputEval: '$(JSON.parse(self[0].contents).clip_sources)'
+  - id: logfile
+    type: File
+    outputBinding:
+      glob: check_demix.log
+label: check_demix
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+     - entryname: check_demix.py
+       entry: |
+        import json
+        import os
+        import lsmtool
+        import numpy, math
+        null = None
+
+        targets = {'CasA'   : {'ra' : 6.123487680622104,  'dec' : 1.0265153995604648},   \
+                   'CygA'   : {'ra' : 5.233686575770755,  'dec' : 0.7109409582180791},   \
+                   'TauA'   : {'ra' : 1.4596748493730913, 'dec' : 0.38422502335921294},  \
+                   'HerA'   : {'ra' : 4.4119087330382163, 'dec' : 0.087135562905816893}, \
+                   'VirA'   : {'ra' : 3.276086511413598,  'dec' : 0.21626589533567378},  \
+                   'Sun'    : {'ra' : 0,                  'dec' : 0                  },  \
+                   'Jupiter': {'ra' : 0,                  'dec' : 0                  },  \
+                   'Moon'   : {'ra' : 0,                  'dec' : 0                  }}
+
+        inputs = json.loads(r"""$(inputs)""")
+
+        demix                 = inputs['demix']
+        demix_sources         = inputs['demix_sources']
+        clip_sources          = inputs['clip_sources']
+        skymodel              = inputs['skymodel']
+        Ateam_separation_file = inputs['Ateam_separation_file']["path"]
+
+        if skymodel is None:
+            skymodel = os.path.expandvars("$PREFACTOR_DATA_ROOT/skymodels/Ateam_LBA_CC.skymodel")
+        else:
+            skymodel = skymodel["path"]
+        s = lsmtool.load(skymodel)
+
+        f = open(Ateam_separation_file, 'r')
+        Ateam_separation = json.load(f)
+        f.close()      
+
+        if demix is None:
+            demix      = 0
+            demix_list = []
+            for item in Ateam_separation:
+                ra  = math.degrees(targets[item['source']]['ra'])
+                dec = math.degrees(targets[item['source']]['dec'])
+                dist_deg = s.getDistance(ra, dec, byPatch=True)
+                if any(dist_deg * 60.0 < 6.0):
+                    for index in numpy.where(dist_deg * 60 < 6.0)[0]:
+                        patch_name = s.getPatchNames()[index]
+                        if patch_name in demix_sources:
+                            demix_list.append(patch_name)
+                            demix = 1
+            demix_sources = demix_list
+        if demix == 1:
+            clip_sources  = list(set(clip_sources).difference(demix_sources))
+        
+        print("The following sources will be demixed: ", demix_sources)
+        print("The following sources will be clipped: ", clip_sources)
+
+        cwl_output = {"demix_sources" : demix_sources,
+                       "clip_sources" : clip_sources,
+                       "demix"        : demix}
+
+        with open('./out.json', 'w') as fp:
+            json.dump(cwl_output, fp)
+
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor3-cwl
+
+stdout: check_demix.log
+stderr: check_demix_err.log

steps/concatenate_files.cwl

@@ -6,7 +6,7 @@ baseCommand:
   - bulk_rename.sh
 inputs:
   - id: file_list
-    type: 'File[]'
+    type: File[]
     inputBinding:
       position: 0
   - id: file_prefix

steps/dppp_make_parset_cal.cwl

@@ -16,7 +16,7 @@ inputs:
     type: int?
     default: 20
   - id: baselines_to_flag
-    type: 'string[]?'
+    type: string[]?
     default: []
   - id: elevation_to_flag
     type: string?
@@ -43,13 +43,19 @@ inputs:
     type: string?
     default: ''
   - id: subtract_sources
-    type: 'string[]?'
+    type: string[]?
     default:
       - CasA
       - CygA
   - id: ntimechunk
     type: int?
     default: 10
+  - id: lbfgs_historysize
+    type: int?
+    default: 10
+  - id: lbfgs_robustdof
+    type: float?
+    default: 200
 outputs:
   - id: output
     type: File
@@ -113,3 +119,7 @@ requirements:
           demix.freqstep                      =   1
           demix.timestep                      =   1
           demix.instrumentmodel               =   instrument
+          demix.uselbfgssolver                =   True
+          demix.lbfgs.historysize             =   $(inputs.lbfgs_historysize)
+          demix.lbfgs.robustdof               =   $(inputs.lbfgs_robustdof)
+          demix.maxiter                       =   20

steps/dppp_make_parset_target.cwl

@@ -31,7 +31,7 @@ inputs:
     type: int?
     default: 20
   - id: baselines_to_flag
-    type: 'string[]?'
+    type: string[]?
     default: []
   - id: elevation_to_flag
     type: string?
@@ -61,7 +61,7 @@ inputs:
     type: string?
     default: 'pointing'
   - id: subtract_sources
-    type: 'string[]?'
+    type: string[]?
     default:
       - CasA
       - CygA

steps/dpppconcat.cwl

@@ -141,7 +141,7 @@ outputs:
         glob: $(inputs.jsonfilename)
         outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
   - id: logfile
-    type: 'File[]'
+    type: File[]
     outputBinding:
       glob: concat*.log
 arguments:

steps/filter_ms_group.cwl

@@ -14,7 +14,7 @@ inputs:
     inputBinding:
       position: 1
   - id: measurement_sets
-    type: 'Directory[]'
+    type: Directory[]
     inputBinding:
       position: 2
 outputs:

steps/filter_predict.cwl

@@ -36,7 +36,7 @@ inputs:
       separate: false
   - default: null
     id: sources
-    type: 'string[]?'
+    type: string[]?
     inputBinding:
       position: 0
       prefix: predict.sources=

steps/gaincal.cwl

@@ -142,7 +142,7 @@ inputs:
       separate: false
   - default: null
     id: blrange
-    type: 'int[]?'
+    type: int[]?
     inputBinding:
       position: 0
       prefix: filter.blrange=

steps/plot_unflagged.cwl

@@ -5,7 +5,7 @@ baseCommand:
   - python3
 inputs:
   - id: msin
-    type: 'Directory[]'
+    type: Directory[]
     inputBinding:
       position: 1
   - id: unflagged_fraction

steps/predict.cwl

@@ -36,7 +36,7 @@ inputs:
       separate: false
   - default: null
     id: sources
-    type: 'string[]?'
+    type: string[]?
     inputBinding:
       position: 0
       prefix: predict.sources=

steps/summary.cwl

@@ -8,7 +8,7 @@ baseCommand:
 inputs:
   - id: flagFiles
     type:
-      - 'File[]'
+      - File[]
       - File
     inputBinding:
       position: 1

steps/uvplot.cwl

@@ -8,7 +8,7 @@ baseCommand:
 inputs:
   - id: MSfiles
     type:
-      - 'Directory[]'
+      - Directory[]
       - Directory
     inputBinding:
       position: 2

steps/wsclean.cwl

@@ -5,7 +5,7 @@ baseCommand:
   - wsclean
 inputs:
   - id: msin
-    type: 'Directory[]'
+    type: Directory[]
     inputBinding:
       position: 2
       shellQuote: false
@@ -15,7 +15,7 @@ inputs:
     default:
       - 2500
       - 2500
-    type: 'int[]'
+    type: int[]
     inputBinding:
       position: 1
       shellQuote: false
@@ -206,7 +206,7 @@ outputs:
     outputBinding:
       glob: [$(inputs.image_name)-MFS-image.fits, $(inputs.image_name)-image.fits]
   - id: logfile
-    type: 'File[]'
+    type: File[]
     outputBinding:
       glob: 'wsclean*.log'
 label: WSClean