Fix plot demix

scripts/plot_demixing_solutions.py

@@ -16,6 +16,23 @@ import matplotlib as mpl
 mpl.use('Agg')
 import matplotlib.pyplot as plt
 
+def input2strlist_nomapfile(invar):
+   """ 
+   from bin/download_IONEX.py
+   give the list of MSs from the list provided as a string
+   """
+   str_list = None
+   if type(invar) is str:
+       if invar.startswith('[') and invar.endswith(']'):
+           str_list = [f.strip(' \'\"') for f in invar.strip('[]').split(',')]
+       else:
+           str_list = [invar.strip(' \'\"')]
+   elif type(invar) is list:
+       str_list = [str(f).strip(' \'\"') for f in invar]
+   else:
+       raise TypeError('input2strlist: Type '+str(type(invar))+' unknown!')
+   return str_list
+
 def globalize(func):
   def result(*args, **kwargs):
     return func(*args, **kwargs)
@@ -26,7 +43,7 @@ def globalize(func):
 
 class PlotGenerator:
     def __init__(self,in_soltables,clipval):
-        self.in_soltab=sorted(glob.glob(in_soltables))
+        self.in_soltab=in_soltables
         self.clipval=clipval
         # initialize values
         self.N=0
@@ -35,6 +52,7 @@ class PlotGenerator:
         self.T=0
         self.F=0
         self.directions=None
+        self.stations=None
 
         self.Nparallel=4
         self.Jnorm=None
@@ -50,10 +68,11 @@ class PlotGenerator:
         self.F=vl.shape[2]
         sol_names_tab=ctab.table(tt.getkeyword('NAMES'),readonly=True)
         sol_names=sol_names_tab.getcol('NAME')
-        self.directions=self.get_directions(sol_names)
+        self.directions,self.stations=self.get_directions(sol_names)
         self.directions.reverse()
         self.K=len(self.directions)
         self.N=n_prod//(8*self.K)
+        assert(self.N==len(self.stations))
         print(f'Processing {self.B} subbands, {self.N} stations, {self.K} directions, time {self.T} freq {self.F}')
         tt.close()
 
@@ -105,33 +124,44 @@ class PlotGenerator:
             for cj in range(ncols):
                 ck=ci*ncols+cj
                 if ck<self.N:
-                  im=axs[ci,cj].imshow(self.Jnorm[ndir,ck],interpolation=None,aspect='auto')
+                  if ncols > 1 :
+                    im=axs[ci,cj].imshow(self.Jnorm[ndir,ck],interpolation=None,aspect='auto')
+                    axs[ci,cj].text(0,0,self.stations[ck])
+                  else :
+                    im=axs[ci].imshow(self.Jnorm[ndir,ck],interpolation=None,aspect='auto')
+                    axs[ci].text(0,0,self.stations[ck])
                 else:
-                  axs[ci,cj].axis('off')
+                  if ncols > 1 :
+                    axs[ci,cj].axis('off')
+                  else:
+                    axs[ci].axis('off')
 
           for ax in fig.get_axes():
             ax.label_outer()
           cb_ax=fig.add_axes([0.9, 0.1, 0.02, 0.8])
           cbar=fig.colorbar(im,cax=cb_ax)
+          cbar.set_label('solution norm')
           fig.suptitle('direction '+str(self.directions[ndir]))
           fig.supxlabel('subbands')
           fig.supylabel('time')
-          plt.savefig(os.path.basename(self.in_soltab[0])+'_dir_'+str(ndir)+'.png')
+          plt.savefig(os.path.basename(self.in_soltab[0])+'_'+str(self.directions[ndir])+'.png')
           plt.close()
 
     def get_directions(self,solution_names):
         """
          solution_names will include all solutions,
          like 'DirectionGain:x:y:Real/Imag:Station:Direction'
-         parse this and find unique direction names
+         parse this and find unique direction names and unique station names
         """
         sourcenames=[]
+        stationnames=[]
         for solname in solution_names:
            sourcenames.append(solname.split(':')[-1])
-        return list(set(sourcenames))
+           stationnames.append(solname.split(':')[-2])
+        return list(set(sourcenames)),list(set(stationnames))
 
 def main(args):
-    pg=PlotGenerator(args.instrument_tables,args.clip)
+    pg=PlotGenerator(input2strlist_nomapfile(args.instrument_tables),args.clip)
     pg.read_solutions()
     pg.plot_solutions()
 

steps/dp3_make_parset_cal.cwl

@@ -113,7 +113,7 @@ requirements:
           demix.ntimechunk                    =   $(inputs.ntimechunk)
           demix.freqstep                      =   1
           demix.timestep                      =   1
-          demix.instrumentmodel               =   instrument
+          demix.instrumentmodel               =   demix_solutions
           demix.uselbfgssolver                =   True
           demix.lbfgs.historysize             =   $(inputs.lbfgs_historysize)
           demix.lbfgs.robustdof               =   $(inputs.lbfgs_robustdof)

steps/dp3_make_parset_target.cwl

@@ -167,7 +167,7 @@ requirements:
           demix.ntimechunk                    =   $(inputs.ntimechunk)
           demix.freqstep                      =   1
           demix.timestep                      =   1
-          demix.instrumentmodel               =   instrument
+          demix.instrumentmodel               =   demix_solutions
           demix.uselbfgssolver                =   True
           demix.lbfgs.historysize             =   $(inputs.lbfgs_historysize)
           demix.lbfgs.robustdof               =   $(inputs.lbfgs_robustdof)

steps/dp3_prep_cal.cwl

@@ -104,6 +104,11 @@ outputs:
     type: Directory
     outputBinding:
       glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+  - id: instrument_table
+    doc: Output Measurement Set
+    type: Directory?
+    outputBinding:
+      glob: 'demix_solutions'
   - id: flagged_fraction_dict
     type: string
     outputBinding:

steps/dp3_prep_target.cwl

@@ -154,6 +154,11 @@ outputs:
     type: Directory
     outputBinding:
       glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+  - id: instrument_table
+    doc: Output Measurement Set
+    type: Directory?
+    outputBinding:
+      glob: 'demix_solutions'
   - id: flagged_fraction_dict_initial
     type: string
     outputBinding:

steps/plot_demix.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: plot_demix
+baseCommand:
+  - plot_demixing_solutions.py
+inputs:
+  - id: instrument_tables
+    type: Directory[]?
+    inputBinding:
+      position: 0
+      prefix: '--instrument_tables'
+      itemSeparator: ','
+      valueFrom: "[$(self.map(function(directory){ return directory.path; }).join(','))]"
+  - id: clip_solutions
+    type: float?
+
+label: plot_demix_solutions
+
+outputs:
+  - id: demix_images
+    doc: Output image
+    type: File[]
+    outputBinding:
+      glob: 'demix_solutions_*.png'
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'plot_demix_solutions*.log'
+
+hints:
+  - class: DockerRequirement
+    dockerPull: 'astronrd/linc'
+requirements:
+  - class: InlineJavascriptRequirement
+stdout: plot_demix_solutions.log
+stderr: plot_demix_solutions_err.log

workflows/linc_calibrator.cwl

@@ -138,7 +138,7 @@ outputs:
   - id: inspection
     linkMerge: merge_flattened
     outputSource:
-      - prep/check_Ateam_separation.png
+      - prep/inspection
       - pa/inspection
       - fr/inspection
       - bp/inspection
@@ -235,7 +235,7 @@ steps:
       - id: outh5parm
       - id: logfiles
       - id: outh5parm_logfile
-      - id: check_Ateam_separation.png
+      - id: inspection
       - id: check_Ateam_separation.json
       - id: flagged_fraction_dict_refant
       - id: flagged_fraction_dict

workflows/linc_calibrator/dp3_prep_cal.cwl

@@ -76,6 +76,10 @@ outputs:
     outputSource:
       - dp3_execute/flagged_fraction_dict
     type: string
+  - id: instrument_tables
+    outputSource:
+      - dp3_execute/instrument_table
+    type: Directory?
 steps:
   - id: define_parset
     in:
@@ -146,6 +150,7 @@ steps:
         source: skymodel
     out:
       - id: msout
+      - id: instrument_table
       - id: flagged_fraction_dict
       - id: logfile
     run: ../../steps/dp3_prep_cal.cwl

workflows/linc_calibrator/prep.cwl

@@ -88,10 +88,13 @@ outputs:
     outputSource:
       - h5parm_collector/outh5parm
     type: File
-  - id: check_Ateam_separation.png
+  - id: inspection
     outputSource:
       - check_ateam_separation/output_imag
-    type: File
+      - plot_demix/demix_images
+    type: File[]
+    linkMerge: merge_flattened
+    pickValue: all_non_null
   - id: msout
     outputSource: predict_calibrate/msout
     type: Directory[]
@@ -126,8 +129,10 @@ outputs:
       - check_ateam_separation/logfile
       - check_demix/logfile
       - aoflag/logfile
+      - concat_logfiles_plot_demix/output
     type: File[]
     linkMerge: merge_flattened
+    pickValue: all_non_null
 steps:
   - id: select
     in:
@@ -193,6 +198,7 @@ steps:
         source: lbfgs_robustdof
     out:
       - id: msout
+      - id: instrument_tables
       - id: flagged_fraction_dict
       - id: logfile
     run: ./dp3_prep_cal.cwl
@@ -277,6 +283,18 @@ steps:
       - id: logfile
     run: ../../steps/check_demix.cwl
     label: check_demix
+  - id: plot_demix
+    in:
+      - id: instrument_tables
+        source: dp3_prep_cal/instrument_tables
+      - id: demix
+        source: check_demix/out_demix
+    out:
+      - id: demix_images
+      - id: logfile
+    run: ../../steps/plot_demix.cwl
+    label: plot_demix_solutions
+    when: $(inputs.demix)
   - id: find_skymodel_cal
     in:
       - id: msin
@@ -361,6 +379,21 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_calib
+  - id: concat_logfiles_plot_demix
+    in:
+      - id: file_list
+        source:
+          - plot_demix/logfile
+        pickValue: all_non_null
+      - id: file_prefix
+        default: plot_demix_solutions
+      - id: demix
+        source: check_demix/out_demix
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_plot_demix
+    when: $(inputs.demix)
   - id: h5parm_collector
     in:
       - id: h5parmFiles

workflows/linc_target/dp3_prep_targ.cwl

@@ -126,6 +126,10 @@ outputs:
       - Ateamclipper/output
     type: File
     pickValue: all_non_null
+  - id: instrument_tables
+    outputSource:
+      - dp3_execute/instrument_table
+    type: Directory?
 steps:
   - id: define_parset
     in:
@@ -253,6 +257,7 @@ steps:
         source: skymodel
     out:
       - id: msout
+      - id: instrument_table
       - id: flagged_fraction_dict_initial
       - id: flagged_fraction_dict_prep
       - id: logfile

workflows/linc_target/prep.cwl

@@ -145,6 +145,7 @@ outputs:
       - check_ateam_separation/output_imag
       - losoto_plot_RM/output_plots
       - plot_Ateamclipper/output_imag
+      - plot_demix/demix_images
     type: File[]
     linkMerge: merge_flattened
     pickValue: all_non_null
@@ -168,6 +169,7 @@ outputs:
       - concat_logfiles_prep_targ/output
       - concat_logfiles_predict_targ/output
       - concat_logfiles_clipper_targ/output
+      - concat_logfiles_plot_demix/output
     type: File[]
     linkMerge: merge_flattened
     pickValue: all_non_null
@@ -478,6 +480,7 @@ steps:
       - id: clipper_logfile
       - id: msout
       - id: clipper_output
+      - id: instrument_tables
     run: ./dp3_prep_targ.cwl
     label: dp3_prep_target
     scatter:
@@ -493,6 +496,18 @@ steps:
     run: ../../steps/plot_Ateamclipper.cwl
     when: $(inputs.execute)
     label: concat_logfiles_clipper_output
+  - id: plot_demix
+    in:
+      - id: instrument_tables
+        source: dp3_prep_target/instrument_tables
+      - id: demix
+        source: check_demix/out_demix
+    out:
+      - id: demix_images
+      - id: logfile
+    run: ../../steps/plot_demix.cwl
+    label: plot_demix_solutions
+    when: $(inputs.demix)
   - id: concat_logfiles_clipper_output
     in:
       - id: file_list
@@ -590,6 +605,21 @@ steps:
       - id: output
     run: ../../steps/concatenate_files.cwl
     label: concat_logfiles_skymodels
+  - id: concat_logfiles_plot_demix
+    in:
+      - id: file_list
+        source:
+          - plot_demix/logfile
+        pickValue: all_non_null
+      - id: file_prefix
+        default: plot_demix_solutions
+      - id: demix
+        source: check_demix/out_demix
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_plot_demix
+    when: $(inputs.demix)
 requirements:
   - class: SubworkflowFeatureRequirement
   - class: ScatterFeatureRequirement