Bugfix ddecal + minor updates

Docker/Dockerfile

@@ -10,6 +10,9 @@ RUN \
   apt-get update && \
   apt-get install -y vim wget casacore-tools && \
   rm -rf /var/cache/apt 
+RUN \
+  pip3 install --upgrade pip && \
+  pip3 install h5py
 RUN \
   pip3 install --upgrade pip && \
   git clone https://github.com/lofar-astron/prefactor.git ${PREFACTOR_NAME} && \

docs/source/running.rst

@@ -41,9 +41,10 @@ Pipeline options for ``cwltool``
 
 The following ``<cwl_options>`` are recommended to use for running **prefactor** with ``cwltool``:
 
-    * **---outdir**: specifies the location of the pipeline outputs
+    * **---outdir**: specifies the location of the final pipeline output directory (results)
     * **---tmpdir-prefix**: specifies the location of the intermediate data products
     * **---parallel**: jobs will run in parallel
+    * **---singularity**: use Singularity instead of Docker
     * **---no-container**: don't use Docker container (only for manual installation)
     * **---preserve-entire-environment**: use system environment variables (only for manual installation)
     * **---debug**: more verbose output (use only for debugging the pipeline)
@@ -74,6 +75,7 @@ The following ``<cwl_options>`` are recommended to use for running **prefactor**
     * **---stats**: creates runtime statistics
     * **---maxLocalJobs**: amount of local jobs to be run at the same time ("max_per_node")
     * **---retryCount**: amount of retries for each failed pipeline job
+    * **---singularity**: use Singularity instead of Docker
     * **---preserve-entire-environment**: use system environment variables (only for manual installation)
     * **---no-container**: don't use Docker container (only for manual installation)
     

lofar-cwl/steps/ddecal.cwl

@@ -18,14 +18,6 @@ hints:
 
 arguments:
   - steps=[ddecal,count]
-  - ddecal.uvlambdamin=300
-  - ddecal.maxiter=50
-  - ddecal.nchan=1
-  - ddecal.solint=1
-  - ddecal.tolerance=1.e-3
-  - ddecal.usemodelcolumn=True
-  - ddecal.flagdivergedonly=True
-  - ddecal.propagateconvergedonly=True
 
 inputs:
   - id: msin
@@ -71,6 +63,16 @@ inputs:
     default: true
     inputBinding:
       prefix: ddecal.propagatesolutions=True
+  - id: propagate_converged_only
+    type: boolean?
+    default: true
+    inputBinding:
+      prefix: ddecal.propagateconvergedonly=True
+  - id: usemodelcolumn
+    type: boolean?
+    default: true
+    inputBinding:
+      prefix: ddecal.usemodelcolumn=True
   - id: flagunconverged
     type: boolean?
     default: false
@@ -78,9 +80,9 @@ inputs:
       Flag unconverged solutions (i.e., those from solves that did not converge
       within maxiter iterations).
     inputBinding:
-      prefix: ddecal.flagdivergedonly=True
+      prefix: ddecal.flagunconverged=True
   - id: flagdivergedonly
-    default: false
+    default: true
     type: boolean?
     doc: |
       Flag only the unconverged solutions for which divergence was detected.
@@ -119,14 +121,49 @@ inputs:
     inputBinding:
       prefix: ddecal.mode=
       separate: false
+  - id: nchan
+    type: int?
+    default: 1
+    inputBinding:
+      position: 0
+      prefix: ddecal.nchan=
+      separate: false
+  - id: maxiter
+    type: int?
+    default: 50
+    inputBinding:
+      position: 0
+      prefix: ddecal.maxiter=
+      separate: false
+  - id: tolerance
+    type: float?
+    default: 1e-3
+    inputBinding:
+      position: 0
+      prefix: ddecal.tolerance=
+      separate: false
+  - id: uvlambdamin
+    type: int?
+    default: 300
+    inputBinding:
+      position: 0
+      prefix: ddecal.uvlambdamin=
+      separate: false
+  - id: solint
+    type: int?
+    default: 1
+    inputBinding:
+      position: 0
+      prefix: ddecal.solint=
+      separate: false
 
+#--------------------
   - id: save2json
     default: true
     type: boolean?
     inputBinding:
       position: 0
       prefix: count.savetojson=True
-      
   - id: jsonfilename
     type: string?
     default: 'out.json'

lofar-cwl/steps/ms_concat.cwl

@@ -71,3 +71,4 @@ requirements:
     inplaceUpdate: true
   - class: InlineJavascriptRequirement
 stdout: ms_concat.log
+stderr: ms_concat_err.log
\ No newline at end of file

steps/check_unflagged_fraction.cwl

@@ -37,15 +37,19 @@ requirements:
 
           output = check_unflagged_fraction(ms, min_fraction=min_fraction, print_fraction=True)
 
-          filename           = os.path.basename(output['flagged'])
+          filename           = os.path.basename(output['filename'].rstrip('/'))
           unflagged_fraction = output['unflagged_fraction']
-
-          if filename != 'None':
-              shutil.move(filename, filename.lstrip('out_'))
+          flagged            = output['flagged']
 
           cwl_output = {}
           cwl_output['unflagged_fraction'] = unflagged_fraction
 
+          if flagged != 'None':
+             shutil.move(filename, filename.lstrip('out_'))
+             cwl_output['filename'] = 'None'
+          else:
+             cwl_output['filename'] = filename
+
           with open('./out.json', 'w') as fp:
               json.dump(cwl_output, fp)
 
@@ -60,6 +64,12 @@ outputs:
         loadContents: true
         glob: 'out.json'
         outputEval: $(JSON.parse(self[0].contents).unflagged_fraction)
+  - id: filenames
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: 'out.json'
+        outputEval: $(JSON.parse(self[0].contents).filename)
   - id: logfile
     type: File[]
     outputBinding:

steps/findRefAnt_join.cwl

@@ -92,7 +92,7 @@ requirements:
 
             min_flagged_fraction = min(flagged_fraction_list)
             refant = flagged_fraction_data[min_flagged_fraction][0]
-            print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
+            print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(100 * min_flagged_fraction) + '%'))
 
             flagged_fraction_antenna['state'] = inputs['state']
 

steps/summary.cwl

@@ -88,12 +88,37 @@ inputs:
       separate: true
       itemSeparator: ','
       valueFrom: '$(self)'
+  - id: removed_bands
+    default: false
+    type:
+      - string?
+      - boolean?
+    inputBinding:
+      position: 0
+      prefix: '--removed_bands'
+      separate: true
+      itemSeparator: ','
+      valueFrom: '$(self)'
+  - id: min_unflagged_fraction
+    default: 0.5
+    type: float?
+    inputBinding:
+      position: 0
+      prefix: '--min_unflagged'
+    doc: minimum fraction of unflagged data per band to continue
   - id: demix
     default: false
     type: boolean?
     inputBinding:
       position: 0
       prefix: '--demix True'
+  - id: refant
+    default: ''
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--refant'
+    doc: Reference antenna used
   - id: output_fname
     type: 
       - boolean?

workflows/prefactor_target.cwl

@@ -413,7 +413,9 @@ steps:
       - id: Ateam_flags_join_out
       - id: inspection
       - id: logfiles
+      - id: removed_bands
       - id: total_bandwidth
+      - id: out_refant
     run: ./prefactor_target/gsmcal.cwl
     label: gsmcal
     'sbg:x': 900
@@ -461,6 +463,12 @@ steps:
         source: demix_sources
       - id: ncores
         source: ncores
+      - id: removed_bands
+        source: gsmcal/removed_bands
+      - id: min_unflagged_fraction
+        source: min_unflagged_fraction
+      - id: refant
+        source: gsmcal/out_refant
     out:
       - id: msout
       - id: solutions

workflows/prefactor_target/finalize.cwl

@@ -5,6 +5,21 @@ label: finalize
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
+  - id: refant
+    type: string?
+    default: 'CS001HBA0'
+    'sbg:x': -1000
+    'sbg:y': -1000
+  - id: min_unflagged_fraction
+    type: float?
+    default: 0.5
+    'sbg:x': -1000
+    'sbg:y': -900
+  - id: removed_bands
+    type: 'string[]?'
+    default: []
+    'sbg:x': -1000
+    'sbg:y':  -800
   - id: demix_sources
     type: 'string[]?'
     default: []
@@ -276,6 +291,13 @@ steps:
       - id: demix_sources
         source: demix_sources
         valueFrom: "$(self.join(','))"
+      - id: removed_bands
+        source: removed_bands
+        valueFrom: "$(self.join(','))"
+      - id: min_unflagged_fraction
+        source: min_unflagged_fraction
+      - id: refant
+        source: refant
     out:
       - id: summary_file
       - id: logfile

workflows/prefactor_target/gsmcal.cwl

@@ -120,6 +120,12 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 1900
     'sbg:y': 800
+  - id: out_refant
+    outputSource:
+      - findRefAnt_join/refant
+    type: string
+    'sbg:x': 1900
+    'sbg:y': 900
   - id: logfiles
     outputSource:
       - concat_logfiles_identify/output
@@ -137,6 +143,12 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 1900
     'sbg:y': 1000
+  - id: removed_bands
+    outputSource:
+      - check_unflagged_fraction/filenames
+    type: 'string[]'
+    'sbg:x': 1900
+    'sbg:y': 1100
   - id: total_bandwidth
     outputSource:
       - sort_times_into_freqGroups/total_bandwidth
@@ -434,6 +446,7 @@ steps:
     out:
       - id: msout
       - id: unflagged_fraction
+      - id: filenames
       - id: logfile
     run: ../../steps/check_unflagged_fraction.cwl
     label: check_unflagged_fraction