bugfix for PA

Former-commit-id: 9d21e566
Former-commit-id: 71c2070b

lofar-cwl/steps/DP3.PreflaggerStepGenerator.cwl

@@ -133,39 +133,31 @@ inputs:
       31*nchan/32..nchan-1] to flag the first and last 2 or 8 channels
       (depending on 64 or 256 channels in the observation).
   - id: amplmin
-    default: -1e+30
-    type: float
+    type: float?
     doc: >-
       Correlation data with amplitude < amplmin will match. It can be given per
       correlation. For example, amplmin=[100,,,100] matches data points with XX
       or YY amplitude < 100. The non-specified amplitudes get the default value.
   - id: amplmax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with amplitude > amplmax will match.
   - id: phasemin
-    default: -1e+30
-    type: float
+    type: float?
     doc: Correlation data with phase < phasemin (in radians) will match.
   - id: phasemax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with phase > phasemax (in radians) will match.
   - id: realmin
-    default: -1e+30
-    type: float
+    type: float?
     doc: Correlation data with real complex part < realmin will match.
   - id: realmax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with real complex part > realmax will match.
   - id: imagmin
-    default: -1e+30
-    type: float
+    type: float?
     doc: Correlation data with imaginary complex part < imagmin will match.
   - id: imagmax
-    default: 1e+30
-    type: float
+    type: float?
     doc: Correlation data with imaginary complex part > imagmax will match.
 outputs:
 - id: augmented_steps

lofar-cwl/steps/DPPP.cwl

@@ -41,12 +41,34 @@ inputs:
     type: boolean
     default: true
     inputBinding:
-      prefix: -msin.autoweight=True
+      prefix: msin.autoweight=True
   - id: baseline
     type: string
     default: ""
     inputBinding:
-      prefix: -msin.baseline=
+      prefix: msin.baseline=
+      separate: false
+  - id: writefullresflag
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.writefullresflag=True
+  - id: overwrite
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.overwrite=True
+  - id: storagemanager
+    type: string
+    default: "Dysco"
+    inputBinding:
+       prefix: msout.storagemanager=
+       separate: false
+  - id: databitrate
+    type: int
+    default: 0
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
        separate: false
 outputs:
   - id: msout
@@ -89,12 +111,13 @@ outputs:
           return output
         }
   - id: logfile
-    type: File?
+    type: File[]
     outputBinding:
-      glob: DPPP.log
+      glob: 'DPPP*.log'
 hints:
   - class: DockerRequirement
     dockerPull: lofareosc/lofar-pipeline
 stdout: DPPP.log
+stderr: DPPP_err.log
 requirements:
   - class: InlineJavascriptRequirement
\ No newline at end of file

lofar-cwl/steps/ddecal.cwl

@@ -95,7 +95,7 @@ inputs:
       prefix: ddecal.flagdivergedonly=True
   - id: storagemanager
     type: string
-    default: dysco
+    default: ""
     inputBinding:
       prefix: msout.storagemanager=
       separate: false
@@ -134,10 +134,11 @@ outputs:
       glob: $(inputs.output_name_h5parm)
       
   - id: logfile
-    type: File
+    type: File[]
     outputBinding:
-      glob: ddecal.log     
+      glob: 'ddecal*.log'
 stdout: ddecal.log
+stderr: ddecal_err.log
 $namespaces:
   lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
 $schema:

lofar-cwl/steps/predict.cwl

@@ -21,11 +21,6 @@ inputs:
       prefix: msin.datacolumn=
       separate: false
     doc: Input data Column
-  - id: storagemanager
-    type: string
-    default: dysco
-    inputBinding:
-      prefix: msout.storagemanager=
   - default: MODEL_DATA
     id: msout_datacolumn
     type: string
@@ -54,6 +49,28 @@ inputs:
     inputBinding:
       position: 0
       prefix: predict.usebeammodel=True
+  - id: writefullresflag
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.writefullresflag=True
+  - id: overwrite
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.overwrite=True
+  - id: storagemanager
+    type: string
+    default: "Dysco"
+    inputBinding:
+       prefix: msout.storagemanager=
+       separate: false
+  - id: databitrate
+    type: int
+    default: 0
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
+       separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set
@@ -61,9 +78,9 @@ outputs:
     outputBinding:
       glob: $(inputs.msin.basename)
   - id: logfile
-    type: File
+    type: File[]
     outputBinding:
-      glob: predict_cal.log
+      glob: 'predict_cal*.log'
 arguments:
   - 'steps=[predict]'
   - predict.beammode=array_factor
@@ -79,3 +96,4 @@ hints:
   - class: DockerRequirement
     dockerPull: lofareosc/prefactor:HBAcalibrator
 stdout: predict_cal.log
+stderr: predict_cal_err.log
\ No newline at end of file

subworkflow/apply_calibrate_bp.cwl

@@ -52,7 +52,7 @@ outputs:
     'sbg:y': 0
   - id: calib_cal.log
     outputSource:
-      - calib_cal/logfile
+      - concat_logfiles_calib_cal/output
     type: File
     'sbg:x': 1000
     'sbg:y': 100
@@ -189,4 +189,17 @@ steps:
     run: ../lofar-cwl/steps/ddecal.cwl
     'sbg:x': 600
     'sbg:y': 0
+  - id: concat_logfiles_calib_cal
+    in:
+      - id: file_list
+        source:
+          - calib_cal/logfile
+      - id: file_prefix
+        default: calib_cal
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_calib_cal
+    'sbg:x': 700
+    'sbg:y': 200
 requirements: []

subworkflow/apply_calibrate_fr.cwl

@@ -46,7 +46,7 @@ outputs:
     'sbg:y': 0
   - id: calib_cal.log
     outputSource:
-      - calib_cal/logfile
+      - concat_logfiles_calib_cal/output
     type: File
     'sbg:x': 1000
     'sbg:y': 100
@@ -112,4 +112,17 @@ steps:
     run: ../lofar-cwl/steps/ddecal.cwl
     'sbg:x': 500
     'sbg:y': 0
+  - id: concat_logfiles_calib_cal
+    in:
+      - id: file_list
+        source:
+          - calib_cal/logfile
+      - id: file_prefix
+        default: calib_cal
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_calib_cal
+    'sbg:x': 700
+    'sbg:y': 200
 requirements: []

subworkflow/apply_calibrate_pa.cwl

@@ -46,7 +46,7 @@ outputs:
     'sbg:y': 0
   - id: calib_cal.log
     outputSource:
-      - calib_cal/logfile
+      - concat_logfiles_calib_cal/output
     type: File
     'sbg:x': 1000
     'sbg:y': 100
@@ -143,4 +143,17 @@ steps:
     run: ../lofar-cwl/steps/ddecal.cwl
     'sbg:x': 500
     'sbg:y': 0
+  - id: concat_logfiles_calib_cal
+    in:
+      - id: file_list
+        source:
+          - calib_cal/logfile
+      - id: file_prefix
+        default: calib_cal
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_calib_cal
+    'sbg:x': 700
+    'sbg:y': 200
 requirements: []

subworkflow/clocktec.cwl

@@ -49,12 +49,26 @@ outputs:
     'sbg:x': 775
     'sbg:y': 169
 steps:
-  - id: flag
+  - id: duplicateAbkp
     in:
       - id: input_h5parm
         source: input_h5parm
       - id: soltab
         default: sol000/amplitude000
+      - id: soltabOut
+        default: amplitudeOrig000
+    out:
+      - id: output_h5parm
+      - id: log
+    run: ../lofar-cwl/steps/LoSoTo.Duplicate.cwl
+    'sbg:x': -542.3984375
+    'sbg:y': -138.5
+  - id: flag
+    in:
+      - id: input_h5parm
+        source: duplicateAbkp/output_h5parm
+      - id: soltab
+        default: sol000/amplitude000
       - id: axesToFlag
         default:
           - time

subworkflow/ndppp_prep_cal.cwl

@@ -82,8 +82,8 @@ outputs:
     'sbg:y': 53.67763900756836
   - id: logfile
     outputSource:
-      - dppp/logfile
-    type: File?
+      - concat_logfiles_dppp/output
+    type: File
     'sbg:x': 2857.9423828125
     'sbg:y': 253.50680541992188
   - id: parset
@@ -256,9 +256,16 @@ steps:
         source: parsetgenerator/output_file_names
       - id: output_directory_names
         source: parsetgenerator/output_directory_names
+      - id: autoweight
+        source: raw_data
       - id: baseline
-        source: 
-          - msin_baseline
+        source: msin_baseline
+      - id: writefullresflag
+        default: false
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        default: 0
     out:
       - id: msout
       - id: secondary_output_files
@@ -278,13 +285,27 @@ steps:
         source: timeresolution
       - id: freqresolution
         source: freqresolution
+      - id: minpoints
+        default: 1
     out:
       - id: augmented_steps
     run: ../lofar-cwl/steps/DP3.AveragerStepGenerator.cwl
     'sbg:x': 2158.297119140625
     'sbg:y': -55.93873977661133
+  - id: concat_logfiles_dppp
+    in:
+      - id: file_list
+        source:
+          - dppp/logfile
+      - id: file_prefix
+        default: dppp
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_dppp
+    'sbg:x': 2759.2265625
+    'sbg:y': 140.91714477539062
 requirements:
   - class: SubworkflowFeatureRequirement
-  - class: ScatterFeatureRequirement
   - class: StepInputExpressionRequirement
   - class: InlineJavascriptRequirement

subworkflow/predict_calibrate.cwl

@@ -40,13 +40,13 @@ outputs:
     'sbg:y': -100
   - id: predict_cal.log
     outputSource:
-      - predict/logfile
+      - concat_logfiles_predict/output
     type: File
     'sbg:x': 1000
     'sbg:y': 0
   - id: calib_cal.log
     outputSource:
-      - calib_cal/logfile
+      - concat_logfiles_calib_cal/output
     type: File
     'sbg:x': 1000
     'sbg:y': 100
@@ -74,10 +74,16 @@ steps:
     in:
       - id: msin
         source: BLsmooth/msout
+      - id: msin_datacolumn
+        default: SMOOTHED_DATA
       - id: msout_datacolumn
         default: MODEL_DATA
       - id: sources_db
         source: sourcedb
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        default: 0
     out:
       - id: msout
       - id: logfile
@@ -92,10 +98,10 @@ steps:
         default: SMOOTHED_DATA
       - id: msin_modelcolum
         default: MODEL_DATA
-      - id: flagunconverged
-        source: flagunconverged
       - id: propagate_solutions
         source: propagatesolutions
+      - id: flagunconverged
+        source: flagunconverged
       - id: mode
         default: rotation+diagonal
     out:
@@ -105,4 +111,30 @@ steps:
     run: ../lofar-cwl/steps/ddecal.cwl
     'sbg:x': 500
     'sbg:y': 0
+  - id: concat_logfiles_predict
+    in:
+      - id: file_list
+        source:
+          - predict/logfile
+      - id: file_prefix
+        default: predict
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_predict
+    'sbg:x': 500
+    'sbg:y': 200
+  - id: concat_logfiles_calib_cal
+    in:
+      - id: file_list
+        source:
+          - calib_cal/logfile
+      - id: file_prefix
+        default: calib_cal
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_calib_cal
+    'sbg:x': 700
+    'sbg:y': 200
 requirements: []

workflows/prefactor_calibrator/fr.cwl

@@ -203,7 +203,7 @@ steps:
       - id: plotFlag
         default: true
       - id: prefix
-        default: fr_amp
+        default: fr_amp_
     out:
       - id: output_plots
       - id: logfile

workflows/prefactor_calibrator/pa.cwl

@@ -200,7 +200,7 @@ steps:
       - id: plotFlag
         default: true
       - id: prefix
-        default: polalign_amp
+        default: polalign_amp_
     out:
       - id: output_plots
       - id: logfile

workflows/prefactor_calibrator/prep.cwl

@@ -309,8 +309,8 @@ steps:
   - id: concat_logfiles_prep_cal
     in:
       - id: file_list
-        source:
-          - ndppp_prep_cal/logfile
+        linkMerge: merge_flattened
+        source: ndppp_prep_cal/logfile
       - id: file_prefix
         default: ndppp_prep_cal
     out:
@@ -335,6 +335,7 @@ steps:
   - id: concat_logfiles_predict
     in:
       - id: file_list
+        linkMerge: merge_flattened
         source:
           - predict_calibrate/predict_cal.log
       - id: file_prefix