diff --git a/lofar-cwl/steps/DP3.PreflaggerStepGenerator.cwl b/lofar-cwl/steps/DP3.PreflaggerStepGenerator.cwl
index 03f6c11d6f16db4d72a38666e4a294a2f568fb28..64cd85154761c971cb2e86655fbcfe4355a19824 100755
--- a/lofar-cwl/steps/DP3.PreflaggerStepGenerator.cwl
+++ b/lofar-cwl/steps/DP3.PreflaggerStepGenerator.cwl
@@ -133,39 +133,31 @@ inputs:
31*nchan/32..nchan-1] to flag the first and last 2 or 8 channels
(depending on 64 or 256 channels in the observation).
- id: amplmin
- default: -1e+30
- type: float
+ type: float?
doc: >-
Correlation data with amplitude < amplmin will match. It can be given per
correlation. For example, amplmin=[100,,,100] matches data points with XX
or YY amplitude < 100. The non-specified amplitudes get the default value.
- id: amplmax
- default: 1e+30
- type: float
+ type: float?
doc: Correlation data with amplitude > amplmax will match.
- id: phasemin
- default: -1e+30
- type: float
+ type: float?
doc: Correlation data with phase < phasemin (in radians) will match.
- id: phasemax
- default: 1e+30
- type: float
+ type: float?
doc: Correlation data with phase > phasemax (in radians) will match.
- id: realmin
- default: -1e+30
- type: float
+ type: float?
doc: Correlation data with real complex part < realmin will match.
- id: realmax
- default: 1e+30
- type: float
+ type: float?
doc: Correlation data with real complex part > realmax will match.
- id: imagmin
- default: -1e+30
- type: float
+ type: float?
doc: Correlation data with imaginary complex part < imagmin will match.
- id: imagmax
- default: 1e+30
- type: float
+ type: float?
doc: Correlation data with imaginary complex part > imagmax will match.
outputs:
- id: augmented_steps
diff --git a/lofar-cwl/steps/DPPP.cwl b/lofar-cwl/steps/DPPP.cwl
index 47e4cf1f905dab76ace240d3fc3f7b6887901cf4..5dc50b4c1175374b73f3d019b7bb459652c49169 100755
--- a/lofar-cwl/steps/DPPP.cwl
+++ b/lofar-cwl/steps/DPPP.cwl
@@ -41,13 +41,35 @@ inputs:
type: boolean
default: true
inputBinding:
- prefix: -msin.autoweight=True
+ prefix: msin.autoweight=True
- id: baseline
type: string
default: ""
inputBinding:
- prefix: -msin.baseline=
+ prefix: msin.baseline=
separate: false
+ - id: writefullresflag
+ type: boolean
+ default: false
+ inputBinding:
+ prefix: msout.writefullresflag=True
+ - id: overwrite
+ type: boolean
+ default: false
+ inputBinding:
+ prefix: msout.overwrite=True
+ - id: storagemanager
+ type: string
+ default: "Dysco"
+ inputBinding:
+ prefix: msout.storagemanager=
+ separate: false
+ - id: databitrate
+ type: int
+ default: 0
+ inputBinding:
+ prefix: msout.storagemanager.databitrate=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
@@ -89,12 +111,13 @@ outputs:
return output
}
- id: logfile
- type: File?
+ type: File[]
outputBinding:
- glob: DPPP.log
+ glob: 'DPPP*.log'
hints:
- class: DockerRequirement
dockerPull: lofareosc/lofar-pipeline
stdout: DPPP.log
+stderr: DPPP_err.log
requirements:
- class: InlineJavascriptRequirement
\ No newline at end of file
diff --git a/lofar-cwl/steps/ddecal.cwl b/lofar-cwl/steps/ddecal.cwl
index 536b080f050c79d2513487860d55977bfd8cd825..b0493198b5c1380b448c87442e422883786fd52a 100644
--- a/lofar-cwl/steps/ddecal.cwl
+++ b/lofar-cwl/steps/ddecal.cwl
@@ -95,7 +95,7 @@ inputs:
prefix: ddecal.flagdivergedonly=True
- id: storagemanager
type: string
- default: dysco
+ default: ""
inputBinding:
prefix: msout.storagemanager=
separate: false
@@ -134,10 +134,11 @@ outputs:
glob: $(inputs.output_name_h5parm)
- id: logfile
- type: File
+ type: File[]
outputBinding:
- glob: ddecal.log
+ glob: 'ddecal*.log'
stdout: ddecal.log
+stderr: ddecal_err.log
$namespaces:
lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
$schema:
diff --git a/lofar-cwl/steps/predict.cwl b/lofar-cwl/steps/predict.cwl
index 388a03d1f54081aa8b55404abb0c311da198529e..3364b3b5c12359449f87008ab9e3e2632b81e43e 100644
--- a/lofar-cwl/steps/predict.cwl
+++ b/lofar-cwl/steps/predict.cwl
@@ -21,11 +21,6 @@ inputs:
prefix: msin.datacolumn=
separate: false
doc: Input data Column
- - id: storagemanager
- type: string
- default: dysco
- inputBinding:
- prefix: msout.storagemanager=
- default: MODEL_DATA
id: msout_datacolumn
type: string
@@ -54,6 +49,28 @@ inputs:
inputBinding:
position: 0
prefix: predict.usebeammodel=True
+ - id: writefullresflag
+ type: boolean
+ default: false
+ inputBinding:
+ prefix: msout.writefullresflag=True
+ - id: overwrite
+ type: boolean
+ default: false
+ inputBinding:
+ prefix: msout.overwrite=True
+ - id: storagemanager
+ type: string
+ default: "Dysco"
+ inputBinding:
+ prefix: msout.storagemanager=
+ separate: false
+ - id: databitrate
+ type: int
+ default: 0
+ inputBinding:
+ prefix: msout.storagemanager.databitrate=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
@@ -61,9 +78,9 @@ outputs:
outputBinding:
glob: $(inputs.msin.basename)
- id: logfile
- type: File
+ type: File[]
outputBinding:
- glob: predict_cal.log
+ glob: 'predict_cal*.log'
arguments:
- 'steps=[predict]'
- predict.beammode=array_factor
@@ -78,4 +95,5 @@ requirements:
hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator
-stdout: predict_cal.log
\ No newline at end of file
+stdout: predict_cal.log
+stderr: predict_cal_err.log
\ No newline at end of file
diff --git a/subworkflow/apply_calibrate_bp.cwl b/subworkflow/apply_calibrate_bp.cwl
index 0f21d792eef4caecf7dc8a7d568307290ab6c112..357cb86a86429f48f2ce22efc9fdb6eef27b879a 100644
--- a/subworkflow/apply_calibrate_bp.cwl
+++ b/subworkflow/apply_calibrate_bp.cwl
@@ -52,7 +52,7 @@ outputs:
'sbg:y': 0
- id: calib_cal.log
outputSource:
- - calib_cal/logfile
+ - concat_logfiles_calib_cal/output
type: File
'sbg:x': 1000
'sbg:y': 100
@@ -189,4 +189,17 @@ steps:
run: ../lofar-cwl/steps/ddecal.cwl
'sbg:x': 600
'sbg:y': 0
+ - id: concat_logfiles_calib_cal
+ in:
+ - id: file_list
+ source:
+ - calib_cal/logfile
+ - id: file_prefix
+ default: calib_cal
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_calib_cal
+ 'sbg:x': 700
+ 'sbg:y': 200
requirements: []
diff --git a/subworkflow/apply_calibrate_fr.cwl b/subworkflow/apply_calibrate_fr.cwl
index f04c3dd05adbc5ef549589b638acfd53260e02cc..80137502e70affb26ccf8550cd462092298e4540 100644
--- a/subworkflow/apply_calibrate_fr.cwl
+++ b/subworkflow/apply_calibrate_fr.cwl
@@ -46,7 +46,7 @@ outputs:
'sbg:y': 0
- id: calib_cal.log
outputSource:
- - calib_cal/logfile
+ - concat_logfiles_calib_cal/output
type: File
'sbg:x': 1000
'sbg:y': 100
@@ -112,4 +112,17 @@ steps:
run: ../lofar-cwl/steps/ddecal.cwl
'sbg:x': 500
'sbg:y': 0
+ - id: concat_logfiles_calib_cal
+ in:
+ - id: file_list
+ source:
+ - calib_cal/logfile
+ - id: file_prefix
+ default: calib_cal
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_calib_cal
+ 'sbg:x': 700
+ 'sbg:y': 200
requirements: []
diff --git a/subworkflow/apply_calibrate_pa.cwl b/subworkflow/apply_calibrate_pa.cwl
index 02c9d1e5dd717929a51ef68aa00bf6ee3d334b8d..3fe54c8d147d24f08194e14c36fd24834233ecad 100644
--- a/subworkflow/apply_calibrate_pa.cwl
+++ b/subworkflow/apply_calibrate_pa.cwl
@@ -46,7 +46,7 @@ outputs:
'sbg:y': 0
- id: calib_cal.log
outputSource:
- - calib_cal/logfile
+ - concat_logfiles_calib_cal/output
type: File
'sbg:x': 1000
'sbg:y': 100
@@ -143,4 +143,17 @@ steps:
run: ../lofar-cwl/steps/ddecal.cwl
'sbg:x': 500
'sbg:y': 0
+ - id: concat_logfiles_calib_cal
+ in:
+ - id: file_list
+ source:
+ - calib_cal/logfile
+ - id: file_prefix
+ default: calib_cal
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_calib_cal
+ 'sbg:x': 700
+ 'sbg:y': 200
requirements: []
diff --git a/subworkflow/clocktec.cwl b/subworkflow/clocktec.cwl
index 95fdceeb9153c74d3ac2ee2961c6146a50e581cb..102a1481e29d567672146c7eb7cb4c7efd5bed52 100644
--- a/subworkflow/clocktec.cwl
+++ b/subworkflow/clocktec.cwl
@@ -49,12 +49,26 @@ outputs:
'sbg:x': 775
'sbg:y': 169
steps:
- - id: flag
+ - id: duplicateAbkp
in:
- id: input_h5parm
source: input_h5parm
- id: soltab
default: sol000/amplitude000
+ - id: soltabOut
+ default: amplitudeOrig000
+ out:
+ - id: output_h5parm
+ - id: log
+ run: ../lofar-cwl/steps/LoSoTo.Duplicate.cwl
+ 'sbg:x': -542.3984375
+ 'sbg:y': -138.5
+ - id: flag
+ in:
+ - id: input_h5parm
+ source: duplicateAbkp/output_h5parm
+ - id: soltab
+ default: sol000/amplitude000
- id: axesToFlag
default:
- time
diff --git a/subworkflow/ndppp_prep_cal.cwl b/subworkflow/ndppp_prep_cal.cwl
index 7bbc7900b2e594933164e40e0e1a0e0b5187f105..6bcdf446b3a632500708dabb251e23deb0820aa6 100644
--- a/subworkflow/ndppp_prep_cal.cwl
+++ b/subworkflow/ndppp_prep_cal.cwl
@@ -82,8 +82,8 @@ outputs:
'sbg:y': 53.67763900756836
- id: logfile
outputSource:
- - dppp/logfile
- type: File?
+ - concat_logfiles_dppp/output
+ type: File
'sbg:x': 2857.9423828125
'sbg:y': 253.50680541992188
- id: parset
@@ -256,9 +256,16 @@ steps:
source: parsetgenerator/output_file_names
- id: output_directory_names
source: parsetgenerator/output_directory_names
+ - id: autoweight
+ source: raw_data
- id: baseline
- source:
- - msin_baseline
+ source: msin_baseline
+ - id: writefullresflag
+ default: false
+ - id: storagemanager
+ default: Dysco
+ - id: databitrate
+ default: 0
out:
- id: msout
- id: secondary_output_files
@@ -278,13 +285,27 @@ steps:
source: timeresolution
- id: freqresolution
source: freqresolution
+ - id: minpoints
+ default: 1
out:
- id: augmented_steps
run: ../lofar-cwl/steps/DP3.AveragerStepGenerator.cwl
'sbg:x': 2158.297119140625
'sbg:y': -55.93873977661133
+ - id: concat_logfiles_dppp
+ in:
+ - id: file_list
+ source:
+ - dppp/logfile
+ - id: file_prefix
+ default: dppp
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_dppp
+ 'sbg:x': 2759.2265625
+ 'sbg:y': 140.91714477539062
requirements:
- class: SubworkflowFeatureRequirement
- - class: ScatterFeatureRequirement
- class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
\ No newline at end of file
+ - class: InlineJavascriptRequirement
diff --git a/subworkflow/predict_calibrate.cwl b/subworkflow/predict_calibrate.cwl
index 1d78ce1573bce53e559bc797f779ca782f255b88..4186dd11968ae6f8bb8e95aa1bf9f54f4b133562 100644
--- a/subworkflow/predict_calibrate.cwl
+++ b/subworkflow/predict_calibrate.cwl
@@ -40,13 +40,13 @@ outputs:
'sbg:y': -100
- id: predict_cal.log
outputSource:
- - predict/logfile
+ - concat_logfiles_predict/output
type: File
'sbg:x': 1000
'sbg:y': 0
- id: calib_cal.log
outputSource:
- - calib_cal/logfile
+ - concat_logfiles_calib_cal/output
type: File
'sbg:x': 1000
'sbg:y': 100
@@ -74,10 +74,16 @@ steps:
in:
- id: msin
source: BLsmooth/msout
+ - id: msin_datacolumn
+ default: SMOOTHED_DATA
- id: msout_datacolumn
default: MODEL_DATA
- id: sources_db
source: sourcedb
+ - id: storagemanager
+ default: Dysco
+ - id: databitrate
+ default: 0
out:
- id: msout
- id: logfile
@@ -92,10 +98,10 @@ steps:
default: SMOOTHED_DATA
- id: msin_modelcolum
default: MODEL_DATA
- - id: flagunconverged
- source: flagunconverged
- id: propagate_solutions
source: propagatesolutions
+ - id: flagunconverged
+ source: flagunconverged
- id: mode
default: rotation+diagonal
out:
@@ -105,4 +111,30 @@ steps:
run: ../lofar-cwl/steps/ddecal.cwl
'sbg:x': 500
'sbg:y': 0
+ - id: concat_logfiles_predict
+ in:
+ - id: file_list
+ source:
+ - predict/logfile
+ - id: file_prefix
+ default: predict
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_predict
+ 'sbg:x': 500
+ 'sbg:y': 200
+ - id: concat_logfiles_calib_cal
+ in:
+ - id: file_list
+ source:
+ - calib_cal/logfile
+ - id: file_prefix
+ default: calib_cal
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_calib_cal
+ 'sbg:x': 700
+ 'sbg:y': 200
requirements: []
diff --git a/workflows/prefactor_calibrator/fr.cwl b/workflows/prefactor_calibrator/fr.cwl
index b907a0432f6d8ef2f6e70756b254c1d502386c32..f37e36ce5dd09e3a57aa84dd9a33c2215c536d1c 100644
--- a/workflows/prefactor_calibrator/fr.cwl
+++ b/workflows/prefactor_calibrator/fr.cwl
@@ -203,7 +203,7 @@ steps:
- id: plotFlag
default: true
- id: prefix
- default: fr_amp
+ default: fr_amp_
out:
- id: output_plots
- id: logfile
diff --git a/workflows/prefactor_calibrator/pa.cwl b/workflows/prefactor_calibrator/pa.cwl
index 6a6e66cdb35d5672ef963d9ef9ff8c1fa63433b7..75cdef6c7ee75347e935ee431ff0098c2d23a6ee 100644
--- a/workflows/prefactor_calibrator/pa.cwl
+++ b/workflows/prefactor_calibrator/pa.cwl
@@ -200,7 +200,7 @@ steps:
- id: plotFlag
default: true
- id: prefix
- default: polalign_amp
+ default: polalign_amp_
out:
- id: output_plots
- id: logfile
diff --git a/workflows/prefactor_calibrator/prep.cwl b/workflows/prefactor_calibrator/prep.cwl
index 313b14746d9144d1c57e7f01945bf9479c23283b..21f2fbf71a3dce207dc71a0e57bbe2939c05e4d8 100644
--- a/workflows/prefactor_calibrator/prep.cwl
+++ b/workflows/prefactor_calibrator/prep.cwl
@@ -309,8 +309,8 @@ steps:
- id: concat_logfiles_prep_cal
in:
- id: file_list
- source:
- - ndppp_prep_cal/logfile
+ linkMerge: merge_flattened
+ source: ndppp_prep_cal/logfile
- id: file_prefix
default: ndppp_prep_cal
out:
@@ -335,6 +335,7 @@ steps:
- id: concat_logfiles_predict
in:
- id: file_list
+ linkMerge: merge_flattened
source:
- predict_calibrate/predict_cal.log
- id: file_prefix