diff --git a/Docker/build_docker.sh b/Docker/build_docker.sh
index 23a6f12e00727094e750c856b4bb37b41beb4d34..10b028d69a30adb851f9a0fb90baf9fb664d613f 100755
--- a/Docker/build_docker.sh
+++ b/Docker/build_docker.sh
@@ -3,8 +3,8 @@ set -e
DOCKER_TAG=latest
-#docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
-#docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
+docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
+docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
docker build --no-cache ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}
diff --git a/lofar-cwl/steps/DPPP.cwl b/lofar-cwl/steps/DPPP.cwl
index 40e8ccc0936d71d23046f8a574c5592641041b89..c543854e95a0424e789aaf5e8f74033e2c5802ab 100755
--- a/lofar-cwl/steps/DPPP.cwl
+++ b/lofar-cwl/steps/DPPP.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: dppp
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: parset
type: File?
diff --git a/lofar-cwl/steps/applybeam.cwl b/lofar-cwl/steps/applybeam.cwl
index 14e02ccb29528fd18007654bf2a1d130ac89d029..e08f8c9d8310abf58c22d28a5a21b6114623c284 100644
--- a/lofar-cwl/steps/applybeam.cwl
+++ b/lofar-cwl/steps/applybeam.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: applybeam
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
diff --git a/lofar-cwl/steps/applycal.cwl b/lofar-cwl/steps/applycal.cwl
index 7ab1a0de4e1936163a47186c87c6a498e83a70ff..030674739721ead2f56f90cdc726f842f1e92629 100644
--- a/lofar-cwl/steps/applycal.cwl
+++ b/lofar-cwl/steps/applycal.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: applycal
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
diff --git a/lofar-cwl/steps/applytarget.cwl b/lofar-cwl/steps/applytarget.cwl
index 3ec0fdd37a64525aae00d5ead4803f49191348a0..027601d74763f16ec25fd856a610b9c3a16d2b03 100644
--- a/lofar-cwl/steps/applytarget.cwl
+++ b/lofar-cwl/steps/applytarget.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: applytarget
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
@@ -80,12 +80,30 @@ inputs:
position: 0
prefix: applycal.updateweights=
separate: false
+ - id: save2json
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count.savetojson=True
+ - id: jsonfilename
+ type: string?
+ default: 'out.json'
+ inputBinding:
+ prefix: count.jsonfilename=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
glob: '$(inputs.msout_name=="."?inputs.msin.basename:inputs.msout_name)'
+ - id: flagged_fraction_dict
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: $(inputs.jsonfilename)
+ outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
- id: logfile
type: File[]
outputBinding:
diff --git a/lofar-cwl/steps/average.cwl b/lofar-cwl/steps/average.cwl
index 06f5e0b7ac6314ebdbebeea7bbfb69b80a6b563f..49ed492f8699f0279e2f2e9df26b01bb64c75452 100644
--- a/lofar-cwl/steps/average.cwl
+++ b/lofar-cwl/steps/average.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: average
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
diff --git a/lofar-cwl/steps/calib_rot_diag.cwl b/lofar-cwl/steps/calib_rot_diag.cwl
index fd35fe5cbec0c3a4069379bb600e1b7ad20db5a4..fe61197b9faf6cbb03360475f21a22762479c333 100644
--- a/lofar-cwl/steps/calib_rot_diag.cwl
+++ b/lofar-cwl/steps/calib_rot_diag.cwl
@@ -5,7 +5,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: calib_rot_diag
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory?
@@ -37,7 +37,7 @@ inputs:
position: 0
prefix: ddecal.h5parm=
separate: false
- - default: out.MS
+ - default: '.'
id: msout_name
type: string
inputBinding:
diff --git a/lofar-cwl/steps/ddecal.cwl b/lofar-cwl/steps/ddecal.cwl
index 4f1a9c390ab6040e087db745a28469fb0a69644a..e89280ec948302705401a758cf3fe62dbebc816d 100644
--- a/lofar-cwl/steps/ddecal.cwl
+++ b/lofar-cwl/steps/ddecal.cwl
@@ -3,7 +3,7 @@
class: CommandLineTool
cwlVersion: v1.1
id: calib_rot_diag
-baseCommand: [DPPP]
+baseCommand: [DP3]
requirements:
InlineJavascriptRequirement: {}
@@ -60,7 +60,7 @@ inputs:
- id: msout_name
type: string?
doc: Output Measurement Set
- default: "OUT.MS"
+ default: '.'
inputBinding:
prefix: msout=
separate: false
@@ -119,6 +119,21 @@ inputs:
inputBinding:
prefix: ddecal.mode=
separate: false
+
+ - id: save2json
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count.savetojson=True
+
+ - id: jsonfilename
+ type: string?
+ default: 'out.json'
+ inputBinding:
+ prefix: count.jsonfilename=
+ separate: false
+
outputs:
- id: msout
doc: Output Measurement Set
@@ -133,6 +148,13 @@ outputs:
outputBinding:
glob: $(inputs.output_name_h5parm)
+ - id: flagged_fraction_dict
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: $(inputs.jsonfilename)
+ outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
+
- id: logfile
type: File[]
outputBinding:
diff --git a/lofar-cwl/steps/dpppconcat.cwl b/lofar-cwl/steps/dpppconcat.cwl
index 9fd3604f2f228e2f76e3e273325605ed7995430a..91e257cd3532400ca895c58e534b752a434b5d53 100644
--- a/lofar-cwl/steps/dpppconcat.cwl
+++ b/lofar-cwl/steps/dpppconcat.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: dpppconcat
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: 'Directory[]'
@@ -118,12 +118,30 @@ inputs:
shellQuote: false
position: 0
default: 12.205kHz
+ - id: save2json
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count.savetojson=True
+ - id: jsonfilename
+ type: string?
+ default: 'out.json'
+ inputBinding:
+ prefix: count.jsonfilename=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
glob: $(inputs.msout_name)
+ - id: flagged_fraction_dict
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: $(inputs.jsonfilename)
+ outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
- id: logfile
type: 'File[]'
outputBinding:
diff --git a/lofar-cwl/steps/filter_predict.cwl b/lofar-cwl/steps/filter_predict.cwl
index e069f929ba6f07f760bc1d0ca575774bc34c83ce..7ce337c5a66af3c6493a53d55bb706cd208f284b 100644
--- a/lofar-cwl/steps/filter_predict.cwl
+++ b/lofar-cwl/steps/filter_predict.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: predict
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
@@ -29,7 +29,9 @@ inputs:
prefix: msout.datacolumn=
separate: false
- id: sources_db
- type: File
+ type:
+ - File
+ - Directory
inputBinding:
position: 0
prefix: predict.sourcedb=
diff --git a/lofar-cwl/steps/gaincal.cwl b/lofar-cwl/steps/gaincal.cwl
index 1cccdaef17ef7abe6ab256f9b32b476717d4e438..f041c924771f548b51721663bfe78539eb1c6326 100644
--- a/lofar-cwl/steps/gaincal.cwl
+++ b/lofar-cwl/steps/gaincal.cwl
@@ -5,7 +5,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: gaincal
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
@@ -34,7 +34,9 @@ inputs:
doc: |
The type of calibration that needs to be performed.
- id: sourcedb
- type: File
+ type:
+ - File
+ - Directory
inputBinding:
position: 0
prefix: gaincal.sourcedb=
@@ -89,7 +91,7 @@ inputs:
position: 0
prefix: gaincal.parmdb=
separate: false
- - default: out.MS
+ - default: '.'
id: msout_name
type: string?
inputBinding:
diff --git a/lofar-cwl/steps/interpolate_visibilities.cwl b/lofar-cwl/steps/interpolate_visibilities.cwl
index 78ad7078e8e3361074a12bcac24c805d93567f24..9e27870f7e2e7a243bd1795203ff86ba73d21a5b 100644
--- a/lofar-cwl/steps/interpolate_visibilities.cwl
+++ b/lofar-cwl/steps/interpolate_visibilities.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: interpolate_visibilities
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
diff --git a/lofar-cwl/steps/predict.cwl b/lofar-cwl/steps/predict.cwl
index 44e5c96b927667d1c78c7543466654acf66b8b88..005156f82ea813e478a99a4b601c9d73a04ad184 100644
--- a/lofar-cwl/steps/predict.cwl
+++ b/lofar-cwl/steps/predict.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: predict
baseCommand:
- - DPPP
+ - DP3
inputs:
- id: msin
type: Directory
@@ -29,7 +29,9 @@ inputs:
prefix: msout.datacolumn=
separate: false
- id: sources_db
- type: File
+ type:
+ - File
+ - Directory
inputBinding:
position: 0
prefix: predict.sourcedb=
diff --git a/lofar-cwl/steps/wsclean.cwl b/lofar-cwl/steps/wsclean.cwl
index 22e5dd97078679bbf060896537dcdaffa2d11d36..67b5dcf024af18d45d6a3ef810d035ca1ae2a17f 100644
--- a/lofar-cwl/steps/wsclean.cwl
+++ b/lofar-cwl/steps/wsclean.cwl
@@ -15,8 +15,8 @@ inputs:
valueFrom: $(concatenate_path_wsclean(self))
- id: image_size
default:
- - 1000
- - 1000
+ - 2500
+ - 2500
type: 'int[]'
inputBinding:
position: 1
@@ -30,7 +30,7 @@ inputs:
prefix: '-scale'
shellQuote: false
- id: niter
- default: 10000
+ default: 1000
type: int?
inputBinding:
position: 1
diff --git a/steps/createRMh5parm.cwl b/steps/createRMh5parm.cwl
index 60de7379922aec4b15f418dcbeb5e83991010390..a18c99e8b6f0abfdb667d2b8d31fd55214e652f4 100644
--- a/steps/createRMh5parm.cwl
+++ b/steps/createRMh5parm.cwl
@@ -113,7 +113,9 @@ hints:
listing:
- entry: $(inputs.h5parm)
writable: true
-
+requirements:
+ - class: NetworkAccess
+ networkAccess: true
stdout: createh5parm.log
stderr: createh5parm_err.log
$namespaces:
diff --git a/steps/dppp_make_parset_cal.cwl b/steps/dppp_make_parset_cal.cwl
index bb6fbfddea240eecd6828377a4005c1cdc9a6f69..def50a36a7708374964c67eb7f26beb02e0f2569 100755
--- a/steps/dppp_make_parset_cal.cwl
+++ b/steps/dppp_make_parset_cal.cwl
@@ -75,7 +75,7 @@ requirements:
listing:
- entryname: input.parset
entry: |+
- steps = [$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,avg]
+ steps = [count,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,avg]
#
msout.storagemanager = "Dysco"
msout.storagemanager.databitrate = 0
diff --git a/steps/dppp_make_parset_target.cwl b/steps/dppp_make_parset_target.cwl
index f31c0a84e56f840adbc301c1be1a17c8afeee024..ba2db4daa61b13ba6f99d3019337aad595d91c33 100755
--- a/steps/dppp_make_parset_target.cwl
+++ b/steps/dppp_make_parset_target.cwl
@@ -93,11 +93,14 @@ requirements:
listing:
- entryname: input.parset
entry: |+
- steps = [$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,applyPA,applybandpass,$(inputs.apply_clock_correction?'applyclock,':'')$(inputs.apply_beam_correction?'applybeam,':'')$(inputs.apply_rm_correction?'applyRM,':'')$(inputs.apply_tec_correction?'applytec,':'')$(inputs.apply_phase_correction?'applyphase,':'')avg]
+ steps = [count1,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,applyPA,applybandpass,$(inputs.apply_clock_correction?'applyclock,':'')$(inputs.apply_beam_correction?'applybeam,':'')$(inputs.apply_rm_correction?'applyRM,':'')$(inputs.apply_tec_correction?'applytec,':'')$(inputs.apply_phase_correction?'applyphase,':'')avg,count2]
#
msout.storagemanager = "Dysco"
msout.storagemanager.databitrate = 0
#
+ count1.type = counter
+ count2.type = counter
+ #
filter.type = filter
filter.baseline = $(inputs.filter_baselines)
filter.remove = true
diff --git a/steps/dppp_prep_cal.cwl b/steps/dppp_prep_cal.cwl
index e8d9dda6ae516669345e6eae3fe2b321ba8df70a..ac6a23481f8686f92ffc79cb8361349fc93c9892 100755
--- a/steps/dppp_prep_cal.cwl
+++ b/steps/dppp_prep_cal.cwl
@@ -75,17 +75,37 @@ inputs:
prefix: msout.storagemanager.databitrate=
separate: false
- id: skymodel
- type: File
+ type:
+ - File
+ - Directory
inputBinding:
position: 0
prefix: demix.skymodel=
separate: false
+ - id: save2json
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count.savetojson=True
+ - id: jsonfilename
+ type: string?
+ default: 'out.json'
+ inputBinding:
+ prefix: count.jsonfilename=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+ - id: flagged_fraction_dict
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: $(inputs.jsonfilename)
+ outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
- id: logfile
type: File[]
outputBinding:
diff --git a/steps/dppp_prep_target.cwl b/steps/dppp_prep_target.cwl
index fa9060db56fe0ac420e2487ea376833c1db3d0cd..b12a3c2bdd1de54ec1d72d51e6741d2e207bf4a0 100755
--- a/steps/dppp_prep_target.cwl
+++ b/steps/dppp_prep_target.cwl
@@ -110,11 +110,37 @@ inputs:
separate: false
valueFrom: $(inputs.solutions.path)
- id: skymodel
- type: File
+ type:
+ - File
+ - Directory
inputBinding:
position: 0
prefix: demix.skymodel=
separate: false
+ - id: save2json1
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count1.savetojson=True
+ - id: jsonfilename1
+ type: string?
+ default: 'out1.json'
+ inputBinding:
+ prefix: count1.jsonfilename=
+ separate: false
+ - id: save2json2
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count2.savetojson=True
+ - id: jsonfilename2
+ type: string?
+ default: 'out2.json'
+ inputBinding:
+ prefix: count2.jsonfilename=
+ separate: false
arguments:
- applyPA.parmdb=$(inputs.solutions.path)
- applybandpass.parmdb=$(inputs.solutions.path)
@@ -124,6 +150,18 @@ outputs:
type: Directory
outputBinding:
glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+ - id: flagged_fraction_dict_initial
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: $(inputs.jsonfilename1)
+ outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
+ - id: flagged_fraction_dict_prep
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: $(inputs.jsonfilename2)
+ outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
- id: logfile
type: File[]
outputBinding:
diff --git a/steps/findRefAnt.cwl b/steps/findRefAnt.cwl
index 77c842ea81b6ad680ac861b67c6c4d4c0f45bc02..701899ba7d21113c782e16daa96b81c0afe808e0 100644
--- a/steps/findRefAnt.cwl
+++ b/steps/findRefAnt.cwl
@@ -2,45 +2,43 @@ class: CommandLineTool
cwlVersion: v1.1
$namespaces:
sbg: 'https://www.sevenbridges.com/'
-id: findRefAnt
+id: dppp
baseCommand:
- - python3
+ - DP3
inputs:
- - id: msin
- type: Directory
- doc: MS to compare with
- inputBinding:
- position: 0
-
-label: findRefAnt.py
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - id: save2json
+ default: true
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: count.savetojson=True
+ - id: jsonfilename
+ type: string?
+ default: 'out.json'
+ inputBinding:
+ prefix: count.jsonfilename=
+ separate: false
arguments:
- - '-c'
- - |
- import sys
- import json
- from findRefAnt import main as findRefAnt
-
- ms = sys.argv[1]
-
- flagged_fraction_dict = str(findRefAnt(ms))
-
- cwl_output = {"flagged_fraction_dict": flagged_fraction_dict}
-
- with open('./out.json', 'w') as fp:
- json.dump(cwl_output, fp)
-
+ - steps=[count]
+ - msout=
outputs:
- id: flagged_fraction_dict
type: string
outputBinding:
loadContents: true
- glob: 'out.json'
+ glob: $(inputs.jsonfilename)
outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
- id: logfile
type: File
outputBinding:
- glob: findRefAnt.log
-
+ glob: findRefAnt.log
requirements:
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
diff --git a/steps/findRefAnt_join.cwl b/steps/findRefAnt_join.cwl
index f7517fb666366bff187f5f9ab458f6e354f7a0c9..20c5fa1495c4e95822ef1140f51d588a3ac6ce64 100644
--- a/steps/findRefAnt_join.cwl
+++ b/steps/findRefAnt_join.cwl
@@ -105,4 +105,4 @@ requirements:
json.dump(flagged_fraction_antenna, fp)
stdout: findRefAnt.log
-stderr: findRefAnt_err.log
+stderr: findRefAnt_err.log
\ No newline at end of file
diff --git a/steps/find_skymodel_target.cwl b/steps/find_skymodel_target.cwl
index 9385f884ab5714ea9fc920bb738158c1695b463a..6b906f1c85576d2fce6e802eb8a9ec8f3c0e65d1 100644
--- a/steps/find_skymodel_target.cwl
+++ b/steps/find_skymodel_target.cwl
@@ -29,6 +29,8 @@ inputs:
default: true
requirements:
- class: InlineJavascriptRequirement
+ - class: NetworkAccess
+ networkAccess: true
- class: InitialWorkDirRequirement
listing:
- entryname: find_sky.py
@@ -66,7 +68,8 @@ outputs:
- id: logfile
type: File
outputBinding:
- glob: find_skymodel_target.log
+ glob: find_skymodel_target.log
+
hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor3-cwl
diff --git a/subworkflow/apply_calibrate_bp.cwl b/subworkflow/apply_calibrate_bp.cwl
index e0e70ea93355022e557ed41ee28d02ac3b53163c..98afcf310e166a29fd1bec4c7dfa9229384c8944 100644
--- a/subworkflow/apply_calibrate_bp.cwl
+++ b/subworkflow/apply_calibrate_bp.cwl
@@ -37,7 +37,7 @@ outputs:
'sbg:y': -300
- id: final_flags_out
outputSource:
- - final_flags/flagged_fraction_dict
+ - calib_cal/flagged_fraction_dict
type: string?
'sbg:x': 1000
'sbg:y': -200
@@ -210,6 +210,7 @@ steps:
out:
- id: msout
- id: h5parm
+ - id: flagged_fraction_dict
- id: logfile
run: ../lofar-cwl/steps/ddecal.cwl
'sbg:x': 600
@@ -279,15 +280,4 @@ steps:
label: concat_logfiles_applyPA
'sbg:x': -200
'sbg:y': 200
- - id: final_flags
- in:
- - id: msin
- source: applyFR/msout
- out:
- - id: flagged_fraction_dict
- - id: logfile
- run: ./../steps/findRefAnt.cwl
- label: final_flags
- 'sbg:x': 700
- 'sbg:y': -200
requirements: []
diff --git a/subworkflow/calib_targ.cwl b/subworkflow/calib_targ.cwl
index b09da67e2e78dcc2bc58f68c3cd75e64e17a6407..9c0caf0ce19fd4c8ce8ab408f6c8ba61c72e8987 100644
--- a/subworkflow/calib_targ.cwl
+++ b/subworkflow/calib_targ.cwl
@@ -10,7 +10,9 @@ inputs:
'sbg:x': -1000
'sbg:y': -200
- id: skymodel
- type: File
+ type:
+ - File
+ - Directory
'sbg:x': -1000
'sbg:y': -100
- id: do_smooth
diff --git a/subworkflow/concat.cwl b/subworkflow/concat.cwl
index a7a3eb0c347219bbd539ac265a1989ffe0438f82..a343cc514890051a10b0dd562c360f0f7b1eab6c 100644
--- a/subworkflow/concat.cwl
+++ b/subworkflow/concat.cwl
@@ -38,6 +38,12 @@ inputs:
'sbg:x': -1000
'sbg:y': 300
outputs:
+ - id: flagged_fraction_dict
+ outputSource:
+ - dpppconcat/flagged_fraction_dict
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': -100
- id: msout
outputSource:
- dpppconcat/msout
@@ -100,6 +106,7 @@ steps:
source: avg_freqresolution_concat
out:
- id: msout
+ - id: flagged_fraction_dict
- id: logfile
run: ../lofar-cwl/steps/dpppconcat.cwl
label: dpppconcat
diff --git a/subworkflow/losoto_bandpass.cwl b/subworkflow/losoto_bandpass.cwl
index b1771a09821fd788aef17bf98fb16c864301f72a..dba230f6af80ef56172307de8e0267457b6fa48c 100644
--- a/subworkflow/losoto_bandpass.cwl
+++ b/subworkflow/losoto_bandpass.cwl
@@ -11,7 +11,7 @@ inputs:
'sbg:x': 730.283935546875
'sbg:y': 152.7667236328125
- id: skipInternational
- type: string?
+ type: boolean?
default: true
'sbg:x': 742.662841796875
'sbg:y': -221.30563354492188
diff --git a/subworkflow/ndppp_prep_cal.cwl b/subworkflow/ndppp_prep_cal.cwl
index 8f46bd8d948e020d0a66c928964f02f83b2c1fef..17b4bc3ad7056be558099cabd12b4fc648649f16 100644
--- a/subworkflow/ndppp_prep_cal.cwl
+++ b/subworkflow/ndppp_prep_cal.cwl
@@ -45,7 +45,9 @@ inputs:
'sbg:x': -500
'sbg:y': -100
- id: skymodel
- type: File
+ type:
+ - File
+ - Directory
'sbg:x': -500
'sbg:y': 0
- id: timeresolution
@@ -101,6 +103,12 @@ outputs:
type: File
'sbg:x': 1000
'sbg:y': 0
+ - id: flagged_fraction_dict
+ outputSource:
+ - dp3_execute/flagged_fraction_dict
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': 100
steps:
- id: define_parset
in:
@@ -167,6 +175,7 @@ steps:
source: skymodel
out:
- id: msout
+ - id: flagged_fraction_dict
- id: logfile
run: ../steps/dppp_prep_cal.cwl
label: DP3.Execute
diff --git a/subworkflow/ndppp_prep_targ.cwl b/subworkflow/ndppp_prep_targ.cwl
index e68210685bd0efef96083c1a33bbe5af69cf2803..81f6e67d2ed5ae7dcbc5d1097c8cc836f485e33f 100644
--- a/subworkflow/ndppp_prep_targ.cwl
+++ b/subworkflow/ndppp_prep_targ.cwl
@@ -40,7 +40,9 @@ inputs:
'sbg:x': -500
'sbg:y': -400
- id: skymodel
- type: File
+ type:
+ - File
+ - Directory
'sbg:x': -500
'sbg:y': -300
- id: timeresolution
@@ -132,9 +134,15 @@ inputs:
'sbg:x': -500
'sbg:y': 1400
outputs:
+ - id: initial_flags_out
+ outputSource:
+ - dp3_execute/flagged_fraction_dict_initial
+ type: string
+ 'sbg:x': 2000
+ 'sbg:y': 300
- id: prep_flags_out
outputSource:
- - prep_flags/flagged_fraction_dict
+ - dp3_execute/flagged_fraction_dict_prep
type: string
'sbg:x': 2000
'sbg:y': 200
@@ -287,6 +295,8 @@ steps:
source: skymodel
out:
- id: msout
+ - id: flagged_fraction_dict_initial
+ - id: flagged_fraction_dict_prep
- id: logfile
run: ../steps/dppp_prep_target.cwl
label: DP3.Execute
@@ -319,17 +329,6 @@ steps:
run: ../lofar-cwl/steps/filter_predict.cwl
'sbg:x': 1000
'sbg:y': 0
- - id: prep_flags
- in:
- - id: msin
- source: dp3_execute/msout
- out:
- - id: flagged_fraction_dict
- - id: logfile
- run: ./../steps/findRefAnt.cwl
- label: prep_flags
- 'sbg:x': 1500
- 'sbg:y': 500
- id: Ateamclipper
in:
- id: msin
diff --git a/subworkflow/predict_calibrate.cwl b/subworkflow/predict_calibrate.cwl
index b212217d880e42a4717a1d6540975588f83a6857..f18c7b4bf139863b5211406deb83931919ca2d1c 100644
--- a/subworkflow/predict_calibrate.cwl
+++ b/subworkflow/predict_calibrate.cwl
@@ -15,7 +15,9 @@ inputs:
'sbg:x': -1000
'sbg:y': -100
- id: sourcedb
- type: File
+ type:
+ - File
+ - Directory
'sbg:x': -1000
'sbg:y': 0
- id: flagunconverged
@@ -53,12 +55,18 @@ outputs:
type: File
'sbg:x': 1000
'sbg:y': 100
+ - id: flagged_fraction_dict
+ outputSource:
+ - calib_cal/flagged_fraction_dict
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': 200
- id: outh5parm
outputSource:
- calib_cal/h5parm
type: File
'sbg:x': 1000
- 'sbg:y': 200
+ 'sbg:y': 300
steps:
- id: BLsmooth
in:
@@ -110,6 +118,7 @@ steps:
out:
- id: msout
- id: h5parm
+ - id: flagged_fraction_dict
- id: logfile
run: ../lofar-cwl/steps/ddecal.cwl
'sbg:x': 500
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 3c29be6e07d05b09cc0c41bc4b6c50e289eb1ae0..e5615f3fa8b622806cadef0626eae6abcf76c20a 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -249,7 +249,7 @@ inputs:
'sbg:y': 3000
- id: ncores
type: int?
- default: 1
+ default: 16
'sbg:x': -1000
'sbg:y': 3100
outputs:
diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl
index d6820b3e35927f8a0732202645786a1eb83866ed..ecea6c2fcfe1ed2776f8674bf42ea476d2d22c58 100644
--- a/workflows/prefactor_calibrator.cwl
+++ b/workflows/prefactor_calibrator.cwl
@@ -237,23 +237,11 @@ outputs:
'sbg:x': 2000
'sbg:y': 0
steps:
- - id: initial_flags
- in:
- - id: msin
- source: msin
- out:
- - id: flagged_fraction_dict
- run: ./../steps/findRefAnt.cwl
- label: initial_flags
- scatter:
- - msin
- 'sbg:x': -500
- 'sbg:y': -500
- id: initial_flags_join
in:
- id: flagged_fraction_dict
source:
- - initial_flags/flagged_fraction_dict
+ - prep/flagged_fraction_dict
- id: filter_station
default: ''
- id: state
@@ -321,29 +309,18 @@ steps:
- id: outh5parm_logfile
- id: check_Ateam_separation.png
- id: check_Ateam_separation.json
+ - id: flagged_fraction_dict_refant
+ - id: flagged_fraction_dict
- id: msout
- id: calibrator_name
run: ./prefactor_calibrator/prep.cwl
label: prep
'sbg:x': 0
'sbg:y': 0
- - id: findRefAnt
- in:
- - id: msin
- source: prep/msout
- out:
- - id: flagged_fraction_dict
- - id: logfile
- run: ./../steps/findRefAnt.cwl
- label: findRefAnt
- scatter:
- - msin
- 'sbg:x': 300
- 'sbg:y': 0
- id: findRefAnt_join
in:
- id: flagged_fraction_dict
- source: findRefAnt/flagged_fraction_dict
+ source: prep/flagged_fraction_dict_refant
- id: filter_station
source: refant
out:
@@ -358,7 +335,6 @@ steps:
- id: file_list
linkMerge: merge_flattened
source:
- - findRefAnt/logfile
- findRefAnt_join/logfile
- id: file_prefix
default: findRefAnt
@@ -368,7 +344,6 @@ steps:
label: concat_logfiles_RefAnt
'sbg:x': 700
'sbg:y': 300
-
- id: pa
in:
- id: msin
diff --git a/workflows/prefactor_calibrator/bp.cwl b/workflows/prefactor_calibrator/bp.cwl
index 9a4328a3d8ecab33275a43e2bc83339dac6cae86..ef8937632a8ca3cd62715b41d2732c7fc0dbabd0 100644
--- a/workflows/prefactor_calibrator/bp.cwl
+++ b/workflows/prefactor_calibrator/bp.cwl
@@ -7,7 +7,7 @@ $namespaces:
inputs:
- id: max_separation_arcmin
type: float?
- default: ''
+ default: 1.0
'sbg:x': -1000
'sbg:y': -800
- id: calibrator_name
diff --git a/workflows/prefactor_calibrator/prep.cwl b/workflows/prefactor_calibrator/prep.cwl
index f77f3b870f1eb08534231f207b9aebe64a918d42..1cd92b919f588618cba5ecdc191d80a8b51c1f72 100644
--- a/workflows/prefactor_calibrator/prep.cwl
+++ b/workflows/prefactor_calibrator/prep.cwl
@@ -164,6 +164,18 @@ outputs:
type: File
'sbg:x': 1554.5106201171875
'sbg:y': 500
+ - id: flagged_fraction_dict_refant
+ outputSource:
+ - predict_calibrate/flagged_fraction_dict
+ type: string[]
+ 'sbg:x': 1550
+ 'sbg:y': 200
+ - id: flagged_fraction_dict
+ outputSource:
+ - ndppp_prep_cal/flagged_fraction_dict
+ type: string[]
+ 'sbg:x': 1550
+ 'sbg:y': 800
- id: logfiles
outputSource:
- concat_logfiles_calib/output
@@ -245,6 +257,7 @@ steps:
- demix_sources
out:
- id: msout
+ - id: flagged_fraction_dict
- id: logfile
run: ../../subworkflow/ndppp_prep_cal.cwl
label: ndppp_prep_cal
@@ -271,6 +284,7 @@ steps:
- id: BLsmooth.log
- id: predict_cal.log
- id: calib_cal.log
+ - id: flagged_fraction_dict
- id: outh5parm
run: ../../subworkflow/predict_calibrate.cwl
label: predict_calibrate
@@ -448,6 +462,7 @@ steps:
label: H5parm_collector
'sbg:x': 1329.937255859375
'sbg:y': 777
+
requirements:
- class: InlineJavascriptRequirement
- class: StepInputExpressionRequirement
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index af440e8f18ce96cc1c0f0ccfa97d85c27929a2ec..b73396f46e96f8ee2824b9cc27c7c479b958709b 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -249,7 +249,7 @@ inputs:
'sbg:y': 3000
- id: ncores
type: int?
- default: 1
+ default: 16
'sbg:x': -1000
'sbg:y': 3100
outputs:
@@ -277,7 +277,6 @@ outputs:
- id: logfiles
outputSource:
- prep/logfiles
- - concat_logfiles_RefAnt/output
- gsmcal/logfiles
- finalize/logfiles
type: 'File[]'
@@ -291,33 +290,6 @@ outputs:
'sbg:x': 2000
'sbg:y': 300
steps:
- - id: initial_flags
- in:
- - id: msin
- source: msin
- out:
- - id: flagged_fraction_dict
- run: ./../steps/findRefAnt.cwl
- label: initial_flags
- scatter:
- - msin
- 'sbg:x': -500
- 'sbg:y': -500
- - id: initial_flags_join
- in:
- - id: flagged_fraction_dict
- source:
- - initial_flags/flagged_fraction_dict
- - id: filter_station
- default: ''
- - id: state
- default: 'initial'
- out:
- - id: flagged_fraction_antenna
- run: ./../steps/findRefAnt_join.cwl
- label: initial_flags_join
- 'sbg:x': -300
- 'sbg:y': -500
- id: prep
in:
- id: msin
@@ -394,68 +366,12 @@ steps:
- id: msout
- id: check_Ateam_separation.json
- id: prep_flags_join_out
+ - id: initial_flags_join_out
- id: logfiles
run: ./prefactor_target/prep.cwl
label: prep
'sbg:x': 0
'sbg:y': 0
- - id: findRefAnt
- in:
- - id: msin
- source: prep/msout
- out:
- - id: flagged_fraction_dict
- - id: logfile
- run: ./../steps/findRefAnt.cwl
- label: findRefAnt
- scatter:
- - msin
- 'sbg:x': 400
- 'sbg:y': 0
- - id: findRefAnt_join
- in:
- - id: flagged_fraction_dict
- source:
- - findRefAnt/flagged_fraction_dict
- - id: filter_station
- source: refant
- out:
- - id: refant
- - id: logfile
- run: ./../steps/findRefAnt_join.cwl
- label: findRefAnt_join
- 'sbg:x': 600
- 'sbg:y': 0
- - id: Ateam_flags_join
- in:
- - id: flagged_fraction_dict
- source:
- - findRefAnt/flagged_fraction_dict
- - id: filter_station
- default: ''
- - id: state
- default: 'Ateam'
- out:
- - id: flagged_fraction_antenna
- run: ./../steps/findRefAnt_join.cwl
- label: Ateam_flags_join
- 'sbg:x': 600
- 'sbg:y': 300
- - id: concat_logfiles_RefAnt
- in:
- - id: file_list
- linkMerge: merge_flattened
- source:
- - findRefAnt/logfile
- - findRefAnt_join/logfile
- - id: file_prefix
- default: findRefAnt
- out:
- - id: output
- run: ./../steps/concatenate_files.cwl
- label: concat_logfiles_RefAnt
- 'sbg:x': 800
- 'sbg:y': 300
- id: gsmcal
in:
- id: msin
@@ -484,7 +400,7 @@ steps:
- id: min_unflagged_fraction
source: min_unflagged_fraction
- id: refant
- source: findRefAnt_join/refant
+ source: refant
- id: min_length
source: min_length
- id: overhead
@@ -494,6 +410,7 @@ steps:
- id: outh5parm
- id: bad_antennas
- id: outh5parm_logfile
+ - id: Ateam_flags_join_out
- id: inspection
- id: logfiles
- id: total_bandwidth
@@ -533,9 +450,9 @@ steps:
- id: flags
linkMerge: merge_flattened
source:
- - initial_flags_join/flagged_fraction_antenna
+ - prep/initial_flags_join_out
- prep/prep_flags_join_out
- - Ateam_flags_join/flagged_fraction_antenna
+ - gsmcal/Ateam_flags_join_out
- id: clip_sources
source: clip_sources
- id: demix
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index f3f032bbd544905c2c5e77533e2766e2cb97f3c0..aba4196902cdd12ad5e3bd4a917af00ce8c0b2ed 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -86,7 +86,7 @@ inputs:
'sbg:y': 900
- id: ncores
type: int?
- default: 1
+ default: 16
'sbg:x': -1000
'sbg:y': 1000
outputs:
@@ -187,6 +187,7 @@ steps:
source: compression_bitrate
out:
- id: msout
+ - id: flagged_fraction_dict
- id: logfile
run: ../../lofar-cwl/steps/applytarget.cwl
label: apply_gsmcal
@@ -196,23 +197,11 @@ steps:
scatterMethod: dotproduct
'sbg:x': 400
'sbg:y': 0
- - id: final_flags
- in:
- - id: msin
- source: apply_gsmcal/msout
- out:
- - id: flagged_fraction_dict
- run: ./../../steps/findRefAnt.cwl
- label: final_flags
- scatter:
- - msin
- 'sbg:x': 400
- 'sbg:y': -400
- id: final_flags_join
in:
- id: flagged_fraction_dict
source:
- - final_flags/flagged_fraction_dict
+ - apply_gsmcal/flagged_fraction_dict
- id: filter_station
default: ''
- id: state
@@ -320,16 +309,18 @@ steps:
- average/msout
- id: image_scale
default: 15asec
+ - id: auto_threshold
+ default: 5
+ - id: size
+ default: [2500,2500]
- id: niter
- default: 1000000
+ default: 10000
- id: nmiter
default: 5
- id: multiscale
default: true
- id: mgain
default: 0.8
- - id: ncpu
- source: ncores
- id: parallel-gridding
source: ncores
valueFrom: $(Math.round((self+1)/4))
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
index 0dac9b7143421e27659a015eb0b398551ec20260..94aeef683e32e5a81d193e0db3f6db401d550d4a 100644
--- a/workflows/prefactor_target/gsmcal.cwl
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -103,6 +103,12 @@ outputs:
type: File
'sbg:x': 1900
'sbg:y': 600
+ - id: Ateam_flags_join_out
+ outputSource:
+ - Ateam_flags_join/flagged_fraction_antenna
+ type: File
+ 'sbg:x': 1900
+ 'sbg:y': 700
- id: inspection
outputSource:
- losoto_plot_P/output_plots
@@ -117,6 +123,7 @@ outputs:
- id: logfiles
outputSource:
- concat_logfiles_identify/output
+ - concat_logfiles_RefAnt/output
- sort_times_into_freqGroups/logfile
- find_skymodel_target/logfile
- make_sourcedb_target/log
@@ -164,6 +171,35 @@ steps:
label: identifybadantennas_join
'sbg:x': 0
'sbg:y': -300
+ - id: findRefAnt_join
+ in:
+ - id: flagged_fraction_dict
+ source:
+ - concat/flagged_fraction_dict
+ - id: filter_station
+ source: refant
+ out:
+ - id: refant
+ - id: logfile
+ run: ../../steps/findRefAnt_join.cwl
+ label: findRefAnt_join
+ 'sbg:x': 200
+ 'sbg:y': -300
+ - id: Ateam_flags_join
+ in:
+ - id: flagged_fraction_dict
+ source:
+ - concat/flagged_fraction_dict
+ - id: filter_station
+ default: ''
+ - id: state
+ default: 'concat'
+ out:
+ - id: flagged_fraction_antenna
+ run: ../../steps/findRefAnt_join.cwl
+ label: Ateam_flags_join
+ 'sbg:x': 400
+ 'sbg:y': -300
- id: sort_times_into_freqGroups
in:
- id: msin
@@ -310,6 +346,20 @@ steps:
label: concat_logfiles_identify
'sbg:x': 1600
'sbg:y': -100
+ - id: concat_logfiles_RefAnt
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - findRefAnt_join/logfile
+ - id: file_prefix
+ default: findRefAnt
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_RefAnt
+ 'sbg:x': 1600
+ 'sbg:y': -300
- id: concat
in:
- id: msin
@@ -329,6 +379,7 @@ steps:
source: min_unflagged_fraction
out:
- id: msout
+ - id: flagged_fraction_dict
- id: dpppconcat.log
run: ../../subworkflow/concat.cwl
label: concat
@@ -485,7 +536,7 @@ steps:
- id: plotFlag
default: true
- id: refAnt
- source: refant
+ source: findRefAnt_join/refant
- id: prefix
default: ph_
out:
@@ -516,7 +567,7 @@ steps:
- id: plotFlag
default: true
- id: refAnt
- source: refant
+ source: findRefAnt_join/refant
- id: prefix
default: ph_
out:
@@ -548,9 +599,9 @@ steps:
- id: plotFlag
default: true
- id: refAnt
- source: refant
+ source: findRefAnt_join/refant
- id: prefix
- default: ph_poldif_
+ default: ph_poldif
out:
- id: output_plots
- id: logfile
@@ -579,7 +630,7 @@ steps:
- id: plotFlag
default: true
- id: refAnt
- source: refant
+ source: findRefAnt_join/refant
- id: prefix
default: ph_poldif_
out:
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index 802954d3471b96ed98cc2ab42d599ea63cc3c913..f150aa9dd715e3284b8d4e1fa72951d779c69e29 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -230,13 +230,35 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 800
+ - id: initial_flags_join_out
+ outputSource:
+ - initial_flags_join/flagged_fraction_antenna
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': 1000
- id: prep_flags_join_out
outputSource:
- prep_flags_join/flagged_fraction_antenna
type: File
'sbg:x': 1000
- 'sbg:y': 1100
+ 'sbg:y': 1200
steps:
+ - id: initial_flags_join
+ in:
+ - id: flagged_fraction_dict
+ source:
+ - dppp_prep_target/initial_flags_out
+ - id: filter_station
+ default: ''
+ - id: state
+ default: initial
+ out:
+ - id: flagged_fraction_antenna
+ - id: logfile
+ run: ./../../steps/findRefAnt_join.cwl
+ label: initial_flags_join
+ 'sbg:x': 600
+ 'sbg:y': -200
- id: prep_flags_join
in:
- id: flagged_fraction_dict
@@ -247,7 +269,6 @@ steps:
- id: state
default: prep
out:
- - id: refant
- id: flagged_fraction_antenna
- id: logfile
run: ./../../steps/findRefAnt_join.cwl
@@ -424,6 +445,7 @@ steps:
- clip_sources
out:
- id: prep_flags_out
+ - id: initial_flags_out
- id: prep_logfile
- id: predict_logfile
- id: clipper_logfile