diff --git a/steps/compare_station_list.cwl b/steps/compare_station_list.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..9657480614b0d409701ef67890ced691baf35fdc
--- /dev/null
+++ b/steps/compare_station_list.cwl
@@ -0,0 +1,64 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: compare_station_list
+baseCommand:
+ - python3
+inputs:
+ - id: msin
+ type: Directory[]
+ doc: MS to compare with
+ inputBinding:
+ position: 0
+ - id: h5parmdb
+ type: File
+ doc: H5parm database to compare with
+ - id: solset_name
+ type: string?
+ doc: Name of the H5parm solset
+ default: 'sol000'
+ - id: filter
+ type: string?
+ default: '*&'
+ doc: Filter these baselines for the comparison
+
+label: compareStationList.py
+arguments:
+ - '-c'
+ - |
+ import sys
+ import json
+ from compareStationList import main as compareStationList
+
+ mss = sys.argv[1:]
+ h5parmdb = $(inputs.h5parmdb)['path']
+ solset_name = '$(inputs.solset_name)'
+ filter = '$(inputs.filter)'
+
+ output = compareStationList(mss, h5parmdb, solset_name, filter)
+
+ filter_out = output['filter']
+ cwl_output = {"filter_out": filter_out}
+
+ with open('./out.json', 'w') as fp:
+ json.dump(cwl_output, fp)
+outputs:
+ - id: filter_out
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).filter_out)
+ - id: logfile
+ type: File?
+ outputBinding:
+ glob: compareStationList.log
+
+requirements:
+ - class: InlineJavascriptRequirement
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: compareStationList.log
\ No newline at end of file
diff --git a/steps/createRMh5parm.cwl b/steps/createRMh5parm.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..d79ceaa783f93cf1dee382d2c398319b5f5ed019
--- /dev/null
+++ b/steps/createRMh5parm.cwl
@@ -0,0 +1,123 @@
+class: CommandLineTool
+cwlVersion: v1.0
+id: createRMh5parm
+label: createRMh5parm
+baseCommand:
+ - python3
+ - /usr/local/bin/createRMh5parm.py
+inputs:
+ - id: msin
+ type:
+ - Directory
+ - type: array
+ items: Directory
+ inputBinding:
+ position: 1
+ doc: Input measurement set
+ - id: h5parm
+ type: File
+ inputBinding:
+ position: 2
+ doc: Input h5parm solutions file
+ - id: ionex_server
+ type: string
+ doc: IONEX Server path
+ default: 'ftp://ftp.aiub.unibe.ch/CODE/'
+ inputBinding:
+ prefix: '--server'
+ position: 0
+ - id: ionex_prefix
+ type: string
+ doc: IONEX Prefix
+ default: 'CODG'
+ inputBinding:
+ prefix: '--prefix'
+ position: 0
+ - id: ionex_path
+ type: string
+ doc: IONEX path
+ default: './'
+ inputBinding:
+ prefix: '--path'
+ position: 0
+ - id: solset
+ type: string
+ doc: solset in which IONEX solution are put
+ default: 'sol000'
+ inputBinding:
+ prefix: '--solsetName'
+ position: 0
+ - id: timestep
+ type: float
+ doc: timestep in seconds
+ default: 300.
+ inputBinding:
+ prefix: '-t'
+ position: 0
+ - id: smart_interpol
+ type: float
+ doc: float parameter describing how much of Earth rotation is taken in to account in interpolation of the IONEX files. 1.0 means time interpolation assumes ionosphere rotates in opposite direction of the Earth
+ default: 0.
+ inputBinding:
+ prefix: '-e'
+ position: 0
+ - id: proxyserver
+ type: string
+ doc: Specify proxy server if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxyserver'
+ position: 0
+ - id: proxyport
+ type: int
+ doc: Specify proxy port if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxyport'
+ position: 0
+ - id: proxytype
+ type: string
+ doc: Specify proxy type if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxytype'
+ position: 0
+ - id: proxyuser
+ type: string
+ doc: Specify proxy user name if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxyuser'
+ position: 0
+ - id: proxypass
+ type: string
+ doc: Specify proxy server password if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxypass'
+ position: 0
+outputs:
+ - id: h5parmout
+ doc: h5parm output
+ type: File
+ format: 'lofar:#H5Parm'
+ outputBinding:
+ glob: $(inputs.h5parm.basename)
+ - id: logfile
+ type: File[]
+ outputBinding:
+ glob: 'createh5parm*.log'
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.h5parm)
+ writable: true
+
+stdout: createh5parm.log
+stderr: createh5parm_err.log
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/h5parmcat.cwl b/steps/h5parmcat.cwl
index 861d08e7a7f49dcbcd09e4b66c48784839b2126a..76f8ae83013bd5123d8695cd43b068cf24470be9 100644
--- a/steps/h5parmcat.cwl
+++ b/steps/h5parmcat.cwl
@@ -66,9 +66,9 @@ outputs:
glob: $(inputs.input_file.basename)
format: lofar:#H5Parm
- id: log
- type: File
+ type: File[]
outputBinding:
- glob: parm_collector_output.log
+ glob: 'parm_collector_output*.log'
label: h5parm_cat
arguments:
- position: 0
@@ -84,5 +84,6 @@ hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator
stdout: parm_collector_output.log
+stderr: parm_collector_output_err.log
$schema:
- 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 57b3cea6a5e5ef984e92e0353899ca1f818a0d48..ff392d0902b5ff2c1f8a9f8b803273d80be2bb6c 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -179,8 +179,10 @@ inputs:
'sbg:x': -1000
'sbg:y': 1700
- id: reference_stationSB
- type: int?
- default: None
+ type:
+ - int?
+ - string?
+ default: ''
'sbg:x': -1000
'sbg:y': 1800
- id: ionex_server
@@ -195,27 +197,27 @@ inputs:
'sbg:y': 2000
- id: proxy_server
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2100
- id: proxy_port
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2200
- id: proxy_type
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2300
- id: proxy_user
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2400
- id: proxy_pass
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2500
outputs:
@@ -285,8 +287,6 @@ steps:
source: overhead
- id: min_separation
source: min_separation
- - id: calibrator_path_skymodel
- source: calibrator_path_skymodel
- id: A-Team_skymodel
source: A-Team_skymodel
- id: target_skymodel
@@ -324,7 +324,6 @@ steps:
out:
- id: logfiles
- id: msout
- - id: parset
- id: solutions
- id: inspection
run: ./prefactor_target.cwl
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index 972705e863d5e083072f148a2edde90c8102c467..ee4591d1704d4d3084a6345e2ba084fa5d0152d9 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -179,8 +179,10 @@ inputs:
'sbg:x': -1000
'sbg:y': 1700
- id: reference_stationSB
- type: int?
- default: None
+ type:
+ - int?
+ - string?
+ default: ''
'sbg:x': -1000
'sbg:y': 1800
- id: ionex_server
@@ -195,35 +197,34 @@ inputs:
'sbg:y': 2000
- id: proxy_server
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2100
- id: proxy_port
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2200
- id: proxy_type
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2300
- id: proxy_user
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2400
- id: proxy_pass
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2500
outputs:
- id: inspection
linkMerge: merge_flattened
outputSource:
- - prep/check_Ateam_separation.png
- - prep/RMextract.png
+ - prep/inspection
type: File[]
'sbg:x': 2000
'sbg:y': -600
@@ -266,8 +267,8 @@ steps:
source: demix
- id: apply_steps
source: apply_steps
- - id: clipAteam
- source: clipAteam
+ - id: clipATeam
+ source: clipATeam
- id: max_dppp_threads
source: max_dppp_threads
- id: memoryperc
@@ -311,9 +312,8 @@ steps:
out:
- id: outh5parm
- id: logfiles
- - id: check_Ateam_separation.png
+ - id: inspection
- id: msout
- - id: target_name
run: ./prefactor_target/prep.cwl
label: prep
'sbg:x': 0
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index 06ef5cc2b2f1888bbe4710e72bc648e07294d8a9..9dbd0663608e11268ffbc2dace52956cf467bcb9 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -10,7 +10,7 @@ inputs:
'sbg:x': -1000
'sbg:y': -1700
- id: cal_solutions
- type: 'File'
+ type: File
'sbg:x': -1000
'sbg:y': -1600
- id: flag_baselines
@@ -106,7 +106,7 @@ inputs:
'sbg:y': 200
- id: avg_timeresolution
type: float?
- default: 4.
+ default: 4
'sbg:x': -1000
'sbg:y': 300
- id: avg_freqresolution
@@ -121,147 +121,67 @@ inputs:
'sbg:y': 500
- id: ionex_prefix
type: string?
- default: 'CODG'
+ default: CODG
'sbg:x': -1000
'sbg:y': 600
- id: proxy_server
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 700
- id: proxy_port
- type: string?
- default: ''
+ type: int?
+ default: null
'sbg:x': -1000
'sbg:y': 800
- id: proxy_type
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 900
- id: proxy_user
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 1000
- id: proxy_pass
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 1100
outputs:
- id: outh5parm
outputSource:
- - h5parm_collector/outh5parm
+ - createRMh5parm/h5parmout
type: File
'sbg:x': 1000
'sbg:y': -500
- - id: check_Ateam_separation.png
+ - id: inspection
outputSource:
- check_ateam_separation/output_imag
- type: File
+ - losoto_plot_RM/output_plots
+ type: 'File[]?'
+ linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': -300
- id: msout
outputSource:
- - predict_calibrate/msout
+ - msin
type: 'Directory[]'
'sbg:x': 1000
'sbg:y': 0
- - id: parset
- outputSource:
- - ndppp_prep_cal/parset
- type: 'File[]'
- 'sbg:x': 1000
- 'sbg:y': 200
- - id: calibrator_name
- outputSource:
- - find_skymodel_cal/model_name
- type: string
- 'sbg:x': 1000
- 'sbg:y': 400
- - id: outh5parm_logfile
- outputSource:
- - h5parm_collector/log
- type: File[]
- 'sbg:x': 1000
- 'sbg:y': 650
- id: logfiles
- linkMerge: merge_flattened
outputSource:
- - concat_logfiles_calib/output
- - concat_logfiles_predict/output
- - concat_logfiles_blsmooth/output
- - make_sourcedb/log
- - find_skymodel_cal/logfile
- - concat_logfiles_prep_cal/output
- make_sourcedb_ateam/log
- check_ateam_separation/logfile
+ - createRMh5parm/logfile
+ - losoto_plot_RM/logfile
+ - compare_station_list/logfile
type: 'File[]'
+ linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 800
steps:
- - id: ndppp_prep_target
- in:
- - id: baselines_to_flag
- default: []
- source:
- - flag_baselines
- - id: elevation_to_flag
- source: elevation
- - id: min_amplitude_to_flag
- source: amplmin
- - id: memoryperc
- default: 20
- source: memoryperc
- - id: raw_data
- default: false
- source: raw_data
- - id: demix
- default: false
- source: demix
- - id: msin
- linkMerge: merge_flattened
- source:
- - msin
- - id: msin_baseline
- default: '*'
- source: filter_baselines
- - id: skymodel
- source: make_sourcedb_ateam/sourcedb
- - id: timeresolution
- default: 1
- source: avg_timeresolution
- - id: freqresolution
- default: 12.21kHz
- source: avg_freqresolution
- - id: demix_timestep
- default: 1
- source: demix_timestep
- - id: demix_freqstep
- default: 1
- source: demix_freqstep
- - id: demix_baseline
- default: '*'
- source: process_baselines_cal
- - id: target_source
- source: demix_target
- - id: ntimechunk
- default: 10
- source: max_dppp_threads
- - id: subtract_sources
- source:
- - demix_sources
- out:
- - id: msout
- - id: logfile
- - id: parset
- run: ../../subworkflow/ndppp_prep_cal.cwl
- label: ndppp_prep_target
- scatter:
- - msin
- 'sbg:x': 0
- 'sbg:y': 0
- id: check_ateam_separation
in:
- id: ms
@@ -274,8 +194,26 @@ steps:
- id: logfile
run: ../../lofar-cwl/steps/check_ateam_separation.cwl
label: check_Ateam_separation
- 'sbg:x': 0
+ 'sbg:x': -500
'sbg:y': -300
+ - id: compare_station_list
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: h5parmdb
+ source: cal_solutions
+ - id: solset_name
+ default: 'calibrator'
+ - id: filter
+ source: filter_baselines
+ out:
+ - id: filter_out
+ - id: logfile
+ run: ../../steps/compare_station_list.cwl
+ label: compare_station_list
+ 'sbg:x': -500
+ 'sbg:y': -500
- id: make_sourcedb_ateam
in:
- id: sky_model
@@ -289,19 +227,55 @@ steps:
label: make_sourcedb_ateam
'sbg:x': -500
'sbg:y': 300
- - id: concat_logfiles_prep_target
+ - id: createRMh5parm
in:
- - id: file_list
- linkMerge: merge_flattened
- source: ndppp_prep_cal/logfile
- - id: file_prefix
- default: ndppp_prep_cal
+ - id: msin
+ source:
+ - msin
+ - id: h5parm
+ source: cal_solutions
+ - id: ionex_server
+ source: ionex_server
+ - id: ionex_prefix
+ source: ionex_prefix
+ - id: solset
+ default: target
+ - id: proxyserver
+ source: proxy_server
+ - id: proxyport
+ source: proxy_port
+ - id: proxytype
+ source: proxy_type
+ - id: proxyuser
+ source: proxy_user
+ - id: proxypass
+ source: proxy_pass
out:
- - id: output
- run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_prep_target
- 'sbg:x': 250
- 'sbg:y': 300
-requirements:
- - class: SubworkflowFeatureRequirement
- - class: ScatterFeatureRequirement
+ - id: h5parmout
+ - id: logfile
+ run: ../../steps/createRMh5parm.cwl
+ label: createRMh5parm
+ 'sbg:x': -500
+ 'sbg:y': 0
+ - id: losoto_plot_RM
+ in:
+ - id: input_h5parm
+ source: createRMh5parm/h5parmout
+ - id: soltab
+ default: target/RMextract
+ - id: axesInPlot
+ default:
+ - time
+ - id: axisInTable
+ default: ant
+ - id: prefix
+ default: RMextract
+ out:
+ - id: output_plots
+ - id: logfile
+ - id: parset
+ run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
+ label: losoto_plot_RM
+ 'sbg:x': -300
+ 'sbg:y': 0
+requirements: []