From 7f3e071bb4a870915bb45096c8a6fba7a75e322c Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@tls-tautenburg.de>
Date: Wed, 8 Jul 2020 14:10:27 +0200
Subject: [PATCH] add target pipeline
Former-commit-id: 787e2e0e566c5993f1ea6dad267651828487ff9d
Former-commit-id: 13b6d4c436d346bba7c6829e9227725ba3f88208
---
steps/compare_station_list.cwl | 64 +++++++++
steps/createRMh5parm.cwl | 123 +++++++++++++++++
steps/h5parmcat.cwl | 5 +-
workflows/HBA_target.cwl | 19 ++-
workflows/prefactor_target.cwl | 26 ++--
workflows/prefactor_target/prep.cwl | 202 ++++++++++++----------------
6 files changed, 300 insertions(+), 139 deletions(-)
create mode 100644 steps/compare_station_list.cwl
create mode 100644 steps/createRMh5parm.cwl
diff --git a/steps/compare_station_list.cwl b/steps/compare_station_list.cwl
new file mode 100644
index 00000000..96574806
--- /dev/null
+++ b/steps/compare_station_list.cwl
@@ -0,0 +1,64 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: compare_station_list
+baseCommand:
+ - python3
+inputs:
+ - id: msin
+ type: Directory[]
+ doc: MS to compare with
+ inputBinding:
+ position: 0
+ - id: h5parmdb
+ type: File
+ doc: H5parm database to compare with
+ - id: solset_name
+ type: string?
+ doc: Name of the H5parm solset
+ default: 'sol000'
+ - id: filter
+ type: string?
+ default: '*&'
+ doc: Filter these baselines for the comparison
+
+label: compareStationList.py
+arguments:
+ - '-c'
+ - |
+ import sys
+ import json
+ from compareStationList import main as compareStationList
+
+ mss = sys.argv[1:]
+ h5parmdb = $(inputs.h5parmdb)['path']
+ solset_name = '$(inputs.solset_name)'
+ filter = '$(inputs.filter)'
+
+ output = compareStationList(mss, h5parmdb, solset_name, filter)
+
+ filter_out = output['filter']
+ cwl_output = {"filter_out": filter_out}
+
+ with open('./out.json', 'w') as fp:
+ json.dump(cwl_output, fp)
+outputs:
+ - id: filter_out
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).filter_out)
+ - id: logfile
+ type: File?
+ outputBinding:
+ glob: compareStationList.log
+
+requirements:
+ - class: InlineJavascriptRequirement
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: compareStationList.log
\ No newline at end of file
diff --git a/steps/createRMh5parm.cwl b/steps/createRMh5parm.cwl
new file mode 100644
index 00000000..d79ceaa7
--- /dev/null
+++ b/steps/createRMh5parm.cwl
@@ -0,0 +1,123 @@
+class: CommandLineTool
+cwlVersion: v1.0
+id: createRMh5parm
+label: createRMh5parm
+baseCommand:
+ - python3
+ - /usr/local/bin/createRMh5parm.py
+inputs:
+ - id: msin
+ type:
+ - Directory
+ - type: array
+ items: Directory
+ inputBinding:
+ position: 1
+ doc: Input measurement set
+ - id: h5parm
+ type: File
+ inputBinding:
+ position: 2
+ doc: Input h5parm solutions file
+ - id: ionex_server
+ type: string
+ doc: IONEX Server path
+ default: 'ftp://ftp.aiub.unibe.ch/CODE/'
+ inputBinding:
+ prefix: '--server'
+ position: 0
+ - id: ionex_prefix
+ type: string
+ doc: IONEX Prefix
+ default: 'CODG'
+ inputBinding:
+ prefix: '--prefix'
+ position: 0
+ - id: ionex_path
+ type: string
+ doc: IONEX path
+ default: './'
+ inputBinding:
+ prefix: '--path'
+ position: 0
+ - id: solset
+ type: string
+ doc: solset in which IONEX solution are put
+ default: 'sol000'
+ inputBinding:
+ prefix: '--solsetName'
+ position: 0
+ - id: timestep
+ type: float
+ doc: timestep in seconds
+ default: 300.
+ inputBinding:
+ prefix: '-t'
+ position: 0
+ - id: smart_interpol
+ type: float
+ doc: float parameter describing how much of Earth rotation is taken in to account in interpolation of the IONEX files. 1.0 means time interpolation assumes ionosphere rotates in opposite direction of the Earth
+ default: 0.
+ inputBinding:
+ prefix: '-e'
+ position: 0
+ - id: proxyserver
+ type: string
+ doc: Specify proxy server if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxyserver'
+ position: 0
+ - id: proxyport
+ type: int
+ doc: Specify proxy port if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxyport'
+ position: 0
+ - id: proxytype
+ type: string
+ doc: Specify proxy type if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxytype'
+ position: 0
+ - id: proxyuser
+ type: string
+ doc: Specify proxy user name if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxyuser'
+ position: 0
+ - id: proxypass
+ type: string
+ doc: Specify proxy server password if necessary
+ default: null
+ inputBinding:
+ prefix: '--proxypass'
+ position: 0
+outputs:
+ - id: h5parmout
+ doc: h5parm output
+ type: File
+ format: 'lofar:#H5Parm'
+ outputBinding:
+ glob: $(inputs.h5parm.basename)
+ - id: logfile
+ type: File[]
+ outputBinding:
+ glob: 'createh5parm*.log'
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.h5parm)
+ writable: true
+
+stdout: createh5parm.log
+stderr: createh5parm_err.log
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/h5parmcat.cwl b/steps/h5parmcat.cwl
index 861d08e7..76f8ae83 100644
--- a/steps/h5parmcat.cwl
+++ b/steps/h5parmcat.cwl
@@ -66,9 +66,9 @@ outputs:
glob: $(inputs.input_file.basename)
format: lofar:#H5Parm
- id: log
- type: File
+ type: File[]
outputBinding:
- glob: parm_collector_output.log
+ glob: 'parm_collector_output*.log'
label: h5parm_cat
arguments:
- position: 0
@@ -84,5 +84,6 @@ hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator
stdout: parm_collector_output.log
+stderr: parm_collector_output_err.log
$schema:
- 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 57b3cea6..ff392d09 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -179,8 +179,10 @@ inputs:
'sbg:x': -1000
'sbg:y': 1700
- id: reference_stationSB
- type: int?
- default: None
+ type:
+ - int?
+ - string?
+ default: ''
'sbg:x': -1000
'sbg:y': 1800
- id: ionex_server
@@ -195,27 +197,27 @@ inputs:
'sbg:y': 2000
- id: proxy_server
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2100
- id: proxy_port
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2200
- id: proxy_type
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2300
- id: proxy_user
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2400
- id: proxy_pass
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2500
outputs:
@@ -285,8 +287,6 @@ steps:
source: overhead
- id: min_separation
source: min_separation
- - id: calibrator_path_skymodel
- source: calibrator_path_skymodel
- id: A-Team_skymodel
source: A-Team_skymodel
- id: target_skymodel
@@ -324,7 +324,6 @@ steps:
out:
- id: logfiles
- id: msout
- - id: parset
- id: solutions
- id: inspection
run: ./prefactor_target.cwl
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index 972705e8..ee4591d1 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -179,8 +179,10 @@ inputs:
'sbg:x': -1000
'sbg:y': 1700
- id: reference_stationSB
- type: int?
- default: None
+ type:
+ - int?
+ - string?
+ default: ''
'sbg:x': -1000
'sbg:y': 1800
- id: ionex_server
@@ -195,35 +197,34 @@ inputs:
'sbg:y': 2000
- id: proxy_server
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2100
- id: proxy_port
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2200
- id: proxy_type
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2300
- id: proxy_user
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2400
- id: proxy_pass
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 2500
outputs:
- id: inspection
linkMerge: merge_flattened
outputSource:
- - prep/check_Ateam_separation.png
- - prep/RMextract.png
+ - prep/inspection
type: File[]
'sbg:x': 2000
'sbg:y': -600
@@ -266,8 +267,8 @@ steps:
source: demix
- id: apply_steps
source: apply_steps
- - id: clipAteam
- source: clipAteam
+ - id: clipATeam
+ source: clipATeam
- id: max_dppp_threads
source: max_dppp_threads
- id: memoryperc
@@ -311,9 +312,8 @@ steps:
out:
- id: outh5parm
- id: logfiles
- - id: check_Ateam_separation.png
+ - id: inspection
- id: msout
- - id: target_name
run: ./prefactor_target/prep.cwl
label: prep
'sbg:x': 0
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index 06ef5cc2..9dbd0663 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -10,7 +10,7 @@ inputs:
'sbg:x': -1000
'sbg:y': -1700
- id: cal_solutions
- type: 'File'
+ type: File
'sbg:x': -1000
'sbg:y': -1600
- id: flag_baselines
@@ -106,7 +106,7 @@ inputs:
'sbg:y': 200
- id: avg_timeresolution
type: float?
- default: 4.
+ default: 4
'sbg:x': -1000
'sbg:y': 300
- id: avg_freqresolution
@@ -121,147 +121,67 @@ inputs:
'sbg:y': 500
- id: ionex_prefix
type: string?
- default: 'CODG'
+ default: CODG
'sbg:x': -1000
'sbg:y': 600
- id: proxy_server
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 700
- id: proxy_port
- type: string?
- default: ''
+ type: int?
+ default: null
'sbg:x': -1000
'sbg:y': 800
- id: proxy_type
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 900
- id: proxy_user
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 1000
- id: proxy_pass
type: string?
- default: ''
+ default: null
'sbg:x': -1000
'sbg:y': 1100
outputs:
- id: outh5parm
outputSource:
- - h5parm_collector/outh5parm
+ - createRMh5parm/h5parmout
type: File
'sbg:x': 1000
'sbg:y': -500
- - id: check_Ateam_separation.png
+ - id: inspection
outputSource:
- check_ateam_separation/output_imag
- type: File
+ - losoto_plot_RM/output_plots
+ type: 'File[]?'
+ linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': -300
- id: msout
outputSource:
- - predict_calibrate/msout
+ - msin
type: 'Directory[]'
'sbg:x': 1000
'sbg:y': 0
- - id: parset
- outputSource:
- - ndppp_prep_cal/parset
- type: 'File[]'
- 'sbg:x': 1000
- 'sbg:y': 200
- - id: calibrator_name
- outputSource:
- - find_skymodel_cal/model_name
- type: string
- 'sbg:x': 1000
- 'sbg:y': 400
- - id: outh5parm_logfile
- outputSource:
- - h5parm_collector/log
- type: File[]
- 'sbg:x': 1000
- 'sbg:y': 650
- id: logfiles
- linkMerge: merge_flattened
outputSource:
- - concat_logfiles_calib/output
- - concat_logfiles_predict/output
- - concat_logfiles_blsmooth/output
- - make_sourcedb/log
- - find_skymodel_cal/logfile
- - concat_logfiles_prep_cal/output
- make_sourcedb_ateam/log
- check_ateam_separation/logfile
+ - createRMh5parm/logfile
+ - losoto_plot_RM/logfile
+ - compare_station_list/logfile
type: 'File[]'
+ linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 800
steps:
- - id: ndppp_prep_target
- in:
- - id: baselines_to_flag
- default: []
- source:
- - flag_baselines
- - id: elevation_to_flag
- source: elevation
- - id: min_amplitude_to_flag
- source: amplmin
- - id: memoryperc
- default: 20
- source: memoryperc
- - id: raw_data
- default: false
- source: raw_data
- - id: demix
- default: false
- source: demix
- - id: msin
- linkMerge: merge_flattened
- source:
- - msin
- - id: msin_baseline
- default: '*'
- source: filter_baselines
- - id: skymodel
- source: make_sourcedb_ateam/sourcedb
- - id: timeresolution
- default: 1
- source: avg_timeresolution
- - id: freqresolution
- default: 12.21kHz
- source: avg_freqresolution
- - id: demix_timestep
- default: 1
- source: demix_timestep
- - id: demix_freqstep
- default: 1
- source: demix_freqstep
- - id: demix_baseline
- default: '*'
- source: process_baselines_cal
- - id: target_source
- source: demix_target
- - id: ntimechunk
- default: 10
- source: max_dppp_threads
- - id: subtract_sources
- source:
- - demix_sources
- out:
- - id: msout
- - id: logfile
- - id: parset
- run: ../../subworkflow/ndppp_prep_cal.cwl
- label: ndppp_prep_target
- scatter:
- - msin
- 'sbg:x': 0
- 'sbg:y': 0
- id: check_ateam_separation
in:
- id: ms
@@ -274,8 +194,26 @@ steps:
- id: logfile
run: ../../lofar-cwl/steps/check_ateam_separation.cwl
label: check_Ateam_separation
- 'sbg:x': 0
+ 'sbg:x': -500
'sbg:y': -300
+ - id: compare_station_list
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: h5parmdb
+ source: cal_solutions
+ - id: solset_name
+ default: 'calibrator'
+ - id: filter
+ source: filter_baselines
+ out:
+ - id: filter_out
+ - id: logfile
+ run: ../../steps/compare_station_list.cwl
+ label: compare_station_list
+ 'sbg:x': -500
+ 'sbg:y': -500
- id: make_sourcedb_ateam
in:
- id: sky_model
@@ -289,19 +227,55 @@ steps:
label: make_sourcedb_ateam
'sbg:x': -500
'sbg:y': 300
- - id: concat_logfiles_prep_target
+ - id: createRMh5parm
in:
- - id: file_list
- linkMerge: merge_flattened
- source: ndppp_prep_cal/logfile
- - id: file_prefix
- default: ndppp_prep_cal
+ - id: msin
+ source:
+ - msin
+ - id: h5parm
+ source: cal_solutions
+ - id: ionex_server
+ source: ionex_server
+ - id: ionex_prefix
+ source: ionex_prefix
+ - id: solset
+ default: target
+ - id: proxyserver
+ source: proxy_server
+ - id: proxyport
+ source: proxy_port
+ - id: proxytype
+ source: proxy_type
+ - id: proxyuser
+ source: proxy_user
+ - id: proxypass
+ source: proxy_pass
out:
- - id: output
- run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_prep_target
- 'sbg:x': 250
- 'sbg:y': 300
-requirements:
- - class: SubworkflowFeatureRequirement
- - class: ScatterFeatureRequirement
+ - id: h5parmout
+ - id: logfile
+ run: ../../steps/createRMh5parm.cwl
+ label: createRMh5parm
+ 'sbg:x': -500
+ 'sbg:y': 0
+ - id: losoto_plot_RM
+ in:
+ - id: input_h5parm
+ source: createRMh5parm/h5parmout
+ - id: soltab
+ default: target/RMextract
+ - id: axesInPlot
+ default:
+ - time
+ - id: axisInTable
+ default: ant
+ - id: prefix
+ default: RMextract
+ out:
+ - id: output_plots
+ - id: logfile
+ - id: parset
+ run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
+ label: losoto_plot_RM
+ 'sbg:x': -300
+ 'sbg:y': 0
+requirements: []
--
GitLab