diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8c75c0ca2bba9bac81a3c8a291547eb01aab987a..6b887625ee0d17d7bf3f6caff60b74882a0ee0e6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,6 +13,7 @@ variables:
TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
TARGET_LBA_RESULTS_NAME: "results_target_lba.tar.gz"
TARGET_HBA_SELFCAL_RESULTS_NAME: "results_target_selfcal.tar.gz"
+ TARGET_HBA_VLBI_RESULTS_NAME: "results_target_VLBI.tar.gz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
BUILD_DOCKER_IMAGE: "0"
GIT_STRATEGY: clone # ensure every job starts with a pristine working copy
@@ -246,6 +247,7 @@ download_data:
- wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
- wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_LBA_RESULTS_NAME -O $TARGET_LBA_RESULTS_NAME && tar xfz $TARGET_LBA_RESULTS_NAME && rm -f $TARGET_LBA_RESULTS_NAME
- wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME
+ - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_VLBI_RESULTS_NAME -O $TARGET_HBA_VLBI_RESULTS_NAME && tar xfz $TARGET_HBA_VLBI_RESULTS_NAME && rm -f $TARGET_HBA_VLBI_RESULTS_NAME
artifacts:
paths:
- data
@@ -323,6 +325,23 @@ run_hba_target_selfcal:
- cal_solutions.tar.gz
when: on_failure
+run_hba_target_VLBI:
+ stage: run_tests
+ image: $INTEGRATION_IMAGE
+ script:
+ - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target_VLBI.cwl test_jobs/HBA_target.json
+ - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_target_VLBI
+ after_script:
+ - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_VLBI_logs.tar.gz --null -T -
+ - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
+ - find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
+ artifacts:
+ paths:
+ - hba_target_VLBI_logs.tar.gz
+ - inspection.tar.gz
+ - cal_solutions.tar.gz
+ when: on_failure
+
run_lba_calibrator:
extends: .run_tests
script:
diff --git a/docs/source/running.rst b/docs/source/running.rst
index dc5ccf8a8da3ac1c169feb8600db22affb17c45b..7a662291880a0953e2009207fdbd2114ab7f2a06 100644
--- a/docs/source/running.rst
+++ b/docs/source/running.rst
@@ -35,12 +35,13 @@ By default, **LINC** will execute the processing steps (like Dp3, etc.) inside a
The following list provides the workflows to call in the command above for standard LOFAR observations. These provide the proper pipeline with pre-defined parameters (defaults) for **HBA** and **LBA** observations:
-============================ ====================== =======================
+============================ ======================== =======================
**LINC workflow** **HBA** **LBA**
----------------------------- ---------------------- -----------------------
-``LINC_calibrator.cwl`` ``HBA_calibrator.cwl`` ``LBA_calibrator.cwl``
-``LINC_target.cwl`` ``HBA_target.cwl`` ``LBA_target.cwl``
-============================ ====================== =======================
+---------------------------- ------------------------ -----------------------
+``LINC_calibrator.cwl`` ``HBA_calibrator.cwl`` ``LBA_calibrator.cwl``
+``LINC_target.cwl`` ``HBA_target.cwl`` ``LBA_target.cwl``
+ ``HBA_target_VLBI.cwl``
+============================ ======================== =======================
.. note::
diff --git a/docs/source/target.rst b/docs/source/target.rst
index 1060131e745d755c70f9d9582f6841eccd87d758..8295726a251cddd4263f26efaeb6af11594bf850 100644
--- a/docs/source/target.rst
+++ b/docs/source/target.rst
@@ -348,6 +348,7 @@ User-defined parameter configuration
- ``selfcal_hba_uvlambdamin``: specify the minimum uv-distance in units of wavelength to be used when performing selfcal with HBA (default: 200)
- ``selfcal_hba_imsize``: specifies the image size in pixels, as a list, to use during HBA self-calibration (default: ``[20000, 20000]``).
- ``calib_nchan``: number of channels to be combined when performing phase calibration (default: ``0`` (one solution per group)). The self-calibration will always be performed with the highest available frequency resolution.
+- ``output_fullres_data``: additionally outputs full-resolution data sets concatenated according to ``num_SBs_per_group`` with final phase solutions and flags applied (default: ``false``)
A comprehensive explanation of the baseline selection syntax can be found `here`_.
diff --git a/steps/check_removed_groups.cwl b/steps/check_removed_groups.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..0ab987c85b9dde136f4abe7fc1eb0e008ed58173
--- /dev/null
+++ b/steps/check_removed_groups.cwl
@@ -0,0 +1,65 @@
+class: CommandLineTool
+cwlVersion: v1.2
+id: check_removed_groups
+baseCommand:
+ - python3
+arguments:
+ - position: 0
+ valueFrom: check_removed_groups.py
+inputs:
+ - id: groups_specification
+ type: File
+ inputBinding:
+ position: 1
+ - id: groupnames
+ type: string[]
+ - id: removed_bands
+ type: string[]?
+outputs:
+ - id: out_groups_specification
+ type: File
+ outputBinding:
+ glob: 'out.json'
+ - id: out_groupnames
+ type: string[]
+ outputBinding:
+ loadContents: true
+ glob: 'groupnames.json'
+ outputEval: '$(JSON.parse(self[0].contents).groupnames)'
+label: check_removed_groups
+
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: check_removed_groups.py
+ entry: |
+ import sys
+ import json
+ import os
+
+ null = []
+
+ inputs = json.loads(r"""$(inputs)""")
+ removed_bands = inputs['removed_bands']
+ groupnames = inputs['groupnames']
+ json_file = sys.argv[1]
+
+ with open(json_file, 'r') as f_stream:
+ all_groups = json.load(f_stream)
+
+ for removed_band in removed_bands:
+ if removed_band == 'None':
+ continue
+ all_groups.pop(removed_band)
+ groupnames.remove(removed_band)
+
+ cwl_output = {"groupnames" : groupnames}
+
+ with open('./out.json', 'w') as fp:
+ json.dump(all_groups, fp)
+
+ with open('./groupnames.json', 'w') as fp:
+ json.dump(cwl_output, fp)
+
+stdout: check_removed_groups.log
+stderr: check_removed_groups_err.log
\ No newline at end of file
diff --git a/steps/collectfiles.cwl b/steps/collectfiles.cwl
index c929ccef1d9cceab075c900acb54f7e89ef36734..4e34968d492b61e0a29f633062add86e5de64d22 100644
--- a/steps/collectfiles.cwl
+++ b/steps/collectfiles.cwl
@@ -45,3 +45,6 @@ requirements:
cp -rL $* $OUTPUT_PATH
writable: false
- class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: astronrd/linc
\ No newline at end of file
diff --git a/steps/dp3concat.cwl b/steps/dp3concat.cwl
index 6f4f7c81a028e8ec3146b8d1964807a0d538ec8e..faa445bf07195bdd7634be9d65ac27e163474b81 100644
--- a/steps/dp3concat.cwl
+++ b/steps/dp3concat.cwl
@@ -40,6 +40,15 @@ inputs:
shellQuote: false
position: 0
default: DATA
+ - id: baseline
+ default: null
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: msin.baseline=
+ shellQuote: true
+ separate: false
+ valueFrom: $(self)
- id: chunkduration
type: float?
inputBinding:
@@ -99,18 +108,10 @@ inputs:
separate: false
shellQuote: false
position: 0
- - id: baseline
- type: string?
- inputBinding:
- prefix: msin.baseline=
- separate: false
- shellQuote: true
- position: 0
- default: '*'
- id: avg_timeresolution
type: int?
inputBinding:
- prefix: avg.timeresolution=
+ prefix: average.timeresolution=
separate: false
shellQuote: false
position: 0
@@ -118,7 +119,7 @@ inputs:
- id: avg_freqresolution
type: string?
inputBinding:
- prefix: avg.freqresolution=
+ prefix: average.freqresolution=
separate: false
shellQuote: false
position: 0
@@ -135,6 +136,44 @@ inputs:
inputBinding:
prefix: count.jsonfilename=
separate: false
+ - id: steps
+ type: string?
+ default: ''
+ - id: parmdb
+ type: File?
+ inputBinding:
+ position: 0
+ prefix: applycal.parmdb=
+ separate: false
+ - id: correction
+ type: string?
+ default: phase000
+ inputBinding:
+ position: 0
+ prefix: applycal.correction=
+ separate: false
+ - id: solset
+ default: 'sol000'
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: applycal.solset=
+ separate: false
+ - id: flag_transfer_source_ms
+ type: Directory?
+ inputBinding:
+ position: 0
+ prefix: flagtransfer.source_ms=
+ separate: false
+ - id: flag_transfer_baselines
+ type: string?
+ inputBinding:
+ prefix: flagtransfer.baseline=
+ separate: false
+ shellQuote: true
+ position: 0
+ valueFrom: $(self)
+ default: null
outputs:
- id: msout
doc: Output Measurement Set
@@ -153,9 +192,8 @@ outputs:
outputBinding:
glob: concat*.log
arguments:
- - steps=[filter,avg,count]
+ - steps=[$(inputs.steps)]
- msin.orderms=False
- - avg.type=average
requirements:
- class: ShellCommandRequirement
- class: InlineJavascriptRequirement
diff --git a/steps/identify_bad_antennas_join.cwl b/steps/identify_bad_antennas_join.cwl
index e091e1a3219f98551f3ad0cc91c16f157701d95f..70343c82a12d078dec5c5aaf5ef0861b5f86ad23 100644
--- a/steps/identify_bad_antennas_join.cwl
+++ b/steps/identify_bad_antennas_join.cwl
@@ -33,11 +33,13 @@ requirements:
else:
flagged_antenna_list = set()
+ bad_antennas = ''
for flagged_antenna in flagged_antenna_list:
if flagged_antenna != '':
- filter += ';!' + flagged_antenna + '*&&*'
+ bad_antennas += ';!' + flagged_antenna + '*&&*'
- cwl_output = {"filter": filter}
+ cwl_output = {"filter": filter + bad_antennas,
+ "bad_antennas": bad_antennas.lstrip(';')}
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
@@ -49,6 +51,12 @@ outputs:
loadContents: true
glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).filter)
+ - id: bad_antennas
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).bad_antennas)
- id: logfile
type: File
outputBinding:
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 864834f667b23e84010948eda0d9d1019278774e..a17b3a0bb5e7a852dea33335db680fb3d6aa421c 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -184,6 +184,9 @@ inputs:
default: true
- id: skymodel_fluxlimit
type: float?
+ - id: output_fullres_data
+ type: boolean?
+ default: false
outputs:
- id: calibrated_data
outputSource:
@@ -330,6 +333,8 @@ steps:
source: skymodel_fluxlimit
- id: selfcal_hba_uvlambdamin
source: selfcal_hba_uvlambdamin
+ - id: output_fullres_data
+ source: output_fullres_data
out:
- id: logfiles
- id: msout
diff --git a/workflows/HBA_target_VLBI.cwl b/workflows/HBA_target_VLBI.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..f434f15d64b3b804dc970125bda9b46fdeacb016
--- /dev/null
+++ b/workflows/HBA_target_VLBI.cwl
@@ -0,0 +1,380 @@
+class: Workflow
+cwlVersion: v1.2
+id: HBA_target_VLBI
+label: HBA_target_VLBI
+inputs:
+ - id: msin
+ type: Directory[]
+ - id: cal_solutions
+ type: File
+ - id: refant
+ type: string?
+ default: 'CS00.*'
+ - id: flag_baselines
+ type: string[]?
+ default: []
+ - id: process_baselines_target
+ type: string?
+ default: '[CR]S*&'
+ - id: filter_baselines
+ type: string?
+ default: '*&'
+ - id: do_smooth
+ type: boolean?
+ default: false
+ - id: rfistrategy
+ type:
+ - File?
+ - string?
+ default: '$LINC_DATA_ROOT/rfistrategies/lofar-hba-wideband.lua'
+ - id: min_unflagged_fraction
+ type: float?
+ default: 0.5
+ - id: compression_bitrate
+ type: int?
+ default: 16
+ - id: raw_data
+ type: boolean?
+ default: false
+ - id: propagatesolutions
+ type: boolean?
+ default: true
+ - id: demix_sources
+ type: string[]?
+ default:
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
+ - id: demix_timeres
+ type: float?
+ default: 10
+ - id: demix_freqres
+ type: string?
+ - id: demix
+ type: boolean?
+ - id: apply_tec
+ type: boolean?
+ default: false
+ - id: apply_clock
+ type: boolean?
+ default: true
+ - id: apply_phase
+ type: boolean?
+ default: false
+ - id: apply_RM
+ type: boolean?
+ default: true
+ - id: apply_beam
+ type: boolean?
+ default: true
+ - id: gsmcal_step
+ type: string?
+ default: 'phase'
+ - id: updateweights
+ type: boolean?
+ default: true
+ - id: max_dp3_threads
+ type: int?
+ default: 10
+ - id: memoryperc
+ type: int?
+ default: 20
+ - id: min_separation
+ type: int?
+ default: 30
+ - id: A-Team_skymodel
+ type: File?
+ - id: target_skymodel
+ type: File?
+ - id: use_target
+ type: boolean?
+ default: true
+ - id: skymodel_source
+ type: string?
+ default: 'TGSS'
+ - id: avg_timeresolution
+ type: int?
+ default: 1
+ - id: avg_freqresolution
+ type: string?
+ default: 12.21kHz
+ - id: avg_timeresolution_concat
+ type: int?
+ default: 8
+ - id: avg_freqresolution_concat
+ type: string?
+ default: 97.64kHz
+ - id: num_SBs_per_group
+ type: int?
+ - id: calib_nchan
+ type: int?
+ - id: reference_stationSB
+ type: int?
+ default: null
+ - id: ionex_server
+ type: string?
+ default: 'http://chapman.upc.es/'
+ - id: ionex_prefix
+ type: string?
+ default: 'UQRG'
+ - id: proxy_server
+ type: string?
+ default: null
+ - id: proxy_port
+ type: int?
+ default: null
+ - id: proxy_type
+ type: string?
+ default: null
+ - id: proxy_user
+ type: string?
+ default: null
+ - id: proxy_pass
+ type: string?
+ default: null
+ - id: clip_sources
+ type: string[]?
+ default:
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
+ - id: clipAteam
+ type: boolean?
+ default: true
+ - id: lbfgs_historysize
+ type: int?
+ - id: lbfgs_robustdof
+ type: float?
+ - id: aoflag_reorder
+ type: boolean?
+ default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
+ - id: aoflag_freqconcat
+ type: boolean?
+ default: true
+ - id: selfcal
+ type: boolean?
+ default: false
+ - id: selfcal_strategy
+ type: string?
+ default: 'HBA'
+ - id: selfcal_hba_imsize
+ type: int[]?
+ default: [20000,20000]
+ - id: selfcal_hba_uvlambdamin
+ type: float?
+ default: 200.
+ - id: selfcal_region
+ type: File?
+ - id: chunkduration
+ type: float?
+ default: 0.0
+ - id: wsclean_tmpdir
+ type: string?
+ - id: make_structure_plot
+ type: boolean?
+ default: true
+ - id: skymodel_fluxlimit
+ type: float?
+ - id: output_fullres_data
+ type: boolean?
+ default: true
+outputs:
+ - id: calibrated_data
+ outputSource:
+ - save_results/dir
+ type: Directory
+ - id: log_files
+ outputSource:
+ - save_logfiles/dir
+ type: Directory
+ - id: inspection_plots
+ outputSource:
+ - save_inspection/dir
+ type: Directory
+ - id: summary
+ outputSource:
+ - linc/summary_file
+ type: File
+ - id: solutions
+ outputSource:
+ - linc/solutions
+ type: File
+steps:
+ - id: linc
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: cal_solutions
+ source: cal_solutions
+ - id: refant
+ source: refant
+ - id: flag_baselines
+ source:
+ - flag_baselines
+ - id: process_baselines_target
+ source: process_baselines_target
+ - id: filter_baselines
+ source: filter_baselines
+ - id: do_smooth
+ source: do_smooth
+ - id: rfistrategy
+ source: rfistrategy
+ - id: min_unflagged_fraction
+ source: min_unflagged_fraction
+ - id: compression_bitrate
+ source: compression_bitrate
+ - id: raw_data
+ source: raw_data
+ - id: propagatesolutions
+ source: propagatesolutions
+ - id: apply_tec
+ source: apply_tec
+ - id: apply_clock
+ source: apply_clock
+ - id: apply_phase
+ source: apply_phase
+ - id: apply_RM
+ source: apply_RM
+ - id: apply_beam
+ source: apply_beam
+ - id: demix_sources
+ source: demix_sources
+ - id: demix_freqres
+ source: demix_freqres
+ - id: demix_timeres
+ source: demix_timeres
+ - id: demix
+ source: demix
+ - id: updateweights
+ source: updateweights
+ - id: max_dp3_threads
+ source: max_dp3_threads
+ - id: memoryperc
+ source: memoryperc
+ - id: min_separation
+ source: min_separation
+ - id: A-Team_skymodel
+ source: A-Team_skymodel
+ - id: target_skymodel
+ source: target_skymodel
+ - id: use_target
+ source: use_target
+ - id: skymodel_source
+ source: skymodel_source
+ - id: avg_timeresolution
+ source: avg_timeresolution
+ - id: avg_freqresolution
+ source: avg_freqresolution
+ - id: avg_timeresolution_concat
+ source: avg_timeresolution_concat
+ - id: avg_freqresolution_concat
+ source: avg_freqresolution_concat
+ - id: num_SBs_per_group
+ source: num_SBs_per_group
+ - id: calib_nchan
+ source: calib_nchan
+ - id: reference_stationSB
+ source: reference_stationSB
+ - id: ionex_server
+ source: ionex_server
+ - id: ionex_prefix
+ source: ionex_prefix
+ - id: proxy_server
+ source: proxy_server
+ - id: proxy_port
+ source: proxy_port
+ - id: proxy_type
+ source: proxy_type
+ - id: proxy_user
+ source: proxy_user
+ - id: proxy_pass
+ source: proxy_pass
+ - id: clip_sources
+ source: clip_sources
+ - id: clipAteam
+ source: clipAteam
+ - id: gsmcal_step
+ source: gsmcal_step
+ - id: lbfgs_historysize
+ source: lbfgs_historysize
+ - id: lbfgs_robustdof
+ source: lbfgs_robustdof
+ - id: aoflag_reorder
+ source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
+ - id: aoflag_freqconcat
+ source: aoflag_freqconcat
+ - id: selfcal
+ source: selfcal
+ - id: selfcal_strategy
+ source: selfcal_strategy
+ - id: selfcal_hba_imsize
+ source: selfcal_hba_imsize
+ - id: selfcal_region
+ source: selfcal_region
+ - id: chunkduration
+ source: chunkduration
+ - id: wsclean_tmpdir
+ source: wsclean_tmpdir
+ - id: make_structure_plot
+ source: make_structure_plot
+ - id: skymodel_fluxlimit
+ source: skymodel_fluxlimit
+ - id: selfcal_hba_uvlambdamin
+ source: selfcal_hba_uvlambdamin
+ - id: output_fullres_data
+ source: output_fullres_data
+ out:
+ - id: logfiles
+ - id: msout
+ - id: solutions
+ - id: inspection
+ - id: summary_file
+ run: ./linc_target.cwl
+ label: linc_target
+ - id: save_logfiles
+ in:
+ - id: files
+ linkMerge: merge_flattened
+ source:
+ - linc/logfiles
+ - id: sub_directory_name
+ default: logs
+ out:
+ - id: dir
+ run: ./../steps/collectfiles.cwl
+ label: save_logfiles
+ - id: save_inspection
+ in:
+ - id: files
+ linkMerge: merge_flattened
+ source:
+ - linc/inspection
+ - id: sub_directory_name
+ default: inspection
+ out:
+ - id: dir
+ run: ./../steps/collectfiles.cwl
+ label: save_inspection
+ - id: save_results
+ in:
+ - id: files
+ linkMerge: merge_flattened
+ source:
+ - linc/msout
+ - id: sub_directory_name
+ default: results
+ out:
+ - id: dir
+ run: ./../steps/collectfiles.cwl
+ label: save_results
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/workflows/LBA_target.cwl b/workflows/LBA_target.cwl
index 48249d85b4cd60b543c4f42e8c0dfe5fee0bb497..8348713ac0d20a354b553ee5afb58c794df883e8 100644
--- a/workflows/LBA_target.cwl
+++ b/workflows/LBA_target.cwl
@@ -166,11 +166,14 @@ inputs:
- id: selfcal_strategy
type: string?
default: 'LBA'
- - id: selfcal_region
- type: File?
- id: selfcal_hba_imsize
type: int[]?
default: [20000,20000]
+ - id: selfcal_hba_uvlambdamin
+ type: float?
+ default: 200.
+ - id: selfcal_region
+ type: File?
- id: chunkduration
type: float?
default: 3600.0
@@ -182,6 +185,9 @@ inputs:
- id: skymodel_fluxlimit
type: float?
default: 1.0
+ - id: output_fullres_data
+ type: boolean?
+ default: false
outputs:
- id: calibrated_data
outputSource:
@@ -326,6 +332,10 @@ steps:
source: make_structure_plot
- id: skymodel_fluxlimit
source: skymodel_fluxlimit
+ - id: selfcal_hba_uvlambdamin
+ source: selfcal_hba_uvlambdamin
+ - id: output_fullres_data
+ source: output_fullres_data
out:
- id: logfiles
- id: msout
diff --git a/workflows/linc_target.cwl b/workflows/linc_target.cwl
index 7a5ef2327dc81e7aac1d58871c81b3a804b85553..3fdb354f51122071d31a9817ac83091c21a7c9ea 100644
--- a/workflows/linc_target.cwl
+++ b/workflows/linc_target.cwl
@@ -184,6 +184,9 @@ inputs:
type: File?
- id: skymodel_fluxlimit
type: float?
+ - id: output_fullres_data
+ type: boolean?
+ default: false
outputs:
- id: inspection
outputSource:
@@ -398,6 +401,7 @@ steps:
- id: msout
- id: outh5parm
- id: outsolutions
+ - id: filter_out
- id: bad_antennas
- id: outh5parm_logfile
- id: Ateam_flags_join_out
@@ -412,6 +416,8 @@ steps:
- id: msin
source:
- gsmcal/msout
+ - id: msin_hires
+ source: prep/msout
- id: input_h5parm
source: gsmcal/outh5parm
- id: inh5parm_logfile
@@ -420,6 +426,10 @@ steps:
source: gsmcal_step
- id: process_baselines_target
source: process_baselines_target
+ - id: filter_baselines
+ source: gsmcal/filter_out
+ - id: compare_stations_filter
+ source: prep/compare_stations_filter
- id: bad_antennas
source: gsmcal/bad_antennas
- id: insolutions
@@ -432,10 +442,6 @@ steps:
source: prep/total_bandwidth
- id: check_Ateam_separation.json
source: prep/check_Ateam_separation.json
- - id: filter_baselines
- source: filter_baselines
- - id: compare_stations_filter
- source: prep/compare_stations_filter
- id: flags
linkMerge: merge_flattened
source:
@@ -464,6 +470,14 @@ steps:
source: apply_RM
- id: ionex_prefix
source: ionex_prefix
+ - id: apply_solutions_fullres
+ source: output_fullres_data
+ - id: groupnames
+ source: prep/groupnames
+ - id: filenames
+ source: prep/filenames
+ - id: chunkduration
+ source: chunkduration
out:
- id: msout
- id: solutions
diff --git a/workflows/linc_target/concat.cwl b/workflows/linc_target/concat.cwl
index 8105370e19b897c36090e510c789ba8806983a8c..3173d8a2cd2f6bb0c241baf549ff75b5bf4986dc 100644
--- a/workflows/linc_target/concat.cwl
+++ b/workflows/linc_target/concat.cwl
@@ -9,6 +9,9 @@ inputs:
type: string
- id: groups_specification
type: File
+ - id: baselines
+ type: string?
+ default: '*&'
- id: filter_baselines
type: string?
default: '*&'
@@ -21,6 +24,20 @@ inputs:
- id: chunkduration
type: float?
default: 0.0
+ - id: steps
+ type: string?
+ default: ''
+ - id: parmdb
+ type: File?
+ - id: correction
+ type: string?
+ - id: solset
+ type: string?
+ - id: databitrate
+ type: int?
+ default: 0
+ - id: flag_transfer_source_ms
+ type: Directory?
outputs:
- id: flagged_fraction_dict
outputSource:
@@ -65,8 +82,10 @@ steps:
default: DATA
- id: msout_datacolumn
default: DATA
+ - id: baseline
+ source: baselines
- id: filter_baselines
- source: filter_baselines
+ source: baselines
- id: filter_remove
default: true
- id: overwrite
@@ -74,15 +93,25 @@ steps:
- id: storagemanager
default: Dysco
- id: databitrate
- default: 0
+ source: databitrate
- id: missingdata
default: true
- - id: baseline
- source: filter_baselines
- id: avg_timeresolution
source: avg_timeresolution_concat
- id: avg_freqresolution
source: avg_freqresolution_concat
+ - id: steps
+ source: steps
+ - id: flag_transfer_source_ms
+ source: flag_transfer_source_ms
+ - id: flag_transfer_baselines
+ source: filter_baselines
+ - id: parmdb
+ source: parmdb
+ - id: correction
+ source: correction
+ - id: solset
+ source: solset
out:
- id: msout
- id: flagged_fraction_dict
diff --git a/workflows/linc_target/finalize.cwl b/workflows/linc_target/finalize.cwl
index cf79ef324a9464ec88c009b56902d7a893889aa8..a588bb6b67589ad3303edda976ffc8b4c68f22be 100644
--- a/workflows/linc_target/finalize.cwl
+++ b/workflows/linc_target/finalize.cwl
@@ -23,6 +23,8 @@ inputs:
default: []
- id: msin
type: Directory[]
+ - id: msin_hires
+ type: Directory[]
- id: input_h5parm
type: File
- id: inh5parm_logfile
@@ -33,9 +35,12 @@ inputs:
- id: process_baselines_target
type: string?
default: '[CR]S*&'
- - id: bad_antennas
+ - id: filter_baselines
type: string?
default: '[CR]S*&'
+ - id: bad_antennas
+ type: string?
+ default: ''
- id: insolutions
type: File
- id: compression_bitrate
@@ -49,9 +54,6 @@ inputs:
default: 50
- id: check_Ateam_separation.json
type: File
- - id: filter_baselines
- type: string?
- default: '[CR]S*&'
- id: compare_stations_filter
type: string?
default: '[CR]S*&'
@@ -72,11 +74,24 @@ inputs:
- id: ionex_prefix
type: string?
default: UQRG
+ - id: apply_solutions_fullres
+ type: boolean?
+ default: false
+ - id: groupnames
+ type: string[]
+ - id: filenames
+ type: File
+ - id: chunkduration
+ type: float?
+ default: 0.0
outputs:
- id: msout
outputSource:
- apply_gsmcal_final/msout
+ - merge_array_concat_hires/output
type: Directory[]
+ pickValue: all_non_null
+ linkMerge: merge_flattened
- id: solutions
outputSource:
- h5parm_pointingname/outh5parm
@@ -89,6 +104,7 @@ outputs:
- concat_logfiles_wsclean/output
- concat_logfiles_summary/output
- concat_logfiles_uvplot/output
+ - concat_logfiles_hires/output
type: File[]
linkMerge: merge_flattened
pickValue: all_non_null
@@ -126,7 +142,7 @@ steps:
- id: filter
source: process_baselines_target
- id: bad_antennas
- source: bad_antennas
+ source: filter_baselines
out:
- id: outh5parm
- id: log
@@ -168,6 +184,75 @@ steps:
- msin
- msout_name
scatterMethod: dotproduct
+ - id: check_removed_groups
+ in:
+ - id: groups_specification
+ source: filenames
+ - id: groupnames
+ source: groupnames
+ - id: removed_bands
+ source: removed_bands
+ out:
+ - id: out_groups_specification
+ - id: out_groupnames
+ run: ../../steps/check_removed_groups.cwl
+ label: check_removed_groups
+ - id: apply_and_concat_hires
+ in:
+ - id: msin
+ source:
+ - msin_hires
+ - id: baselines
+ source: bad_antennas
+ - id: group_id
+ source: check_removed_groups/out_groupnames
+ - id: groups_specification
+ source: check_removed_groups/out_groups_specification
+ - id: filter_baselines
+ source: process_baselines_target
+ - id: chunkduration
+ source: chunkduration
+ - id: parmdb
+ source: write_solutions/outh5parm
+ - id: correction
+ source:
+ - skymodel_source
+ - gsmcal_step
+ valueFrom: $(self.join('')+"_final")
+ - id: solset
+ default: target
+ - id: databitrate
+ source: compression_bitrate
+ - id: steps
+ default: 'filter,flagtransfer,applycal,count'
+ - id: apply_solutions_fullres
+ source: apply_solutions_fullres
+ - id: flag_transfer_source_ms
+ source: msin
+ out:
+ - id: msout
+ - id: flagged_fraction_dict
+ - id: dp3concat.log
+ run: ./concat.cwl
+ label: apply_and_concat_hires
+ scatter:
+ - group_id
+ - flag_transfer_source_ms
+ scatterMethod: dotproduct
+ when: $(inputs.apply_solutions_fullres)
+ - id: merge_array_concat_hires
+ in:
+ - id: input
+ source:
+ - apply_and_concat_hires/msout
+ pickValue: all_non_null
+ - id: apply_solutions_fullres
+ source: apply_solutions_fullres
+ out:
+ - id: output
+ run: ../../steps/merge_array.cwl
+ label: merge_array_concat_hires
+ when: $(inputs.apply_solutions_fullres)
- id: final_flags_join
in:
- id: flagged_fraction_dict
@@ -221,10 +306,10 @@ steps:
- id: run_type
default: 'target'
- id: filter
- source: filter_baselines
+ source: process_baselines_target
- id: bad_antennas
source:
- - bad_antennas
+ - filter_baselines
- compare_stations_filter
valueFrom: $(self.join(''))
- id: Ateam_separation_file
@@ -454,6 +539,22 @@ steps:
- id: output
run: ../../steps/concatenate_files.cwl
label: concat_logfiles_wsclean
+ - id: concat_logfiles_hires
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - apply_and_concat_hires/dp3concat.log
+ pickValue: all_non_null
+ - id: file_prefix
+ default: apply_and_concat_hires
+ - id: apply_solutions_fullres
+ source: apply_solutions_fullres
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_hires
+ when: $(inputs.apply_solutions_fullres)
- id: concat_logfiles_summary
in:
- id: file_list
diff --git a/workflows/linc_target/gsmcal.cwl b/workflows/linc_target/gsmcal.cwl
index cf7e8a32dc64d979b000a5964dd1e09cb77668b2..57d74d49ded53f49d467700e9ea62a4294039a96 100644
--- a/workflows/linc_target/gsmcal.cwl
+++ b/workflows/linc_target/gsmcal.cwl
@@ -111,10 +111,14 @@ outputs:
- insolutions
pickValue: first_non_null
type: File
- - id: bad_antennas
+ - id: filter_out
outputSource:
- identifybadantennas_join/filter_out
type: string
+ - id: bad_antennas
+ outputSource:
+ - identifybadantennas_join/bad_antennas
+ type: string
- id: outh5parm_logfile
outputSource:
- concat_logfiles_losoto/output
@@ -180,6 +184,7 @@ steps:
source: filter_baselines
out:
- id: filter_out
+ - id: bad_antennas
- id: logfile
run: ../../steps/identify_bad_antennas_join.cwl
label: identifybadantennas_join
@@ -315,7 +320,7 @@ steps:
source: groupnames
- id: groups_specification
source: filenames
- - id: filter_baselines
+ - id: baselines
source: identifybadantennas_join/filter_out
- id: avg_timeresolution_concat
source: avg_timeresolution_concat
@@ -323,6 +328,8 @@ steps:
source: avg_freqresolution_concat
- id: chunkduration
source: chunkduration
+ - id: steps
+ default: 'filter,average,count'
out:
- id: msout
- id: flagged_fraction_dict