diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 78ac04789d6f5a3867267962cf5e76eef81507fe..ff1c34bd0b1a8c0b9fc70861b427f5be24cf55d6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -183,6 +183,9 @@ run_hba_target_selfcal:
artifacts:
paths:
- hba_target_selfcal_logs.tar.gz
+ - cal_solutions.tar.gz
+ - inspection.tar.gz
+ - results.tar.gz
when: on_failure
build_doc:
diff --git a/docs/source/preparation.rst b/docs/source/preparation.rst
index caf981ee6904914da109ce0c3c0cb07fc2a2dcf2..2b054a233b826856b8812d03fa3d64798cbe8c22 100644
--- a/docs/source/preparation.rst
+++ b/docs/source/preparation.rst
@@ -14,7 +14,7 @@ typically obtained from the LOFAR `Long-Term Archive`_.
the full bandwidth.
- For the target pipeline you will need to have internet access from the machine you are running **LINC**.
- It is required in order to retrieve RM values from `CODE`_ and a global sky model (`TGSS`_ or `GSM`_). Both are hosted as online services.
+ It is required in order to retrieve RM values from `GSSC`_ and a global sky model (`TGSS`_ or `GSM`_). Both are hosted as online services.
It is also possible to provide an own target skymodel to **LINC** (using the parameters ``target_skymodel``, and ``use_target``, see the :doc:`target<target>` pipeline parameter information).
.. note::
@@ -25,7 +25,7 @@ typically obtained from the LOFAR `Long-Term Archive`_.
Older versions of **LINC**: All input measurement-sets for one pipeline run need to be in the same directory.
-.. _CODE: http://ftp.aiub.unibe.ch/CODE/
+.. _GSSC: ftp://gssc.esa.int/gnss/products/ionex/
.. _TGSS: http://tgssadr.strw.leidenuniv.nl/doku.php
.. _GSM: http://172.104.228.177/
.. _Long-Term Archive: https://lta.lofar.eu
diff --git a/docs/source/target.rst b/docs/source/target.rst
index c7fc87212d5b9cbcb5818ea27a6e4897cce9aa49..0b5fc508269797b833fd49a3c3cf79e561afafcd 100644
--- a/docs/source/target.rst
+++ b/docs/source/target.rst
@@ -338,8 +338,8 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
*RMextract settings*
-- ``ionex_server``: URL of the *IONEX* server (default: ``"http://ftp.aiub.unibe.ch/CODE/"``)
-- ``ionex_prefix``: the prefix of the *IONEX* files (default: ``CODG``)
+- ``ionex_server``: URL of the *IONEX* server (default: ``"ftp://gssc.esa.int/gnss/products/ionex/"``)
+- ``ionex_prefix``: the prefix of the *IONEX* files (default: ``UQRG``)
- ``proxy_server``: specify URL or IP of proxy server if needed
- ``proxy_port``: port of proxy server if needed
- ``proxy_user``: user name of proxy server if needed
diff --git a/scripts/createRMh5parm.py b/scripts/createRMh5parm.py
index cf288f0a0d498b6d84f3b10c928cde51698d10f7..fccbbbbc338384b84cf9e353f8240774c16d4529 100755
--- a/scripts/createRMh5parm.py
+++ b/scripts/createRMh5parm.py
@@ -44,8 +44,8 @@ def makesolset(MS, data, solset_name):
def main(MSfiles, h5parmdb, solset_name = "sol000",all_stations=True,timestepRM=300,
- ionex_server="http://ftp.aiub.unibe.ch/CODE/",
- ionex_prefix='CODG',ionexPath="./",earth_rot=0,proxyServer=None,proxyPort=None,proxyType=None,proxyUser=None,proxyPass=None):
+ ionex_server="ftp://gssc.esa.int/gnss/products/ionex/",
+ ionex_prefix='UQRG',ionexPath="./",earth_rot=0,proxyServer=None,proxyPort=None,proxyType=None,proxyUser=None,proxyPass=None):
'''Add rotation measure to existing h5parmdb
Args:
@@ -214,10 +214,10 @@ if __name__ == '__main__':
help='MS for which the parmdb should be created.')
parser.add_argument('h5parm', type=str,
help='H5parm to which the results of the CommonRotationAngle is added.')
- parser.add_argument('--server', type=str, default='http://ftp.aiub.unibe.ch/CODE/',
- help='URL of the server to use. (default: http://ftp.aiub.unibe.ch/CODE/)')
- parser.add_argument('--prefix', type=str, default='CODG',
- help='Prefix of the IONEX files. (default: \"CODG\")')
+ parser.add_argument('--server', type=str, default='ftp://gssc.esa.int/gnss/products/ionex/',
+ help='URL of the server to use. (default: ftp://gssc.esa.int/gnss/products/ionex/)')
+ parser.add_argument('--prefix', type=str, default='UQRG',
+ help='Prefix of the IONEX files. (default: \"UQRG\")')
parser.add_argument('--ionexpath', '--path', type=str, default='./',
help='Directory where to store the IONEX files. (default: \"./\")')
parser.add_argument('--solsetName', '--solset', type=str, default='sol000',
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index 8d967fe18c46a5eaf7437a6b57b8216787e5602d..90484c3aae31730d51d4dc31661537ece87f5276 100755
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -7,7 +7,7 @@ import json
import numpy
###############################################################################
-def main(flagFiles = None, pipeline = 'LINC', run_type = 'calibrator', filtered_antennas = '[CR]S*&', bad_antennas = '[CR]S*&', output_fname = None, structure_file = None, Ateam_separation_file = None, solutions = None, clip_sources = '', demix_sources = '', demix = False, removed_bands = '', min_unflagged = 0.5, refant = ''):
+def main(flagFiles = None, pipeline = 'LINC', run_type = 'calibrator', filtered_antennas = '[CR]S*&', bad_antennas = '[CR]S*&', output_fname = None, structure_file = None, Ateam_separation_file = None, solutions = None, clip_sources = '', demix_sources = '', demix = False, removed_bands = '', min_unflagged = 0.5, refant = '', ionex = None):
"""
Creates summary of a given LINC run
@@ -152,6 +152,11 @@ def main(flagFiles = None, pipeline = 'LINC', run_type = 'calibrator', filtered_
except:
json_output['metrics'][pipeline]['stations'].append( {'station' : antenna, 'removed' : 'yes'})
f.close()
+
+ ## Used RMextract parameters
+ if not ionex:
+ ionex = 'NONE'
+ print('IONEX data used for this calibration run: ' + str(ionex) + '\n')
## printing results human readable
antennas = sorted([item['station'] for (i, item) in enumerate(station_statistics)])
@@ -235,10 +240,11 @@ if __name__=='__main__':
parser.add_argument('--removed_bands', type=str, default='', help='Comma-separated list of bands that were removed from the data')
parser.add_argument('--min_unflagged', type=float, default=0.5, help='Minimum fraction of unflagged data per band')
parser.add_argument('--refant', type=str, default='', help='Reference antenna used')
+ parser.add_argument('--ionex', type=str, default=None, help='Used IONEX data for calibration')
args = parser.parse_args()
# start running script
- main(flagFiles = args.flagFiles, pipeline = args.pipeline, run_type = args.run_type, filtered_antennas = args.filtered_antennas, bad_antennas = args.bad_antennas, output_fname = args.output_fname, structure_file = args.structure_file, Ateam_separation_file = args.Ateam_separation_file, solutions = args.solutions, clip_sources = args.clip_sources, demix_sources = args.demix_sources, demix = args.demix, removed_bands = args.removed_bands, min_unflagged = args.min_unflagged, refant = args.refant)
+ main(flagFiles = args.flagFiles, pipeline = args.pipeline, run_type = args.run_type, filtered_antennas = args.filtered_antennas, bad_antennas = args.bad_antennas, output_fname = args.output_fname, structure_file = args.structure_file, Ateam_separation_file = args.Ateam_separation_file, solutions = args.solutions, clip_sources = args.clip_sources, demix_sources = args.demix_sources, demix = args.demix, removed_bands = args.removed_bands, min_unflagged = args.min_unflagged, refant = args.refant, ionex = args.ionex)
sys.exit(0)
diff --git a/steps/createRMh5parm.cwl b/steps/createRMh5parm.cwl
index cd4eb114602ad58eb887cc08f7ec8f592c588e2c..7f1fb4c243befd6e48f714bd6e2be1dde3ed217e 100644
--- a/steps/createRMh5parm.cwl
+++ b/steps/createRMh5parm.cwl
@@ -21,14 +21,14 @@ inputs:
- id: ionex_server
type: string?
doc: IONEX Server path
- default: 'http://ftp.aiub.unibe.ch/CODE/'
+ default: 'ftp://gssc.esa.int/gnss/products/ionex/'
inputBinding:
prefix: '--server'
position: 0
- id: ionex_prefix
type: string?
doc: IONEX Prefix
- default: 'CODG'
+ default: 'UQRG'
inputBinding:
prefix: '--prefix'
position: 0
diff --git a/steps/summary.cwl b/steps/summary.cwl
index 99b256ad9586e3080cf0f408d5cf8d83afd8959a..45f0de39a9c75be45a863739452b01f82b09943e 100644
--- a/steps/summary.cwl
+++ b/steps/summary.cwl
@@ -129,6 +129,17 @@ inputs:
position: 0
prefix: --output_fname
doc: Name of the output file
+ - id: ionex
+ default: false
+ type:
+ - string?
+ - boolean?
+ inputBinding:
+ position: 0
+ prefix: --ionex
+ separate: true
+ itemSeparator: ','
+ valueFrom: '$(self)'
outputs:
- id: summary_file
doc: Summary File in JSON format
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 27edfe6b3d3170df9e930fd8cdad731757e98dda..5df447f08b75bc99c697589dfed6f6774f947884 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -115,10 +115,10 @@ inputs:
default: null
- id: ionex_server
type: string?
- default: 'http://ftp.aiub.unibe.ch/CODE/'
+ default: 'ftp://gssc.esa.int/gnss/products/ionex/'
- id: ionex_prefix
type: string?
- default: 'CODG'
+ default: 'UQRG'
- id: proxy_server
type: string?
default: null
diff --git a/workflows/LBA_target.cwl b/workflows/LBA_target.cwl
index 49f9d45b0b8358620922caeebb292e75611d71ff..ffd2efca462d8e71bcf480069006f8767c2810c1 100644
--- a/workflows/LBA_target.cwl
+++ b/workflows/LBA_target.cwl
@@ -115,10 +115,10 @@ inputs:
default: null
- id: ionex_server
type: string?
- default: 'http://ftp.aiub.unibe.ch/CODE/'
+ default: 'ftp://gssc.esa.int/gnss/products/ionex/'
- id: ionex_prefix
type: string?
- default: 'CODG'
+ default: 'UQRG'
- id: proxy_server
type: string?
default: null
diff --git a/workflows/linc_target.cwl b/workflows/linc_target.cwl
index 6174b8e867f74fc44aa1535fdf44efa862c6989d..4ff7af68d8115ebca7df7a2bcda092ddb414b286 100644
--- a/workflows/linc_target.cwl
+++ b/workflows/linc_target.cwl
@@ -115,10 +115,10 @@ inputs:
default: null
- id: ionex_server
type: string?
- default: 'http://ftp.aiub.unibe.ch/CODE/'
+ default: 'ftp://gssc.esa.int/gnss/products/ionex/'
- id: ionex_prefix
type: string?
- default: CODG
+ default: UQRG
- id: proxy_server
type: string?
default: null
@@ -465,6 +465,10 @@ steps:
source: wsclean_tmpdir
- id: make_structure_plot
source: make_structure_plot
+ - id: apply_RM
+ source: apply_RM
+ - id: ionex_prefix
+ source: ionex_prefix
- id: selfcal
source: selfcal
- id: selfcal_strategy
diff --git a/workflows/linc_target/finalize.cwl b/workflows/linc_target/finalize.cwl
index 6dd849a62a363ea8e6960adcf60201139254cd83..32211d1584a86aa3c82466dc1eaa4375fff5b369 100644
--- a/workflows/linc_target/finalize.cwl
+++ b/workflows/linc_target/finalize.cwl
@@ -66,6 +66,12 @@ inputs:
- id: make_structure_plot
type: boolean?
default: true
+ - id: apply_RM
+ type: boolean?
+ default: true
+ - id: ionex_prefix
+ type: string?
+ default: UQRG
- id: selfcal
type: boolean?
default: false
@@ -267,6 +273,11 @@ steps:
source: min_unflagged_fraction
- id: refant
source: refant
+ - id: ionex
+ source:
+ - apply_RM
+ - ionex_prefix
+ valueFrom: '$(self[0] ? self[1] : false)'
out:
- id: summary_file
- id: logfile
diff --git a/workflows/linc_target/prep.cwl b/workflows/linc_target/prep.cwl
index b5f11754281fe00b036e4123b5d9ec4fd2dd36c1..f52973202ca10eff08493c8e987d08dfdfa334da 100644
--- a/workflows/linc_target/prep.cwl
+++ b/workflows/linc_target/prep.cwl
@@ -87,10 +87,10 @@ inputs:
default: 48.82kHz
- id: ionex_server
type: string?
- default: 'http://ftp.aiub.unibe.ch/CODE/'
+ default: 'ftp://gssc.esa.int/gnss/products/ionex/'
- id: ionex_prefix
type: string?
- default: CODG
+ default: UQRG
- id: proxy_server
type: string?
default: null