Change default server info for RMextract

.gitlab-ci.yml

@@ -183,6 +183,9 @@ run_hba_target_selfcal:
   artifacts:
     paths:
       - hba_target_selfcal_logs.tar.gz
+      - cal_solutions.tar.gz
+      - inspection.tar.gz
+      - results.tar.gz
     when: on_failure
 
 build_doc:

docs/source/preparation.rst

@@ -14,7 +14,7 @@ typically obtained from the LOFAR `Long-Term Archive`_.
   the full bandwidth.
 
 - For the target pipeline you will need to have internet access from the machine you are running **LINC**.
-  It is required in order to retrieve RM values from `CODE`_ and a global sky model (`TGSS`_ or `GSM`_). Both are hosted as online services.
+  It is required in order to retrieve RM values from `GSSC`_ and a global sky model (`TGSS`_ or `GSM`_). Both are hosted as online services.
   It is also possible to provide an own target skymodel to **LINC** (using the parameters ``target_skymodel``, and ``use_target``, see the :doc:`target<target>` pipeline parameter information).
 
 .. note::
@@ -25,7 +25,7 @@ typically obtained from the LOFAR `Long-Term Archive`_.
     
     Older versions of **LINC**: All input measurement-sets for one pipeline run need to be in the same directory.
 
-.. _CODE: http://ftp.aiub.unibe.ch/CODE/
+.. _GSSC: ftp://gssc.esa.int/gnss/products/ionex/
 .. _TGSS: http://tgssadr.strw.leidenuniv.nl/doku.php
 .. _GSM:  http://172.104.228.177/
 .. _Long-Term Archive: https://lta.lofar.eu

docs/source/target.rst

@@ -338,8 +338,8 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
 
 *RMextract settings*
 
-- ``ionex_server``: URL of the *IONEX* server (default: ``"http://ftp.aiub.unibe.ch/CODE/"``)
-- ``ionex_prefix``: the prefix of the *IONEX* files (default: ``CODG``)
+- ``ionex_server``: URL of the *IONEX* server (default: ``"ftp://gssc.esa.int/gnss/products/ionex/"``)
+- ``ionex_prefix``: the prefix of the *IONEX* files (default: ``UQRG``)
 - ``proxy_server``: specify URL or IP of proxy server if needed
 - ``proxy_port``: port of proxy server if needed
 - ``proxy_user``: user name of proxy server if needed

scripts/createRMh5parm.py

@@ -44,8 +44,8 @@ def makesolset(MS, data, solset_name):
 
 
 def main(MSfiles, h5parmdb, solset_name = "sol000",all_stations=True,timestepRM=300,
-                ionex_server="http://ftp.aiub.unibe.ch/CODE/",
-                ionex_prefix='CODG',ionexPath="./",earth_rot=0,proxyServer=None,proxyPort=None,proxyType=None,proxyUser=None,proxyPass=None):
+                ionex_server="ftp://gssc.esa.int/gnss/products/ionex/",
+                ionex_prefix='UQRG',ionexPath="./",earth_rot=0,proxyServer=None,proxyPort=None,proxyType=None,proxyUser=None,proxyPass=None):
     '''Add rotation measure to existing h5parmdb
     
     Args:
@@ -214,10 +214,10 @@ if __name__ == '__main__':
                         help='MS for which the parmdb should be created.')
     parser.add_argument('h5parm', type=str,
                         help='H5parm to which the results of the CommonRotationAngle is added.')
-    parser.add_argument('--server', type=str, default='http://ftp.aiub.unibe.ch/CODE/',
-                        help='URL of the server to use. (default: http://ftp.aiub.unibe.ch/CODE/)')
-    parser.add_argument('--prefix', type=str, default='CODG',
-                        help='Prefix of the IONEX files. (default: \"CODG\")')
+    parser.add_argument('--server', type=str, default='ftp://gssc.esa.int/gnss/products/ionex/',
+                        help='URL of the server to use. (default: ftp://gssc.esa.int/gnss/products/ionex/)')
+    parser.add_argument('--prefix', type=str, default='UQRG',
+                        help='Prefix of the IONEX files. (default: \"UQRG\")')
     parser.add_argument('--ionexpath', '--path', type=str, default='./',
                         help='Directory where to store the IONEX files. (default: \"./\")')
     parser.add_argument('--solsetName', '--solset', type=str, default='sol000',

scripts/make_summary.py

@@ -7,7 +7,7 @@ import json
 import numpy
 
 ###############################################################################
-def main(flagFiles = None, pipeline = 'LINC', run_type = 'calibrator', filtered_antennas = '[CR]S*&', bad_antennas = '[CR]S*&', output_fname = None, structure_file = None, Ateam_separation_file = None, solutions = None, clip_sources = '', demix_sources = '', demix = False, removed_bands = '', min_unflagged = 0.5, refant = ''):
+def main(flagFiles = None, pipeline = 'LINC', run_type = 'calibrator', filtered_antennas = '[CR]S*&', bad_antennas = '[CR]S*&', output_fname = None, structure_file = None, Ateam_separation_file = None, solutions = None, clip_sources = '', demix_sources = '', demix = False, removed_bands = '', min_unflagged = 0.5, refant = '', ionex = None):
 	"""
 	Creates summary of a given LINC run
 	
@@ -152,6 +152,11 @@ def main(flagFiles = None, pipeline = 'LINC', run_type = 'calibrator', filtered_
 			except:
 				json_output['metrics'][pipeline]['stations'].append(  {'station' : antenna, 'removed' : 'yes'})
 		f.close()
+	
+	## Used RMextract parameters
+	if not ionex:
+		ionex = 'NONE'
+	print('IONEX data used for this calibration run: ' + str(ionex) + '\n')
 
 	## printing results human readable
 	antennas = sorted([item['station'] for (i, item) in enumerate(station_statistics)])
@@ -235,10 +240,11 @@ if __name__=='__main__':
 	parser.add_argument('--removed_bands', type=str, default='', help='Comma-separated list of bands that were removed from the data')
 	parser.add_argument('--min_unflagged', type=float, default=0.5, help='Minimum fraction of unflagged data per band')
 	parser.add_argument('--refant', type=str, default='', help='Reference antenna used')
+	parser.add_argument('--ionex', type=str, default=None, help='Used IONEX data for calibration')
 	
 	args = parser.parse_args()
 	
 	# start running script
-	main(flagFiles = args.flagFiles, pipeline = args.pipeline, run_type = args.run_type, filtered_antennas = args.filtered_antennas, bad_antennas = args.bad_antennas, output_fname = args.output_fname, structure_file = args.structure_file, Ateam_separation_file = args.Ateam_separation_file, solutions = args.solutions, clip_sources = args.clip_sources, demix_sources = args.demix_sources, demix = args.demix, removed_bands = args.removed_bands, min_unflagged = args.min_unflagged, refant = args.refant)
+	main(flagFiles = args.flagFiles, pipeline = args.pipeline, run_type = args.run_type, filtered_antennas = args.filtered_antennas, bad_antennas = args.bad_antennas, output_fname = args.output_fname, structure_file = args.structure_file, Ateam_separation_file = args.Ateam_separation_file, solutions = args.solutions, clip_sources = args.clip_sources, demix_sources = args.demix_sources, demix = args.demix, removed_bands = args.removed_bands, min_unflagged = args.min_unflagged, refant = args.refant, ionex = args.ionex)
 	
 	sys.exit(0)

steps/createRMh5parm.cwl

@@ -21,14 +21,14 @@ inputs:
   - id: ionex_server
     type: string?
     doc: IONEX Server path
-    default: 'http://ftp.aiub.unibe.ch/CODE/'
+    default: 'ftp://gssc.esa.int/gnss/products/ionex/'
     inputBinding:
       prefix: '--server'
       position: 0
   - id: ionex_prefix
     type: string?
     doc: IONEX Prefix
-    default: 'CODG'
+    default: 'UQRG'
     inputBinding:
       prefix: '--prefix'
       position: 0

steps/summary.cwl

@@ -129,6 +129,17 @@ inputs:
       position: 0
       prefix: --output_fname
     doc: Name of the output file
+  - id: ionex
+    default: false
+    type:
+      - string?
+      - boolean?
+    inputBinding:
+      position: 0
+      prefix: --ionex
+      separate: true
+      itemSeparator: ','
+      valueFrom: '$(self)'
 outputs:
   - id: summary_file
     doc: Summary File in JSON format

workflows/HBA_target.cwl

@@ -115,10 +115,10 @@ inputs:
     default: null
   - id: ionex_server
     type: string?
-    default: 'http://ftp.aiub.unibe.ch/CODE/'
+    default: 'ftp://gssc.esa.int/gnss/products/ionex/'
   - id: ionex_prefix
     type: string?
-    default: 'CODG'
+    default: 'UQRG'
   - id: proxy_server
     type: string?
     default: null

workflows/LBA_target.cwl

@@ -115,10 +115,10 @@ inputs:
     default: null
   - id: ionex_server
     type: string?
-    default: 'http://ftp.aiub.unibe.ch/CODE/'
+    default: 'ftp://gssc.esa.int/gnss/products/ionex/'
   - id: ionex_prefix
     type: string?
-    default: 'CODG'
+    default: 'UQRG'
   - id: proxy_server
     type: string?
     default: null

workflows/linc_target.cwl

@@ -115,10 +115,10 @@ inputs:
     default: null
   - id: ionex_server
     type: string?
-    default: 'http://ftp.aiub.unibe.ch/CODE/'
+    default: 'ftp://gssc.esa.int/gnss/products/ionex/'
   - id: ionex_prefix
     type: string?
-    default: CODG
+    default: UQRG
   - id: proxy_server
     type: string?
     default: null
@@ -465,6 +465,10 @@ steps:
         source: wsclean_tmpdir
       - id: make_structure_plot
         source: make_structure_plot
+      - id: apply_RM
+        source: apply_RM
+      - id: ionex_prefix
+        source: ionex_prefix
       - id: selfcal
         source: selfcal
       - id: selfcal_strategy

workflows/linc_target/finalize.cwl

@@ -66,6 +66,12 @@ inputs:
   - id: make_structure_plot
     type: boolean?
     default: true
+  - id: apply_RM
+    type: boolean?
+    default: true
+  - id: ionex_prefix
+    type: string?
+    default: UQRG
   - id: selfcal
     type: boolean?
     default: false
@@ -267,6 +273,11 @@ steps:
         source: min_unflagged_fraction
       - id: refant
         source: refant
+      - id: ionex
+        source:
+          - apply_RM
+          - ionex_prefix
+        valueFrom: '$(self[0] ? self[1] : false)'
     out:
       - id: summary_file
       - id: logfile

workflows/linc_target/prep.cwl

@@ -87,10 +87,10 @@ inputs:
     default: 48.82kHz
   - id: ionex_server
     type: string?
-    default: 'http://ftp.aiub.unibe.ch/CODE/'
+    default: 'ftp://gssc.esa.int/gnss/products/ionex/'
   - id: ionex_prefix
     type: string?
-    default: CODG
+    default: UQRG
   - id: proxy_server
     type: string?
     default: null