postprocessing script for widefield

scripts/astrometry.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Feb  3 11:15:16 2025
+
+@author: pwcc62
+"""
+
+from astropy.table import Table
+from astropy import units as u
+import matplotlib.pyplot as plt
+import argparse
+import os
+
+##example input below
+##python3 astrometry.py --crossmatch_fits "/home/pwcc62/AGN_outflows/Bootes_nonAGN/bootes_final_cross_match_catalogue-v1.0.fits" 
+#--source_fits "test_source_catalouge.fits" --ra1 "optRA" --ra2 "RA_bdsf" --dec1 "optDec" --dec2 "DEC_bdsf" --error 5
+
+#DR1 = Table.read('/home/pwcc62/AGN_outflows/Bootes_FITS/bootes_optIR_catalogue_full.fits')
+#source_cat = Table.read('/home/pwcc62/Bootes/test_source_catalog.fits')
+
+def astrometry(crossmatch_fits, source_fits, ra1, ra2, dec1, dec2, error):
+    
+    """
+    Script to determine the astrometry off set between e.g lotss and another image
+    
+    """
+
+    cmd1 = 'stilts tskymatch2 in1="{}" in2="{}" out=source_matches.fits ra1={} dec1={} ra2={} dec2={} error={} join=1and2'.format(crossmatch_fits, source_fits, ra1, dec1, ra2, dec2, error)
+    #cmd1 = 'stilts tskymatch2 in1=\"/home/pwcc62/AGN_outflows/Bootes_nonAGN/bootes_final_cross_match_catalogue-v1.0.fits\" in2="test_source_catalouge.fits" out=source_matches.fits ra1=optRA dec1=optDEC ra2=RA_bdsf dec2=DEC_bdsf error=1 join=1and2'
+    print(cmd1)
+    with open('match_test.sh','w') as f:
+        f.write(cmd1)
+        f.write('\n')
+
+    os.system('bash match_test.sh')
+
+    matches = Table.read("source_matches.fits") ## want RA_bdsf as this is from source_catalouge
+
+    bdsf_ra = matches['{}'.format(ra2)]
+    bdsf_dec = matches['{}'.format(dec2)]
+    opt_ra = matches['{}'.format(ra1)]
+    opt_dec = matches['{}'.format(dec1)]
+
+    ra_offset = (opt_ra - bdsf_ra) * u.deg
+    dec_offset = (opt_dec - bdsf_dec) * u.deg
+
+    ra_off_arcsec = ra_offset.to(u.arcsec)
+    dec_off_arcsec = dec_offset.to(u.arcsec)
+    ra_off = Table()
+    ra_off['ra_offset'] = ra_off_arcsec
+    dec_off = Table()
+    dec_off['dec_offset'] = dec_off_arcsec
+    ra_off.write('ra_offset.csv', format='csv', overwrite=True)
+    dec_off.write('dec_offset.csv', format='csv', overwrite=True)
+    
+    new_cmap = plt.cm.plasma(np.linspace(0,1,255))
+
+    fig = plt.figure(figsize=(12, 10))
+    grd = plt.GridSpec(4, 4, hspace=0.2, wspace=0.2)
+
+    ax = fig.add_subplot(grd[1:, :-1]) #main plot
+    lax = fig.add_subplot(grd[1:, -1], sharey=ax) #left plot
+    bax = fig.add_subplot(grd[0, :-1], sharex=ax) #top plot
+    #fig = plt.figure()
+    
+    ax.scatter(ra_off_arcsec, dec_off_arcsec, color=new_cmap[50], alpha=0.4, marker='*')
+    lax.hist(dec_off_arcsec, orientation='horizontal', color=new_cmap[50], alpha=0.4)
+    bax.hist(ra_off_arcsec, color=new_cmap[50], alpha=0.4)
+    
+    ax.set_xlabel('RA_offset')
+    ax.set_ylabel('DEC_offset')
+    plt.savefig('astrometry_offset.png')
+    
+    return ra_off, dec_off
+
+
+def parse_args():
+    
+     parser = argparse.ArgumentParser(description='Find Astrometry offset to 6" image')
+     parser.add_argument('--crossmatch_fits', type=str, help='6" catalouge with flux values')
+     parser.add_argument('--source_fits', type=str, help='source_catalouge from pyBDSF')
+     parser.add_argument('--ra1', type=str, help='column name of ra from crossmatch catalouge')
+     parser.add_argument('--ra2', type=str, help='column name of ra from pyBDSF catalouge')
+     parser.add_argument('--dec1', type=str, help='column name of dec from crossmatch catalouge')
+     parser.add_argument('--dec2', type=str, help='column name of dec from pyBDSF catalouge')
+     parser.add_argument('--error', type=str, help='Error for source location for crossmatching', default=5)
+
+     return parser.parse_args()
+ 
+def main():
+    """
+    Main function
+    """
+
+    args = parse_args()
+    astrometry(args.crossmatch_fits, args.source_fits, args.ra1, args.ra2, args.dec1, args.dec2, args.error)
+
+
+if __name__ == '__main__':
+    main()

scripts/flux_scaling.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Feb  3 15:12:32 2025
+
+@author: pwcc62
+"""
+from astropy.table import Table
+import matplotlib.pyplot as plt
+import argparse
+
+def flux_scale(fitsfile, lotss_flux, image_flux):
+    """
+    Script to determine the flux scaling required between lotss and an input image
+    """
+
+    matches = Table.read(fitsfile)
+
+    flux_6 = matches['{}'.format(lotss_flux)] ##Total_flux_1
+    bdsf_flux = matches['{}'.format(image_flux)] ##_2
+
+    flux_scale = bdsf_flux/flux_6
+
+    scale = Table()
+    scale['flux_scale'] = flux_scale
+    scale.write('flux_scale.csv', format='csv', overwrite=True)
+
+    plt.scatter(bdsf_flux, flux_6, color='purple', marker='o')
+    plt.xlabel('1.2" flux')
+    plt.ylabel('6" flux')
+    plt.savefig('flux_scaling.png')
+
+    return flux_scale
+
+def parse_args():
+    
+     parser = argparse.ArgumentParser(description='Find flux scaling between 6" and pyBDSF image')
+     parser.add_argument('--fitsfile', type=str, help='source_catalouge from pyBDSF with both image and 6" flux values')
+     parser.add_argument('--lotss_flux', type=str, help='column name of total flux from 6" catalouge')
+     parser.add_argument('--image_flux', type=str, help='column name of total flux from pyBDSF catalouge')
+     return parser.parse_args()
+ 
+def main():
+    """
+    Main function
+    """
+
+    args = parse_args()
+    flux_scale(args.fitsfile, args.lotss_flux, args.image_flux)
+
+if __name__ == '__main__':
+    main()
+ 
+

scripts/source_finding.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Feb  3 09:47:43 2025
+
+@author: pwcc62
+"""
+
+import bdsf
+import argparse
+from astropy.table import Table
+
+def source_finder(input_image, rmsbox, thresh_isl, thresh_pix):
+    """
+    Script to create source catalouge for an imput fits image
+    """
+    
+    prefix = input_image.split('/')[-1].replace('.fits','')
+    img = bdsf.process_image(input_image, thresh_isl=thresh_isl, thresh_pix=thresh_pix, rms_box=(int(rmsbox),int(rmsbox)), rms_box_bright=(int(rmsbox/3),int(rmsbox/3)))
+    img.write_catalog(clobber=True, outfile='source_catalouge.fits', format='fits', catalog_type='srl')
+    img = Table.read('source_catalouge.fits')
+    img_ra = img['RA']
+    img['RA'] = (img_ra + 360) % 360
+    img['RA'].name = 'RA_bdsf'
+    img['DEC'].name = 'DEC_bdsf'
+    img.write('source_catalouge.fits', overwrite=True, format='fits')
+    
+    return img
+
+def parse_args():
+    
+     parser = argparse.ArgumentParser(description='Source Finding with pyBDSF')
+     parser.add_argument('--rmsbox', type=float, help='rms box pybdsf', default=120)
+     parser.add_argument('--thresh_isl', type=float, help='sigma threshold for island detections with pybdsf', default=5)
+     parser.add_argument('--thresh_pix', type=float, help='sigma threshold for pixel with pybdsf', default=5)
+     parser.add_argument('--input_image', help='input image for source finding')
+     return parser.parse_args()
+ 
+
+def main():
+    """
+    Main function
+    """
+
+    args = parse_args()
+    source_finder(args.input_image, args.rmsbox, args.thresh_isl, args.thresh_pix)
+
+
+if __name__ == '__main__':
+    main()
+

steps/astrometry.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: astrometry
+doc: Script to determine astrometry offset between known positions e.g LoTSS and widefield VLBI image
+
+baseCommand:
+  - /home/pwcc62/Bootes/postprocessing/astrometry.py
+
+inputs:
+  - id: crossmatch_fits
+    type: File
+    doc: Fits file containing positions of sources to crossmatch with new image
+    inputBinding:
+      position: 0
+      prefix: '--crossmatch_fits'
+  - id: source_fits
+    type: File
+    doc: Fits file containing positions of source from image created using widefield mode
+    inputBinding:
+      position: 1
+      prefix: '--source_fits'
+  - id: ra1
+    type: string
+    doc: Column name of RA from crossmatch catalouge
+    inputBinding:
+      position: 2
+      prefix: '--ra1'
+  - id: ra2
+    type: string
+    doc: Column name of RA from source catalouge
+    inputBinding:
+      position: 3
+      prefix: '--ra2'
+  - id: dec1       
+    type: string
+    doc: Column name of Dec from crossmatch catalouge
+    inputBinding:
+      position: 4
+      prefix: '--dec1'
+  - id: dec2       
+    type: string
+    doc: Column name of Dec from source catalouge
+    inputBinding:
+      position: 5
+      prefix: '--dec2'
+  - id: error
+    type: float?
+    doc: Error for crossmatching region
+    default: 5
+    inputBinding:
+      position: 6
+      prefix: '--error'
+
+outputs:
+   - id: astrometry_plot
+     type: File
+     doc: Plot to probe the astrometry offsets between crossmatch catalouge and image catalouge
+     outputBinding:
+       glob: astrometry_offset.png
+   - id: ra_offset
+     type: File
+     doc: Table containing the RA offsets between the crossmatch catalouge and image catalouge
+     outputBinding:
+       glob: ra_offset.csv
+   - id: dec_offset
+     type: File
+     doc: Table containing the Dec offsets between the crossmatch catalouge and image catalouge
+     outputBinding:
+       glob: dec_offset.csv
+   - id: match_submission
+     type: File
+     doc: Bash script used to creat matched catalouge
+     outputBinding:
+       glob: match_test.sh
+   - id: source_matches
+     type: File
+     doc: Table containing the crossmatched catalouge containing both crossmatch and source positions
+     outputBinding:
+       glob: source_matches.fits

steps/flux_scaling.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: flux_scaling
+doc: Script to probe flux scaling required between 6" flux density and input image flux 
+
+baseCommand:
+  - /home/pwcc62/Bootes/postprocessing/flux_scaling.py
+
+inputs:
+  - id: fitsfile
+    type: File
+    doc: Fits file containing flux measurements for VLBI image which require flux scaling
+    inputBinding:
+      position: 0
+      prefix: '--fitsfile'
+  - id: lotss_flux
+    type: string
+    doc: Column name of total flux from LoTSS
+    inputBinding:
+      position: 1
+      prefix: '--lotss_flux'
+  - id: image_flux
+    type: string
+    doc: Column name of total flux from image catalouge which requires flux scaling
+    inputBinding:
+      position: 2
+      prefix: '--image_flux'
+
+outputs:
+   - id: flux_scale_plot
+     type: File
+     doc: Plot to probe the flux scaling between the lotss and VLBI image
+     outputBinding:
+       glob: flux_scaling.png
+   - id: flux_scale
+     type: File
+     doc: Table containing flux scaling values (input/6")
+     outputBinding:
+       glob: flux_scale.csv

steps/source_finding.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: source_finding
+doc: Generates a source catalouge of input image using pyBDSF
+
+baseCommand:
+  - /home/pwcc62/Bootes/postprocessing/source_finding.py
+
+inputs:
+  - id: input_image
+    type: File
+    doc: fits image to create catalouge from
+    inputBinding:
+      position: 0
+      prefix: '--input_image'
+  - id: rmsbox
+    type: float?
+    doc: Size in pixels of noise area for pyBDSF
+    default: 120
+    inputBinding:
+      position: 1
+      prefix: '--rmsbox'
+  - id: thresh_isl
+    type: float?
+    doc: Sigma threshold for island detections with pyBDSF                
+    default: 5
+    inputBinding:
+      position: 2
+      prefix: '--thresh_isl'
+  - id: thresh_pix
+    type: float?
+    doc: Sigma threshold for pixel detections with pyBDSF                
+    default: 5
+    inputBinding:
+      position: 3
+      prefix: '--thresh_pix'
+
+outputs:
+   - id: catalouge
+     type: File
+     doc: source catalouge
+     outputBinding:
+       glob: source_catalouge.fits

workflows/post-processing.cwl

+cwlVersion: v1.2
+class: Workflow
+id: Post-Processing
+label: Post Processing
+doc: |
+      This runs the post-processing step required to adjust wide-field images to comply with LoTSS fields. A catalouge will be generated using pyBDSF, astrometry offset will be analysed as well as the flux scaling require by comparing the image with 6".
+
+inputs:
+    - id: input_image
+      type: File
+      doc: The input image to produce catalouge
+    - id: rmsbox
+      type: float?
+      default: 120
+      doc: rmsbox for pyBDSF
+    - id: thresh_isl
+      type: float?
+      default: 5
+      doc: Sigma threshold for island detections with pyBDSF
+    - id: thresh_pix
+      type: float?
+      default: 5
+      doc: Sigma threshold for pixels with pyBDSF
+    - id: crossmatch_fits
+      type: File
+      doc: The catalouge to base astrometry of sources
+    - id: source_fits
+      type: File
+      doc: The catalouge created from the image astromerty needs to be corrected for
+    - id: ra1
+      type: string
+      doc: Column name of RA from crossmatch catalouge
+    - id: ra2
+      type: string
+      doc: Column name of RA from pyBDSF catalouge which needs correcting
+    - id: dec1
+      type: string
+      doc: Column name of Dec from crossmatch catalouge
+    - id: dec2
+      type: string
+      doc: Column name of Dec from pyBDSF catalouge which needs correcting
+    - id: error
+      type: float?
+      default: 5
+      doc: Error for source location in pyBDSF
+    - id: fitsfile
+      type: File
+      doc: Source catalouge with both image and 6" flux values
+    - id: lotss_flux
+      type: string
+      doc: Column name of total flux from 6" catalouge
+    - id: image_flux
+      type: string
+      doc: Column anme of total flux from image catalouge
+
+steps:
+        - id: source_finding
+          label: Source Finder
+          in:
+            - id: input_image
+              source: input_image
+            - id: rmsbox
+              source: rmsbox
+            - id: thresh_isl
+              source: thresh_isl
+            - id: thresh_pix
+              source: thresh_pix
+          out:
+            - id: catalouge
+          run: source_finding.cwl
+
+        - id: astrometry
+          label: astrometry
+          in:
+            - id: crossmatch_fits
+              source: crossmatch_fits
+            - id: source_fits
+              source: source_finding/catalouge
+            - id: ra1
+              source: ra1
+            - id: ra2
+              source: ra2
+            - id: dec1
+              source: dec1
+            - id: dec2
+              source: dec2
+            - id: error
+              source: error
+          out:
+            - id: ra_offset
+            - id: dec_offset
+            - id: astrometry_plot
+            - id: match_submission
+            - id: source_matches
+          run: astrometry.cwl
+
+        - id: flux_scaling
+          label: Flux Scaling
+          in:
+             - id: fitsfile
+               source: astrometry/source_matches
+             - id: lotss_flux
+               source: lotss_flux
+             - id: image_flux
+               source: image_flux
+          out:
+             - id: flux_scale
+             - id: flux_scale_plot
+          run: flux_scaling.cwl
+outputs:
+        - id: source_finding_out
+          type: File
+          outputSource: source_finding/catalouge
+        - id: astrometry_out
+          type: File[]
+          outputSource:
+            - astrometry/ra_offset
+            - astrometry/dec_offset
+            - astrometry/astrometry_plot
+            - astrometry/match_submission
+            - astrometry/source_matches
+        - id: flux_scaling_out
+          type: File[]
+          outputSource:
+            - flux_scaling/flux_scale
+            - flux_scaling/flux_scale_plot
+
+requirements:
+  - class: MultipleInputFeatureRequirement