Merge branch 'RAP-176' into 'master'

Resolve RAP-176

Closes RAP-176

See merge request eosc/prefactor3-cwl!31

Former-commit-id: 32ba306b

.gitlab-ci.yml

@@ -8,7 +8,6 @@ stages:
   - download_data
   - validate_cwl_scripts
   - test_steps
-  - integration_test
 
 before_script:
   - mkdir workdir
@@ -52,10 +51,4 @@ check_ateam_separation:
   allow_failure: true
   needs: ["download_data"]
   script:
-    - cwltool --no-container steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
-
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
\ No newline at end of file
-prefactor_calibrator:
-  stage: integration_test
-  needs: ["download_data"]
-  script:
-    - cwltool --no-container --preserve-environment PYTHONPATH --preserve-environment PREFACTOR_DATA_ROOT workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json

Docker/build_docker.sh

@@ -3,8 +3,8 @@ set -e
 
 DOCKER_TAG=latest
 
-docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
-docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
+#docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
+#docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
 
-docker build ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
+docker build --no-cache ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
 docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}

steps/check_ateam_separation.cwl

@@ -29,6 +29,11 @@ outputs:
     type: File?
     outputBinding:
       glob: $(inputs.output_image_name)
+  - id: output_json
+    doc: Output JSON
+    type: File
+    outputBinding:
+      glob: '*.json'
   - id: logfile
     type: File?
     outputBinding:

steps/findRefAnt_join.cwl

@@ -15,6 +15,10 @@ inputs:
       type: string?
       default: '*&'
       doc: Filter these baselines for the comparison
+    - id: state
+      type: string?
+      default: 'NONE'
+      doc: Provide state information for collecting antenna statistics
       
 label: findRefAnt_join
 
@@ -25,6 +29,10 @@ outputs:
         loadContents: true
         glob: 'out.json'
         outputEval: $(JSON.parse(self[0].contents).refant)
+  - id: flagged_fraction_antenna
+    type: File?
+    outputBinding:
+      glob: 'flagged_fraction_antenna.json'
   - id: logfile
     type: File?
     outputBinding:
@@ -69,10 +77,13 @@ requirements:
 
             flagged_fraction_list = []
             sorted_stations = sorted(flagged_fraction_data.keys())
+            
+            flagged_fraction_antenna = {}
 
             for antenna in sorted_stations:
                 flagged_fraction = sum(flagged_fraction_data[antenna]) / len(flagged_fraction_data[antenna])
                 flagged_fraction_list.append(flagged_fraction)
+                flagged_fraction_antenna[antenna] = flagged_fraction
                 try:
                     flagged_fraction_data[flagged_fraction].append(antenna)
                 except KeyError:
@@ -82,10 +93,15 @@ requirements:
             refant = flagged_fraction_data[min_flagged_fraction][0]
             print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
 
+            flagged_fraction_antenna['state'] = '$(inputs.state)'
+            
             cwl_output = {'refant': str(refant)}
 
             with open('./out.json', 'w') as fp:
                 json.dump(cwl_output, fp)
+                
+            with open('./flagged_fraction_antenna.json', 'w') as fp:
+                json.dump(flagged_fraction_antenna, fp)
             
 stdout: findRefAnt.log
 stderr: findRefAnt_err.log

steps/summary.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
+  sbg: 'https://www.sevenbridges.com/'
+id: summary
+baseCommand:
+  - make_summary_CWL.py
+inputs:
+  - id: flagFiles
+    type:
+      - 'File[]'
+      - File
+    inputBinding:
+      position: 1
+    doc: List of files with flag information (JSON)
+  - id: pipeline
+    type: string?
+    default: 'prefactor'
+    inputBinding:
+      position: 0
+      prefix: '--pipeline'
+    doc: Name of the pipeline
+  - id: run_type
+    default: 'calibrator'
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--run_type'
+    doc: Type of the pipeline
+  - id: filter
+    default: '*&'
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--filtered_antennas'
+    doc: Filter these antenna string from the processing.
+  - id: bad_antennas
+    default: '*&'
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '--bad_antennas'
+    doc: Antenna string to be processed
+  - id: structure_file
+    default: false
+    type: 
+      - boolean?
+      - File?
+    inputBinding:
+      position: 0
+      prefix: '--structure_file'
+  - id: Ateam_separation_file
+    default: false
+    type: 
+      - boolean?
+      - File?
+    inputBinding:
+      position: 0
+      prefix: '--Ateam_separation_file'
+  - id: solutions
+    default: false
+    type: 
+      - boolean?
+      - File?
+    inputBinding:
+      position: 0
+      prefix: '--solutions'
+  - id: clip_sources
+    default: false
+    type: 
+      - string?
+      - boolean?
+    inputBinding:
+      position: 0
+      prefix: '--clip_sources'
+      separate: true
+      itemSeparator: ','
+      valueFrom: '$(self)'
+  - id: demix_sources
+    default: false
+    type:
+      - string?
+      - boolean?
+    inputBinding:
+      position: 0
+      prefix: '--demix_sources'
+      separate: true
+      itemSeparator: ','
+      valueFrom: '$(self)'
+  - id: demix
+    default: false
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: '--demix True'
+  - id: output_fname
+    type: 
+      - boolean?
+      - string?
+    default: false
+    inputBinding:
+      position: 0
+      prefix: '--output_fname'
+    doc: Name of the output file
+outputs:
+  - id: summary_file
+    doc: Summary File in JSON format
+    type: File
+    outputBinding:
+      glob: '*.json'
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'summary*.log'
+label: summary
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor3-cwl
+stdout: summary.log
+stderr: summary_err.log
+$schema:
+  - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'

steps/uvplot.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
+  sbg: 'https://www.sevenbridges.com/'
+id: uvplot
+baseCommand:
+  - plot_uvcov.py
+inputs:
+  - id: MSfiles
+    type:
+      - 'Directory[]'
+      - Directory
+    inputBinding:
+      position: 2
+      shellQuote: false
+    doc: List of MS for UV-plotting
+  - id: output_name
+    type: string?
+    default: 'uv-coverage.png'
+    inputBinding:
+      position: 1
+      prefix: '--output'
+      shellQuote: false
+    doc: Name of the output image
+  - id: title
+    default: 'uv-coverage'
+    type: string?
+    inputBinding:
+      position: 1
+      prefix: '--title'
+      shellQuote: false
+    doc: Title of the plots
+  - id: limits
+    default: ',,,'
+    type: string?
+    inputBinding:
+      position: 1
+      prefix: '--limits'
+      shellQuote: false
+  - id: timeslots
+    default: '0,10,0'
+    type: string?
+    inputBinding:
+      position: 1
+      prefix: '--timeslots'
+      shellQuote: false
+  - id: antennas
+    default: '-1'
+    type: string?
+    inputBinding:
+      position: 1
+      prefix: '--antennas'
+      shellQuote: false
+  - id: wideband
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: '--wideband True'
+      shellQuote: false
+outputs:
+  - id: output_image
+    doc: Plot of the uv-coverage
+    type: File[]
+    outputBinding:
+      glob: "*$(inputs.output_name.split('.').pop())"
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'uvplot*.log'
+label: uvplot
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor3-cwl
+requirements:
+  - class: ShellCommandRequirement
+  - class: InlineJavascriptRequirement
+stdout: uvplot.log
+stderr: uvplot_err.log
+$schema:
+  - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'

subworkflow/apply_calibrate_bp.cwl

@@ -32,6 +32,12 @@ outputs:
     type: File
     'sbg:x': 1000
     'sbg:y': -300
+  - id: final_flags_out
+    outputSource:
+      - final_flags/flagged_fraction_dict
+    type: string?
+    'sbg:x': 1000
+    'sbg:y': -200
   - id: BLsmooth.log
     outputSource:
       - BLsmooth/logfile
@@ -79,12 +85,12 @@ steps:
         source: input_h5parm
       - id: msout_datacolumn
         default: CORRECTED_DATA
+      - id: correction
+        default: polalign
       - id: storagemanager
         default: Dysco
       - id: databitrate
         default: 0
-      - id: correction
-        default: polalign
     out:
       - id: msout
       - id: logfile
@@ -102,12 +108,12 @@ steps:
         source: input_h5parm
       - id: msout_datacolumn
         default: CORRECTED_DATA
+      - id: correction
+        default: bandpass
       - id: storagemanager
         default: Dysco
       - id: databitrate
         default: 0
-      - id: correction
-        default: bandpass
       - id: updateweights
         default: 'True'
     out:
@@ -125,12 +131,12 @@ steps:
         default: CORRECTED_DATA
       - id: msout_datacolumn
         default: CORRECTED_DATA
+      - id: type
+        default: applybeam
       - id: storagemanager
         default: Dysco
       - id: databitrate
         default: 0
-      - id: type
-        default: applybeam
       - id: updateweights
         default: 'true'
       - id: usechannelfreq
@@ -156,12 +162,12 @@ steps:
         source: input_h5parm
       - id: msout_datacolumn
         default: CORRECTED_DATA
+      - id: correction
+        default: faraday
       - id: storagemanager
         default: Dysco
       - id: databitrate
         default: 0
-      - id: correction
-        default: faraday
     out:
       - id: msout
       - id: logfile
@@ -199,6 +205,7 @@ steps:
       - id: mode
         default: scalarphase
     out:
+      - id: msout
       - id: h5parm
       - id: logfile
     run: ../lofar-cwl/steps/ddecal.cwl
@@ -269,4 +276,15 @@ steps:
     label: concat_logfiles_applyPA
     'sbg:x': -200
     'sbg:y': 200
+  - id: final_flags
+    in:
+      - id: msin
+        source: applyFR/msout
+    out:
+      - id: flagged_fraction_dict
+      - id: logfile
+    run: ./../steps/findRefAnt.cwl
+    label: final_flags
+    'sbg:x': 700
+    'sbg:y': -200
 requirements: []

subworkflow/ndppp_prep_targ.cwl

@@ -101,7 +101,22 @@ inputs:
     type: boolean
     'sbg:x': 2234.90185546875
     'sbg:y': -393.8929748535156
+  - id: clip_sources
+    type: 'string[]?'
+    default:
+      - VirA_4_patch
+      - CygAGG
+      - CasA_4_patch
+      - TauAGG
+    'sbg:x': 4000
+    'sbg:y': -700
 outputs:
+  - id: prep_flags_out
+    outputSource:
+      - prep_flags/flagged_fraction_dict
+    type: string?
+    'sbg:x': 5500
+    'sbg:y': 200
   - id: prep_logfile
     outputSource:
       - concat_logfiles_dppp/output
@@ -122,7 +137,7 @@ outputs:
     'sbg:y': -100
   - id: msout
     outputSource:
-      - predict/msout
+      - Ateamclipper/msout
     type: Directory
     'sbg:x': 5502.318359375
     'sbg:y': -279.49835205078125
@@ -437,11 +452,7 @@ steps:
       - id: sources_db
         source: skymodel
       - id: sources
-        default:
-          - VirA_4_patch
-          - CygAGG
-          - CasA_4_patch
-          - TauAGG
+        source: clip_sources
       - id: usebeammodel
         default: true
       - id: onebeamperpatch
@@ -456,6 +467,17 @@ steps:
     run: ../lofar-cwl/steps/filter_predict.cwl
     'sbg:x': 4500
     'sbg:y': -344.5472106933594
+  - id: prep_flags
+    in:
+      - id: msin
+        source: dp3_execute/msout
+    out:
+      - id: flagged_fraction_dict
+      - id: logfile
+    run: ./../steps/findRefAnt.cwl
+    label: prep_flags
+    'sbg:x': 4500
+    'sbg:y': -600
   - id: Ateamclipper
     in:
       - id: msin
@@ -470,6 +492,6 @@ steps:
     'sbg:x': 4700
     'sbg:y': -344.5472106933594
 requirements:
-  - class: SubworkflowFeatureRequirement
   - class: InlineJavascriptRequirement
   - class: StepInputExpressionRequirement
+  - class: SubworkflowFeatureRequirement

workflows/HBA_calibrator.cwl

@@ -296,6 +296,7 @@ steps:
       - id: logfiles
       - id: solutions
       - id: inspection
+      - id: summary_file
     run: ./prefactor_calibrator.cwl
     label: prefactor_calibrator
     'sbg:x': 0
@@ -313,7 +314,7 @@ steps:
     run: ./../steps/collectlog.cwl
     label: save_logfiles
     'sbg:x': 1000
-    'sbg:y': -200
+    'sbg:y': -300
   - id: save_inspection
     in:
       - id: log_files
@@ -327,7 +328,7 @@ steps:
     run: ./../steps/collectlog.cwl
     label: save_inspection
     'sbg:x': 1000
-    'sbg:y': 0
+    'sbg:y': -100
   - id: save_solutions
     in:
       - id: log_files
@@ -340,7 +341,20 @@ steps:
     run: ./../steps/collectlog.cwl
     label: save_solutions
     'sbg:x': 1000
-    'sbg:y': 200
+    'sbg:y': 100
+  - id: save_summary
+    in:
+      - id: log_files
+        source:
+          - prefactor/summary_file
+      - id: sub_directory_name
+        default: summary
+    out:
+      - id: log_dir
+    run: ./../steps/collectlog.cwl
+    label: save_summary
+    'sbg:x': 1000
+    'sbg:y': 300
   - id: save_results
     in:
       - id: log_files
@@ -349,6 +363,7 @@ steps:
           - save_solutions/log_dir
           - save_inspection/log_dir
           - save_logfiles/log_dir
+          - save_summary/log_dir
       - id: sub_directory_name
         default: results
     out:

workflows/HBA_target.cwl

@@ -238,11 +238,20 @@ inputs:
     default: null
     'sbg:x': -1000
     'sbg:y': 2900
+  - id: clip_sources
+    type: 'string[]?'
+    default:
+      - VirA_4_patch
+      - CygAGG
+      - CasA_4_patch
+      - TauAGG
+    'sbg:x': -1000
+    'sbg:y': 3000
   - id: ncores
     type: int?
     default: 1
     'sbg:x': -1000
-    'sbg:y': 3000
+    'sbg:y': 3100
 outputs:
   - id: results
     outputSource:
@@ -348,6 +357,8 @@ steps:
         source: proxy_user
       - id: proxy_pass
         source: proxy_pass
+      - id: clip_sources
+        source: clip_sources
       - id: ncores
         source: ncores
     out:
@@ -355,6 +366,7 @@ steps:
       - id: msout
       - id: solutions
       - id: inspection
+      - id: summary_file
     run: ./prefactor_target.cwl
     label: prefactor_target
     'sbg:x': 0
@@ -372,7 +384,7 @@ steps:
     run: ./../steps/collectlog.cwl
     label: save_logfiles
     'sbg:x': 1000
-    'sbg:y': -200
+    'sbg:y': -300
   - id: save_inspection
     in:
       - id: log_files
@@ -386,7 +398,7 @@ steps:
     run: ./../steps/collectlog.cwl
     label: save_inspection
     'sbg:x': 1000
-    'sbg:y': 0
+    'sbg:y': -100
   - id: save_solutions
     in:
       - id: log_files
@@ -399,7 +411,20 @@ steps:
     run: ./../steps/collectlog.cwl
     label: save_solutions
     'sbg:x': 1000
-    'sbg:y': 200
+    'sbg:y': 100
+  - id: save_summary
+    in:
+      - id: log_files
+        source:
+          - prefactor/summary_file
+      - id: sub_directory_name
+        default: summary
+    out:
+      - id: log_dir
+    run: ./../steps/collectlog.cwl
+    label: save_summary
+    'sbg:x': 1000
+    'sbg:y': 300
   - id: save_results
     in:
       - id: log_files
@@ -408,6 +433,7 @@ steps:
           - save_solutions/log_dir
           - save_inspection/log_dir
           - save_logfiles/log_dir
+          - save_summary/log_dir
           - prefactor/msout
       - id: sub_directory_name
         default: results

workflows/prefactor_calibrator.cwl

@@ -230,7 +230,40 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 2000
     'sbg:y': 600
+  - id: summary_file
+    outputSource:
+      - ion/summary_file
+    type: File
+    'sbg:x': 2000
+    'sbg:y': 0
 steps:
+  - id: initial_flags
+    in:
+      - id: msin
+        source: msin
+    out:
+      - id: flagged_fraction_dict
+    run: ./../steps/findRefAnt.cwl
+    label: initial_flags
+    scatter:
+      - msin
+    'sbg:x': -500
+    'sbg:y': -500
+  - id: initial_flags_join
+    in:
+      - id: flagged_fraction_dict
+        source:
+          - initial_flags/flagged_fraction_dict
+      - id: filter_station
+        default: ''
+      - id: state
+        default: 'initial'
+    out:
+      - id: flagged_fraction_antenna
+    run: ./../steps/findRefAnt_join.cwl
+    label: initial_flags_join
+    'sbg:x': -300
+    'sbg:y': -500
   - id: prep
     in:
       - id: msin
@@ -287,6 +320,7 @@ steps:
       - id: logfiles
       - id: outh5parm_logfile
       - id: check_Ateam_separation.png
+      - id: check_Ateam_separation.json
       - id: msout
       - id: parset
       - id: calibrator_name
@@ -449,6 +483,7 @@ steps:
       - id: logfiles
       - id: outh5parm
       - id: outh5parm_logfile
+      - id: final_flags_join_out
     run: ./prefactor_calibrator/bp.cwl
     label: BP
     'sbg:x': 1300
@@ -475,7 +510,21 @@ steps:
         source: clock_smooth
       - id: calibrator_name
         source: prep/calibrator_name
+      - id: flags
+        linkMerge: merge_flattened
+        source:
+          - initial_flags_join/flagged_fraction_antenna
+          - bp/final_flags_join_out
+      - id: demix
+        source: demix
+      - id: demix_sources
+        source: demix_sources
+      - id: filter_baselines
+        source: filter_baselines
+      - id: check_Ateam_separation.json
+        source: prep/check_Ateam_separation.json
     out:
+      - id: summary_file
       - id: outsolutions
       - id: inspection
       - id: logfiles

workflows/prefactor_calibrator/bp.cwl

@@ -87,6 +87,12 @@ outputs:
     type: File
     'sbg:x': 1000
     'sbg:y': -200
+  - id: final_flags_join_out
+    outputSource:
+      - final_flags_join/flagged_fraction_antenna
+    type: File?
+    'sbg:x': 1000
+    'sbg:y': -100
   - id: inspection
     outputSource:
       - losoto_plot_A1/output_plots
@@ -423,12 +429,28 @@ steps:
       - id: outh5parm
       - id: apply_pa.log
       - id: applybeam.log
+      - id: final_flags_out
     run: ../../subworkflow/apply_calibrate_bp.cwl
     label: apply_calibrate_bp
     scatter:
       - msin
     'sbg:x': 500
     'sbg:y': 0
+  - id: final_flags_join
+    in:
+      - id: flagged_fraction_dict
+        source:
+          - apply_calibrate_bp/final_flags_out
+      - id: filter_station
+        default: ''
+      - id: state
+        default: 'final'
+    out:
+      - id: flagged_fraction_antenna
+    run: ./../../steps/findRefAnt_join.cwl
+    label: final_flags_join
+    'sbg:x': 750
+    'sbg:y': 150
   - id: concat_logfiles_blsmooth
     in:
       - id: file_list

workflows/prefactor_calibrator/ion.cwl

@@ -5,6 +5,27 @@ label: ion
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
+  - id: filter_baselines
+    type: string?
+    default: '*&'
+    'sbg:x': -1000
+    'sbg:y': -600
+  - id: demix_sources
+    type: 'string[]?'
+    default:
+      - CasA
+      - CygA
+    'sbg:x': -1000
+    'sbg:y': -500
+  - id: demix
+    type: boolean?
+    default: false
+    'sbg:x': -1000
+    'sbg:y': -400
+  - id: flags
+    type: File[]
+    'sbg:x': -1000
+    'sbg:y': -300
   - id: calibrator_name
     type: string
     'sbg:x': -1000
@@ -41,7 +62,17 @@ inputs:
     type: File
     'sbg:x': -1000
     'sbg:y': 600
+  - id: check_Ateam_separation.json
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 700
 outputs:
+  - id: summary_file
+    outputSource:
+      - summary/summary_file
+    type: File
+    'sbg:x': 1000
+    'sbg:y': -100
   - id: inspection
     outputSource:
       - losoto_plot_P3/output_plots
@@ -63,6 +94,7 @@ outputs:
   - id: logfiles
     outputSource:
       - concat_logfiles_ion/output
+      - concat_logfiles_summary/output
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 1000
@@ -255,6 +287,35 @@ steps:
     label: losoto_plot_Pr
     'sbg:x': 0
     'sbg:y': 300
+  - id: summary
+    in:
+      - id: flagFiles
+        source: flags
+        linkMerge: merge_flattened
+      - id: pipeline
+        default: 'prefactor'
+      - id: run_type
+        default: 'calibrator'
+      - id: filter
+        source: filter_baselines
+      - id: bad_antennas
+        default: ''
+      - id: Ateam_separation_file
+        source: check_Ateam_separation.json
+      - id: solutions
+        source: h5parm_pointingname/outh5parm
+      - id: demix
+        source: demix
+      - id: demix_sources
+        source: demix_sources
+        valueFrom: "$(self.join(','))"
+    out:
+      - id: summary_file
+      - id: logfile
+    run: ../../steps/summary.cwl
+    label: summary
+    'sbg:x': 500
+    'sbg:y': 0
   - id: concat_logfiles_ion
     in:
       - id: file_list
@@ -278,6 +339,21 @@ steps:
     label: concat_logfiles_ion
     'sbg:x': 500
     'sbg:y': 500
+  - id: concat_logfiles_summary
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - summary/logfile
+      - id: file_prefix
+        source: calibrator_name
+        valueFrom: $(self+'_summary')
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_summary
+    'sbg:x': 700
+    'sbg:y': 500
   - id: write_solutions
     in:
       - id: h5parmFile
@@ -315,5 +391,7 @@ steps:
     'sbg:x': 0
     'sbg:y': 700
 requirements:
+  - class: InlineJavascriptRequirement
+  - class: StepInputExpressionRequirement
   - class: SubworkflowFeatureRequirement
   - class: MultipleInputFeatureRequirement

workflows/prefactor_calibrator/prep.cwl

@@ -146,6 +146,12 @@ outputs:
     type: 'File[]'
     'sbg:x': 1554.5106201171875
     'sbg:y': 777
+  - id: check_Ateam_separation.json
+    outputSource:
+      - check_ateam_separation/output_json
+    type: File
+    'sbg:x': 1554.5106201171875
+    'sbg:y': 500
   - id: logfiles
     outputSource:
       - concat_logfiles_calib/output
@@ -310,6 +316,7 @@ steps:
         source: min_separation
     out:
       - id: output_imag
+      - id: output_json
       - id: logfile
     run: ../../steps/check_ateam_separation.cwl
     label: check_Ateam_separation

workflows/prefactor_target.cwl

@@ -238,11 +238,20 @@ inputs:
     default: null
     'sbg:x': -1000
     'sbg:y': 2900
+  - id: clip_sources
+    type: 'string[]?'
+    default:
+      - VirA_4_patch
+      - CygAGG
+      - CasA_4_patch
+      - TauAGG
+    'sbg:x': -1000
+    'sbg:y': 3000
   - id: ncores
     type: int?
     default: 1
     'sbg:x': -1000
-    'sbg:y': 3000
+    'sbg:y': 3100
 outputs:
   - id: inspection
     outputSource:
@@ -275,7 +284,40 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 2000
     'sbg:y': 600
+  - id: summary_file
+    outputSource:
+      - finalize/summary_file
+    type: File
+    'sbg:x': 2000
+    'sbg:y': 300
 steps:
+  - id: initial_flags
+    in:
+      - id: msin
+        source: msin
+    out:
+      - id: flagged_fraction_dict
+    run: ./../steps/findRefAnt.cwl
+    label: initial_flags
+    scatter:
+      - msin
+    'sbg:x': -500
+    'sbg:y': -500
+  - id: initial_flags_join
+    in:
+      - id: flagged_fraction_dict
+        source:
+          - initial_flags/flagged_fraction_dict
+      - id: filter_station
+        default: ''
+      - id: state
+        default: 'initial'
+    out:
+      - id: flagged_fraction_antenna
+    run: ./../steps/findRefAnt_join.cwl
+    label: initial_flags_join
+    'sbg:x': -300
+    'sbg:y': -500
   - id: prep
     in:
       - id: msin
@@ -292,8 +334,6 @@ steps:
         source: filter_baselines
       - id: raw_data
         source: raw_data
-      - id: propagatesolutions
-        source: propagatesolutions
       - id: demix_sources
         source:
           - demix_sources
@@ -345,10 +385,15 @@ steps:
         source: proxy_user
       - id: proxy_pass
         source: proxy_pass
+      - id: clip_sources
+        source: clip_sources
     out:
+      - id: compare_stations_filter
       - id: outh5parm
       - id: inspection
       - id: msout
+      - id: check_Ateam_separation.json
+      - id: prep_flags_join_out
       - id: logfiles
     run: ./prefactor_target/prep.cwl
     label: prep
@@ -381,6 +426,21 @@ steps:
     label: findRefAnt_join
     'sbg:x': 600
     'sbg:y': 0
+  - id: Ateam_flags_join
+    in:
+      - id: flagged_fraction_dict
+        source:
+          - findRefAnt/flagged_fraction_dict
+      - id: filter_station
+        default: ''
+      - id: state
+        default: 'Ateam'
+    out:
+      - id: flagged_fraction_antenna
+    run: ./../steps/findRefAnt_join.cwl
+    label: Ateam_flags_join
+    'sbg:x': 600
+    'sbg:y': 300
   - id: concat_logfiles_RefAnt
     in:
       - id: file_list
@@ -425,18 +485,18 @@ steps:
         source: min_unflagged_fraction
       - id: refant
         source: findRefAnt_join/refant
-      - id: overhead
-        source: overhead
       - id: min_length
         source: min_length
+      - id: overhead
+        source: overhead
     out:
       - id: msout
       - id: outh5parm
       - id: bad_antennas
-      - id: total_bandwidth
       - id: outh5parm_logfile
       - id: inspection
       - id: logfiles
+      - id: total_bandwidth
     run: ./prefactor_target/gsmcal.cwl
     label: gsmcal
     'sbg:x': 900
@@ -464,6 +524,24 @@ steps:
         source: skymodel_source
       - id: total_bandwidth
         source: gsmcal/total_bandwidth
+      - id: check_Ateam_separation.json
+        source: prep/check_Ateam_separation.json
+      - id: filter_baselines
+        source: filter_baselines
+      - id: compare_stations_filter
+        source: prep/compare_stations_filter
+      - id: flags
+        linkMerge: merge_flattened
+        source:
+          - initial_flags_join/flagged_fraction_antenna
+          - prep/prep_flags_join_out
+          - Ateam_flags_join/flagged_fraction_antenna
+      - id: clip_sources
+        source: clip_sources
+      - id: demix
+        source: demix
+      - id: demix_sources
+        source: demix_sources
       - id: ncores
         source: ncores
     out:
@@ -471,6 +549,7 @@ steps:
       - id: solutions
       - id: logfiles
       - id: inspection
+      - id: summary_file
     run: ./prefactor_target/finalize.cwl
     label: finalize
     'sbg:x': 1200

workflows/prefactor_target/finalize.cwl

@@ -5,6 +5,18 @@ label: finalize
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
+  - id: demix_sources
+    type: 'string[]'
+    'sbg:x': -1000
+    'sbg:y':  -700
+  - id: demix
+    type: boolean
+    'sbg:x': -1000
+    'sbg:y': -600
+  - id: clip_sources
+    type: 'string[]'
+    'sbg:x': -1000
+    'sbg:y': -500
   - id: msin
     type: 'Directory[]'
     'sbg:x': -1000
@@ -45,11 +57,27 @@ inputs:
     type: int
     'sbg:x': -1000
     'sbg:y': 500
+  - id: check_Ateam_separation.json
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 600
+  - id: filter_baselines
+    type: string
+    'sbg:x': -1000
+    'sbg:y': 700
+  - id: compare_stations_filter
+    type: string
+    'sbg:x': -1000
+    'sbg:y': 800
+  - id: flags
+    type: File[]
+    'sbg:x': -1000
+    'sbg:y': 900
   - id: ncores
     type: int?
     default: 1
     'sbg:x': -1000
-    'sbg:y': 600
+    'sbg:y': 1000
 outputs:
   - id: msout
     outputSource:
@@ -69,6 +97,8 @@ outputs:
       - concat_logfiles_solutions/output
       - concat_logfiles_structure/output
       - concat_logfiles_wsclean/output
+      - concat_logfiles_summary/output
+      - concat_logfiles_uvplot/output
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 1300
@@ -77,10 +107,17 @@ outputs:
     outputSource:
       - structure_function/structure_plot
       - wsclean/image
+      - uvplot/output_image
     type: 'File[]?'
     linkMerge: merge_flattened
     'sbg:x': 1300
     'sbg:y': 500
+  - id: summary_file
+    outputSource:
+      - summary/summary_file
+    type: File
+    'sbg:x': 1300
+    'sbg:y': 1000
 steps:
   - id: add_missing_stations
     in:
@@ -148,6 +185,33 @@ steps:
     scatterMethod: dotproduct
     'sbg:x': 400
     'sbg:y': 0
+  - id: final_flags
+    in:
+      - id: msin
+        source: apply_gsmcal/msout
+    out:
+      - id: flagged_fraction_dict
+    run: ./../../steps/findRefAnt.cwl
+    label: final_flags
+    scatter:
+      - msin
+    'sbg:x': 400
+    'sbg:y': -400
+  - id: final_flags_join
+    in:
+      - id: flagged_fraction_dict
+        source:
+          - final_flags/flagged_fraction_dict
+      - id: filter_station
+        default: ''
+      - id: state
+        default: 'final'
+    out:
+      - id: flagged_fraction_antenna
+    run: ./../../steps/findRefAnt_join.cwl
+    label: final_flags_join
+    'sbg:x': 600
+    'sbg:y': -400
   - id: average
     in:
       - id: msin
@@ -180,6 +244,64 @@ steps:
     scatterMethod: dotproduct
     'sbg:x': 600
     'sbg:y': -200
+  - id: summary
+    in:
+      - id: flagFiles
+        source: 
+          - flags
+          - final_flags_join/flagged_fraction_antenna
+        linkMerge: merge_flattened
+      - id: pipeline
+        default: 'prefactor'
+      - id: run_type
+        default: 'target'
+      - id: filter
+        source: filter_baselines
+      - id: bad_antennas
+        source:
+          - bad_antennas
+          - compare_stations_filter
+        valueFrom: $(self.join(''))
+      - id: structure_file
+        source: structure_function/structure_txt
+      - id: Ateam_separation_file
+        source: check_Ateam_separation.json
+      - id: solutions
+        source: h5parm_pointingname/outh5parm
+      - id: clip_sources
+        source: clip_sources
+        valueFrom: "$(self.join(','))"
+      - id: demix
+        source: demix
+      - id: demix_sources
+        source: demix_sources
+        valueFrom: "$(self.join(','))"
+    out:
+      - id: summary_file
+      - id: logfile
+    run: ../../steps/summary.cwl
+    label: summary
+    'sbg:x': 800
+    'sbg:y': 0
+  - id: uvplot
+    in:
+      - id: MSfiles
+        source: apply_gsmcal/msout
+      - id: output_name
+        source: get_targetname/targetname
+        valueFrom: $(self)_uv-coverage.png
+      - id: title
+        source: get_targetname/targetname
+        valueFrom: '"uv coverage of the target pointing: $(self)"'
+      - id: wideband
+        default: true
+    out:
+      - id: output_image
+      - id: logfile
+    run: ../../steps/uvplot.cwl
+    label: uvplot
+    'sbg:x': 800
+    'sbg:y': 300
   - id: wsclean
     in:
       - id: msin
@@ -359,6 +481,35 @@ steps:
     label: concat_logfiles_wsclean
     'sbg:x': 1100
     'sbg:y': 500
+  - id: concat_logfiles_summary
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - summary/logfile
+      - id: file_prefix
+        source: get_targetname/targetname
+        valueFrom: $(self+'_summary')
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_summary
+    'sbg:x': 1050
+    'sbg:y': 0
+  - id: concat_logfiles_uvplot
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - uvplot/logfile
+      - id: file_prefix
+        default: uvplot
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_uvplot
+    'sbg:x': 1050
+    'sbg:y': 300
   - id: get_targetname
     in:
       - id: msin

workflows/prefactor_target/prep.cwl

@@ -38,9 +38,13 @@ inputs:
     default: false
     'sbg:x': -1000
     'sbg:y': -1100
-  - id: propagatesolutions
-    type: boolean?
-    default: true
+  - id: clip_sources
+    type: 'string[]?'
+    default:
+      - VirA_4_patch
+      - CygAGG
+      - CasA_4_patch
+      - TauAGG
     'sbg:x': -1000
     'sbg:y': -1000
   - id: demix_sources
@@ -180,6 +184,12 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1700
 outputs:
+  - id: compare_stations_filter
+    outputSource:
+      - compare_station_list/filter_out
+    type: string
+    'sbg:x': 1000
+    'sbg:y': -800    
   - id: outh5parm
     outputSource:
       - createRMh5parm/h5parmout
@@ -201,6 +211,12 @@ outputs:
     type: 'Directory[]'
     'sbg:x': 1000
     'sbg:y': 0
+  - id: check_Ateam_separation.json
+    outputSource:
+      - check_ateam_separation/output_json
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 500
   - id: logfiles
     outputSource:
       - make_sourcedb_ateam/log
@@ -214,7 +230,28 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 1000
     'sbg:y': 800
+  - id: prep_flags_join_out
+    outputSource:
+      - prep_flags_join/flagged_fraction_antenna
+    type: File?
+    'sbg:x': 1000
+    'sbg:y': 1100
 steps:
+  - id: prep_flags_join
+    in:
+      - id: flagged_fraction_dict
+        source:
+          - dppp_prep_target/prep_flags_out
+      - id: filter_station
+        default: ''
+      - id: state
+        default: 'prep'
+    out:
+      - id: flagged_fraction_antenna
+    run: ./../../steps/findRefAnt_join.cwl
+    label: prep_flags_join
+    'sbg:x': 400
+    'sbg:y': -200
   - id: check_ateam_separation
     in:
       - id: ms
@@ -224,6 +261,7 @@ steps:
         source: min_separation
     out:
       - id: output_imag
+      - id: output_json
       - id: logfile
     run: ../../steps/check_ateam_separation.cwl
     label: check_Ateam_separation
@@ -381,12 +419,15 @@ steps:
         source: compare_station_list/filter_out
       - id: updateweights
         source: updateweights
+      - id: clip_sources
+        source: clip_sources
     out:
       - id: msout
       - id: prep_logfile
       - id: predict_logfile
       - id: clipper_logfile
       - id: clipper_output
+      - id: prep_flags_out
       - id: parset
     run: ../../subworkflow/ndppp_prep_targ.cwl
     label: ndppp_prep_target