parallelize identifyBadAntennas

Former-commit-id: 2e65109c [formerly 9847f252]
Former-commit-id: d27c4965
Former-commit-id: 80107342

steps/identify_bad_antennas.cwl

@@ -7,14 +7,10 @@ baseCommand:
   - python3
 inputs:
     - id: msin
-      type: Directory[]
+      type: Directory
       doc: MS to compare with
       inputBinding:
         position: 0
-    - id: filter
-      type: string?
-      default: '*&'
-      doc: Filter these baselines for the comparison
 
 label: identifyBadAntennas.py
 arguments:
@@ -22,25 +18,26 @@ arguments:
   - |
     import sys
     import json
-    from identifyBadAntennas import main as identifyBadAntennas
+    from identifyBadAntennas_CWL import main as identifyBadAntennas
     
-    mss = sys.argv[1:]
+    ms = sys.argv[1]
     filter = '$(inputs.filter)'
 
-    output = identifyBadAntennas(mss, filter)
+    output = identifyBadAntennas(ms)
 
-    filter_out = output['filter']
-    cwl_output = {"filter_out": filter_out}
+    flaggedants = output['flaggedants']
+    cwl_output  = {"flaggedants": flaggedants}
 
     with open('./out.json', 'w') as fp:
         json.dump(cwl_output, fp)
+        
 outputs:
-  - id: filter_out
+  - id: flaggedants
     type: string
     outputBinding:
         loadContents: true
         glob: 'out.json'
-        outputEval: $(JSON.parse(self[0].contents).filter_out)
+        outputEval: $(JSON.parse(self[0].contents).flaggedants)
   - id: logfile
     type: File?
     outputBinding:

steps/identify_bad_antennas_join.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: identify_bad_antennas_join
+baseCommand:
+  - python3
+inputs:
+    - id: flaggedants
+      type: string[]?
+      default: []
+      doc: list of flagged antennas
+    - id: filter
+      type: string?
+      default: '*&'
+      doc: Filter these baselines for the comparison
+      
+label: identifyBadAntennas_join
+arguments:
+  - '-c'
+  - |
+
+    flaggedants_list = sys.argv[1:]
+    filter = '$(inputs.filter)'
+    
+    flagged_antenna_list = set.intersection(*map(set, flaggedants_list)) 
+    
+    for flagged_antenna in flagged_antenna_list:
+        filter += ';!' + flagged_antenna + '*&&*'
+
+    print('Identified bad antennas: ' + str(flagged_antenna_list))
+    
+    cwl_output  = {"filter": str(filter)}
+
+    with open('./out.json', 'w') as fp:
+        json.dump(cwl_output, fp)
+        
+outputs:
+  - id: filter_out
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: 'out.json'
+        outputEval: $(JSON.parse(self[0].contents).filter)
+  - id: logfile
+    type: File?
+    outputBinding:
+      glob: identifyBadAntennas.log    
+        
+requirements:
+  - class: InlineJavascriptRequirement
+
+stdout: identifyBadAntennas.log
+stderr: identifyBadAntennas_err.log
\ No newline at end of file

workflows/prefactor_target/gsmcal.cwl

@@ -73,7 +73,7 @@ outputs:
     'sbg:y': 200
   - id: bad_antennas
     outputSource:
-      - identifybadantennas/filter_out
+      - identifybadantennas_join/filter_out
     type: string
     'sbg:x': 1500
     'sbg:y': 400
@@ -96,7 +96,7 @@ outputs:
     'sbg:y': 800
   - id: logfiles
     outputSource:
-      - identifybadantennas/logfile
+      - concat_logfiles_identify/output
       - sort_times_into_freqGroups/logfile
       - find_skymodel_target/logfile
       - make_sourcedb_target/log
@@ -112,16 +112,30 @@ steps:
   - id: identifybadantennas
     in:
       - id: msin
-        source:
+        source: msin
+    out:
+      - id: flaggedants
+      - id: logfile
+    run: ../../steps/identify_bad_antennas.cwl
+    label: identifybadantennas
+    scatter:
       - msin
+    'sbg:x': -200
+    'sbg:y': -300
+  - id: identifybadantennas_join
+    in:
+      - id: flaggedants
+        source: identifybadantennas/flaggedants
       - id: filter
         source: filter_baselines
     out:
       - id: filter_out
       - id: logfile
-    run: ../../steps/identify_bad_antennas.cwl
-    label: identifybadantennas
-    'sbg:x': -200
+    run: ../../steps/identify_bad_antennas_join.cwl
+    label: identifybadantennas_join
+    scatter:
+      - msin
+    'sbg:x': 0
     'sbg:y': -300
   - id: sort_times_into_freqGroups
     in:
@@ -170,10 +184,10 @@ steps:
     in:
       - id: sky_model
         source: find_skymodel_target/skymodel
-      - id: logname
-        default: make_sourcedb_target.log
       - id: output_file_name
         default: target.sourcedb
+      - id: logname
+        default: make_sourcedb_target.log
     out:
       - id: sourcedb
       - id: log
@@ -253,6 +267,21 @@ steps:
     label: concat_logfiles_unflagged
     'sbg:x': 1200
     'sbg:y': 900
+  - id: concat_logfiles_identify
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - identifybadantennas/logfile
+          - identifybadantennas_join/logfile
+      - id: file_prefix
+        default: identifyBadAntennas
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_identify
+    'sbg:x': 1200
+    'sbg:y': -100
   - id: concat
     in:
       - id: msin
@@ -263,7 +292,7 @@ steps:
       - id: groups_specification
         source: sort_times_into_freqGroups/filenames
       - id: filter_baselines
-        source: identifybadantennas/filter_out
+        source: identifybadantennas_join/filter_out
       - id: avg_timeresolution_concat
         source: avg_timeresolution_concat
       - id: avg_freqresolution_concat
@@ -271,9 +300,9 @@ steps:
       - id: min_unflagged_fraction
         source: min_unflagged_fraction
     out:
+      - id: msout_old
       - id: unflagged_fraction
       - id: msout
-      - id: msout_old
       - id: dpppconcat.log
       - id: check_unflagged_fraction.log
     run: ../../subworkflow/concat.cwl
@@ -335,9 +364,11 @@ steps:
   - id: plot_unflagged
     in:
       - id: msin
-        source: concat/msout_old
+        source:
+          - concat/msout_old
       - id: unflagged_fraction
-        source: concat/unflagged_fraction
+        source:
+          - concat/unflagged_fraction
     out:
       - id: output_imag
       - id: logfile
@@ -370,6 +401,7 @@ steps:
     out:
       - id: output_plots
       - id: logfile
+      - id: parset
     run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
     label: losoto_plot_P
     'sbg:x': 1000
@@ -400,6 +432,7 @@ steps:
     out:
       - id: output_plots
       - id: logfile
+      - id: parset
     run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
     label: losoto_plot_P2
     'sbg:x': 1000
@@ -431,6 +464,7 @@ steps:
     out:
       - id: output_plots
       - id: logfile
+      - id: parset
     run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
     label: losoto_plot_Pd
     'sbg:x': 1000
@@ -461,6 +495,7 @@ steps:
     out:
       - id: output_plots
       - id: logfile
+      - id: parset
     run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
     label: losoto_plot_Pd2
     'sbg:x': 1000
@@ -470,3 +505,4 @@ requirements:
   - class: InlineJavascriptRequirement
   - class: SubworkflowFeatureRequirement
   - class: ScatterFeatureRequirement
+  - class: MultipleInputFeatureRequirement