update logging output

Former-commit-id: 3ffe3789 [formerly 38fe83c8]
Former-commit-id: f41d337d
Former-commit-id: bf85d7cf

lofar-cwl/steps/applytarget.cwl

@@ -17,7 +17,7 @@ inputs:
     type: string
     inputBinding:
       position: 0
-      prefix: msout=msout_name
+      prefix: msout=
       separate: false
     doc: Name of output MS
   - default: DATA
@@ -67,6 +67,7 @@ inputs:
     default: ""
     inputBinding:
       prefix: msout.storagemanager=
+      separate: false
   - id: databitrate
     type: int?
     inputBinding:
@@ -85,7 +86,7 @@ outputs:
     outputBinding:
       glob: $(inputs.msin.basename)
   - id: logfile
-    type: File
+    type: File[]
     outputBinding:
       glob: 'applycal_$(inputs.correction).log'
 stdout: applycal_$(inputs.correction).log

steps/check_unflagged_fraction.cwl

@@ -27,7 +27,6 @@ arguments:
 
     from check_unflagged_fraction import main as check_unflagged_fraction
 
-    print(sys.argv)
     ms = sys.argv[1]
     min_fraction = float($(inputs.min_fraction))
     
@@ -36,10 +35,6 @@ arguments:
     filename           = os.path.basename(output['flagged'])
     unflagged_fraction = output['unflagged_fraction']
 
-    print(os.getcwd())
-    print(filename)
-    print(filename.lstrip('out_'))
-    
     if filename != 'None':
         shutil.move(filename, filename.lstrip('out_'))
         
@@ -61,9 +56,9 @@ outputs:
         glob: 'out.json'
         outputEval: $(JSON.parse(self[0].contents).unflagged_fraction)
   - id: logfile
-    type: File?
+    type: File[]?
     outputBinding:
-      glob: check_unflagged_fraction.log    
+      glob: 'check_unflagged_fraction*.log'
         
 requirements:
   - class: InlineJavascriptRequirement
@@ -76,3 +71,4 @@ hints:
   - class: DockerRequirement
     dockerPull: lofareosc/prefactor:HBAcalibrator
 stdout: check_unflagged_fraction.log
+stderr: check_unflagged_fraction_err.log
\ No newline at end of file

steps/compare_station_list.cwl

@@ -51,9 +51,9 @@ outputs:
         glob: 'out.json'
         outputEval: $(JSON.parse(self[0].contents).filter_out)
   - id: logfile
-    type: File?
+    type: File[]?
     outputBinding:
-      glob: compareStationList.log    
+      glob: 'compareStationList*.log'
         
 requirements:
   - class: InlineJavascriptRequirement
@@ -62,3 +62,4 @@ hints:
   DockerRequirement:
     dockerPull: lofareosc/prefactor:HBAcalibrator
 stdout: compareStationList.log
+stderr: compareStationList_err.log
\ No newline at end of file

steps/merge_array_files.cwl

+id: merge_array_files
+label: merge_array_files
+class: ExpressionTool
+
+cwlVersion: v1.0
+inputs: 
+    - id: input
+      type:
+        - type: array
+          items:
+            - type: array
+              items: File
+outputs: 
+    - id: output
+      type: File[]
+
+expression: |
+  ${
+    var out_file = []
+    for(var i=0; i<inputs.input.length; i++){
+        var item = inputs.input[i]
+        if(item != null){
+            out_file = out_file.concat(item)
+        }
+    }
+    return {'output': out_file}
+  }
+
+
+requirements:
+  - class: InlineJavascriptRequirement
\ No newline at end of file

steps/plot_unflagged.cwl

@@ -24,19 +24,11 @@ arguments:
 
     from plot_unflagged_fraction import main as plot_unflagged_fraction
 
-    print(sys.argv)
-    print(len(sys.argv))
-    
     center = int((len(sys.argv) - 1) / 2 + 1)
     
-    print(center)
     mss = sys.argv[1:center]
     unflagged_fraction = sys.argv[center:]
     
-    print(mss)
-    print(unflagged_fraction)
-    print(os.getcwd() + '/unflagged_fraction.png')
-
     output = plot_unflagged_fraction(ms_list = mss, frac_list = unflagged_fraction, outfile = os.getcwd() + '/unflagged_fraction.png')
 
 outputs:
@@ -45,6 +37,11 @@ outputs:
     type: File
     outputBinding:
       glob: 'unflagged_fraction.png'
+  - id: logfile
+    type: File[]?
+    outputBinding:
+      glob: 'plot_unflagged_fraction*.log'
+
     
 hints:
   - class: DockerRequirement

subworkflow/concat.cwl

@@ -6,7 +6,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
   - id: msin
-    type: Directory[]
+    type: 'Directory[]'
     'sbg:x': -1000
     'sbg:y': -300
   - id: group_id
@@ -49,7 +49,7 @@ outputs:
   - id: msout
     outputSource:
       - check_unflagged_fraction/msout
-    type: Directory[]
+    type: 'Directory[]'
     'sbg:x': 1000
     'sbg:y': 0
   - id: dpppconcat.log
@@ -60,11 +60,10 @@ outputs:
     'sbg:y': 100
   - id: check_unflagged_fraction.log
     outputSource:
-      - check_unflagged_fraction/logfile
+      - concat_logfiles_unflagged/output
     type: File
     'sbg:x': 1000
     'sbg:y': 200
-
 steps:
   - id: filter_ms_group
     in:
@@ -73,7 +72,8 @@ steps:
       - id: groups_specification
         source: groups_specification
       - id: measurement_sets
-        source: msin
+        source:
+          - msin
     out:
       - id: output
     run: ../steps/filter_ms_group.cwl
@@ -83,27 +83,28 @@ steps:
   - id: dpppconcat
     in:
       - id: msin
-        source: filter_ms_group/output
+        source:
+          - filter_ms_group/output
       - id: msout_name
         source: group_id
       - id: msin_datacolumn
         default: DATA
       - id: msout_datacolumn
         default: DATA
-      - id: baseline
-        source: filter_baselines
       - id: filter_baselines
         source: filter_baselines
       - id: filter_remove
         default: true
       - id: overwrite
         default: false
-      - id: missingdata
-        default: True
       - id: storagemanager
-        default: "Dysco"
+        default: Dysco
       - id: databitrate
         default: 0
+      - id: missingdata
+        default: true
+      - id: baseline
+        source: filter_baselines
       - id: avg_timeresolution
         source: avg_timeresolution_concat
       - id: avg_freqresolution
@@ -142,4 +143,17 @@ steps:
     label: concat_logfiles_dpppconcat
     'sbg:x': 600
     'sbg:y': 200
+  - id: concat_logfiles_unflagged
+    in:
+      - id: file_list
+        source:
+          - check_unflagged_fraction/logfile
+      - id: file_prefix
+        default: check_unflagged_fraction
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_unflagged
+    'sbg:x': 600
+    'sbg:y': 400
 requirements: []

workflows/prefactor_target/finalize.cwl

@@ -112,7 +112,7 @@ steps:
       - id: msout_datacolumn
         default: DATA
       - id: storagemanager
-        default: Dysco
+        default: "Dysco"
       - id: databitrate
         source: compression_bitrate
       - id: correction
@@ -124,7 +124,7 @@ steps:
         default: 'target'
       - id: msout_name
         source: msin
-        valueFrom: $(self.basename+'_prep.ms')
+        valueFrom: $(self.items.basename+'_pre-cal.ms')
     out:
       - id: msout
       - id: logfile
@@ -134,6 +134,18 @@ steps:
       - msin
     'sbg:x': 400
     'sbg:y': 0
+  - id: merge_array_files
+    in:
+      - id: input
+        source:
+          - apply_gsmcal/logfile
+    out:
+      - id: output
+    run: ../../steps/merge_array_files.cwl
+    label: merge_array_files
+    'sbg:x': 600
+    'sbg:y': 0
+
   - id: write_solutions
     in:
       - id: h5parmFile
@@ -202,7 +214,7 @@ steps:
     in:
       - id: file_list
         source:
-          - apply_gsmcal/logfile
+          - merge_array_files/output
       - id: file_prefix
         default: apply_gsmcal
     out:

workflows/prefactor_target/gsmcal.cwl

@@ -103,7 +103,7 @@ outputs:
       - concat_logfiles_calib/output
       - concat_logfiles_dpppconcat/output
       - concat_logfiles_blsmooth/output
-      - concat/check_unflagged_fraction.log
+      - concat_logfiles_unflagged/output
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 1500
@@ -238,6 +238,21 @@ steps:
     label: concat_logfiles_losoto
     'sbg:x': 1200
     'sbg:y': 700
+  - id: concat_logfiles_unflagged
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - concat/check_unflagged_fraction.log
+          - plot_unflagged/logfile
+      - id: file_prefix
+        default: check_unflagged_fraction
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_unflagged
+    'sbg:x': 1200
+    'sbg:y': 900
   - id: concat
     in:
       - id: msin
@@ -326,6 +341,7 @@ steps:
         source: concat/unflagged_fraction
     out:
       - id: output_imag
+      - id: logfile
     run: ../../steps/plot_unflagged.cwl
     label: plot_unflagged
     'sbg:x': 500

workflows/prefactor_target/prep.cwl

@@ -205,7 +205,7 @@ outputs:
     outputSource:
       - make_sourcedb_ateam/log
       - check_ateam_separation/logfile
-      - compare_station_list/logfile
+      - concat_logfiles_stationlist/output
       - concat_logfiles_RMextract/output
       - concat_logfiles_prep_targ/output
       - concat_logfiles_predict_targ/output
@@ -473,5 +473,19 @@ steps:
     label: concat_logfiles_RMextract
     'sbg:x': 400
     'sbg:y': 600
+  - id: concat_logfiles_stationlist
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - compare_station_list/logfile
+      - id: file_prefix
+        default: compareStationList
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_stationlist
+    'sbg:x': 400
+    'sbg:y': 700
 requirements:
   - class: ScatterFeatureRequirement
\ No newline at end of file