Merge branch 'aoflagging' into 'master'

Aoflagging

See merge request !3

steps/aoflagger.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+id: aoflagging
+label: aoflagging
+
+baseCommand: DP3
+
+inputs:
+    - id: msin
+      type: Directory
+      inputBinding:
+        position: 0
+        prefix: msin=
+        separate: false
+        shellQuote: false
+      doc: Input Measurement Set
+    - id: msin_datacolumn
+      type: string?
+      default: DATA
+      inputBinding:
+        position: 0
+        prefix: msin.datacolumn=
+        separate: false
+        shellQuote: true
+      doc: Input data Column
+    - id: memoryperc
+      type: int?
+      default: 15
+      inputBinding:
+        position: 0
+        prefix: aoflagger.memoryperc=
+        separate: false
+        shellQuote: false
+      doc: Indicates the percentage of pc memory to use
+    - id: keepstatistics
+      type: boolean?
+      default:  true
+      inputBinding:
+        prefix: aoflagger.keepstatistics=True
+      doc: Indicates whether statistics should be written to file.
+    - id: strategy
+      type:
+        - File?
+        - string?
+      default: $PREFACTOR_DATA_ROOT/rfistrategies/lofar-default.lua
+      inputBinding:
+        position: 0
+        prefix: aoflagger.strategy=
+        separate: false
+        shellQuote: false
+      doc: specifies a customized strategy
+
+arguments:
+    - steps=[aoflagger]
+    - aoflagger.type=aoflagger
+    - msout=.
+
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msin.basename)
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: aoflag*.log
+
+requirements:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: ShellCommandRequirement
+
+hints:
+  DockerRequirement:
+    dockerPull: vlbi-cwl
+
+stdout: aoflag.log
+stderr: aoflag_err.log

test-single.yaml

 ---
 msin:
-    class: "Directory"
-    path : "/home/tjlv53/software/test_data/L693719_SB000_uv-compressed.MS"
+    - class: "Directory"
+      path : "/home/tjlv53/software/test_data/L693719_SB000_uv-compressed.MS"
 #    - class: "Directory"
 #      path : "/home/tjlv53/software/test_data/L693719_SB001_uv-compressed.MS"
 #    - class: "Directory"

test.yaml

@@ -28,38 +28,38 @@ msin:
       path : "/home/tjlv53/software/test_data/L693719_SB012_uv-compressed.MS"
     - class: "Directory"
       path : "/home/tjlv53/software/test_data/L693719_SB013_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB014_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB015_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB016_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB017_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB018_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB019_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB020_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB021_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB022_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB023_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB024_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB025_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB026_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB027_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB028_uv-compressed.MS"
-    - class: "Directory"
-      path : "/home/tjlv53/software/test_data/L693719_SB029_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB014_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB015_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB016_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB017_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB018_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB019_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB020_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB021_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB022_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB023_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB024_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB025_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB026_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB027_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB028_uv-compressed.MS"
+#    - class: "Directory"
+#      path : "/home/tjlv53/software/test_data/L693719_SB029_uv-compressed.MS"
 
 
 solset:

workflows/sort-concat.cwl → workflows/concatenate-flag.cwl

 class: Workflow
 cwlVersion: v1.2
-id: sort-concat
-label: sort-concat
+id: sort-concat-flag
+label: sort-concat-flag
 
 inputs:
   - id: msin
@@ -18,7 +18,7 @@ steps:
       - id: logfile
     run: ../steps/sort_concatmap.cwl
     label: sort_concatmap
-  - id: concatenate
+  - id: concatenate-flag
     in:
       - id: msin
         source:
@@ -29,21 +29,33 @@ steps:
         source: sort_concatmap/filenames
     out:
       - id: msout
-      - id: logfile
+      - id: aoflag_logfile
+      - id: concatenate_logfile
     run: ./subworkflows/concatenation.cwl
     scatter: group_id
-    label: concatenation
+    label: concatenation-flag
   - id: concatenate_logfiles_concatenate
     in:
       - id: file_list
         source:
-          - concatenate/logfile
+          - concatenate-flag/concatenate_logfile
       - id: file_prefix
         default: concatenate
     out:
       - id: output
     run: ../steps/concatenate_files.cwl
     label: concatenate_logfiles_concatenate
+  - id: concatenate_logfiles_aoflagging
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source: concatenate-flag/aoflag_logfile
+      - id: file_prefix
+        default: AOflagging
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_AOflagging
   - id: save_logfiles
     in:
       - id: files
@@ -51,8 +63,9 @@ steps:
         source:
             - sort_concatmap/logfile
             - concatenate_logfiles_concatenate/output
+            - concatenate_logfiles_aoflagging/output
       - id: sub_directory_name
-        default: 'sort-concat'
+        default: 'sort-concat-flag'
     out:
       - id: dir
     run: ../steps/collectfiles.cwl
@@ -63,7 +76,7 @@ outputs:
       outputSource: save_logfiles/dir
       type: Directory
     - id: msout
-      outputSource: concatenate/msout
+      outputSource: concatenate-flag/msout
       type: Directory[]
 
 requirements:

workflows/delay-calibration.cwl

@@ -54,26 +54,15 @@ steps:
       run: ./clipAteam.cwl
       label: clipAteam
 
-    - id: sort-concatenate
+    - id: sort-concatenate-flag
       in:
         - id: msin
           source: clipAteam/msout
       out:
         - id: logdir
         - id: msout
-      run: ./sort-concat.cwl
-      label: sort-concatenate
-
-#    - id: aoflagging
-#      in:
-#        - id: input1
-#          source: input1
-#        - id: input2
-#          source: input2
-#      out:
-#        - id: output1
-#      run: ../steps/step1.cwl
-#      label: step1
+      run: ./concatenate-flag.cwl
+      label: sort-concatenate-flag
 
     - id: store_logs
       in:
@@ -82,7 +71,8 @@ steps:
           source:
             - setup/logdir
             - clipAteam/logdir
-            - sort-concatenate/logdir
+            - sort-concatenate-flag/logdir
+#            - aoflagging/logdir
         - id: sub_directory_name
           default: logs
       out:
@@ -136,7 +126,7 @@ steps:
 
 outputs:
   - id: msout
-    outputSource: sort-concatenate/msout
+    outputSource: sort-concatenate-flag/msout
     type: Directory[]
   - id: logs
     outputSource: store_logs/dir

workflows/subworkflows/concatenation.cwl

@@ -37,6 +37,26 @@ steps:
       - id: logfile
     run: ../../steps/dp3_concat.cwl
     label: dp3_concat
+  - id: AOflagging
+    in:
+      - id: msin
+        source: dp3_concat/msout
+    out:
+      - id: msout
+      - id: logfile
+    run: ../../steps/aoflagger.cwl
+    label: AOflagging
+  - id: concat_logfiles_aoflagging
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source: AOflagging/logfile
+      - id: file_prefix
+        default: AOflagging
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_AOflagging
   - id: dp3_concatenate_logfiles
     in:
       - id: file_list
@@ -51,8 +71,11 @@ steps:
 
 outputs:
   - id: msout
-    outputSource: dp3_concat/msout
+    outputSource: AOflagging/msout
     type: Directory
-  - id: logfile
+  - id: concatenate_logfile
     outputSource: dp3_concatenate_logfiles/output
     type: File
+  - id: aoflag_logfile
+    outputSource: concat_logfiles_aoflagging/output
+    type: File