Merge branch 'flag_counter' into 'master'

Flag counter

See merge request eosc/prefactor3-cwl!38

Former-commit-id: 07c34604
Former-commit-id: 4a0807dc

Docker/build_docker.sh

@@ -3,8 +3,8 @@ set -e
 
 DOCKER_TAG=latest
 
-#docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
-#docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
+docker build ${PWD} -f Dockerfile-base -t lofareosc/lofar-pipeline:${DOCKER_TAG}
+docker push lofareosc/lofar-pipeline:${DOCKER_TAG}
 
 docker build --no-cache ${PWD} -t lofareosc/prefactor3-cwl:${DOCKER_TAG}
 docker push lofareosc/prefactor3-cwl:${DOCKER_TAG}

lofar-cwl/steps/DPPP.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: dppp
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: parset
     type: File?

lofar-cwl/steps/applybeam.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: applybeam
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory

lofar-cwl/steps/applycal.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: applycal
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory

lofar-cwl/steps/applytarget.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: applytarget
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory
@@ -80,12 +80,30 @@ inputs:
       position: 0
       prefix: applycal.updateweights=
       separate: false
+  - id: save2json
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count.savetojson=True
+  - id: jsonfilename
+    type: string?
+    default: 'out.json'
+    inputBinding:
+      prefix: count.jsonfilename=
+      separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set
     type: Directory
     outputBinding:
       glob: '$(inputs.msout_name=="."?inputs.msin.basename:inputs.msout_name)'
+  - id: flagged_fraction_dict
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: $(inputs.jsonfilename)
+        outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
   - id: logfile
     type: File[]
     outputBinding:

lofar-cwl/steps/average.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: average
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory

lofar-cwl/steps/calib_rot_diag.cwl

@@ -5,7 +5,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: calib_rot_diag
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory?
@@ -37,7 +37,7 @@ inputs:
       position: 0
       prefix: ddecal.h5parm=
       separate: false
-  - default: out.MS
+  - default: '.'
     id: msout_name
     type: string
     inputBinding:

lofar-cwl/steps/ddecal.cwl

@@ -3,7 +3,7 @@
 class: CommandLineTool
 cwlVersion: v1.1
 id: calib_rot_diag
-baseCommand: [DPPP]
+baseCommand: [DP3]
 
 requirements:
   InlineJavascriptRequirement: {}
@@ -60,7 +60,7 @@ inputs:
   - id: msout_name
     type: string?
     doc: Output Measurement Set
-    default: "OUT.MS"
+    default: '.'
     inputBinding:
       prefix: msout=
       separate: false
@@ -119,6 +119,21 @@ inputs:
     inputBinding:
       prefix: ddecal.mode=
       separate: false
+      
+  - id: save2json
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count.savetojson=True
+      
+  - id: jsonfilename
+    type: string?
+    default: 'out.json'
+    inputBinding:
+      prefix: count.jsonfilename=
+      separate: false
+
 outputs:
   - id: msout
     doc: Output Measurement Set
@@ -133,6 +148,13 @@ outputs:
     outputBinding:
       glob: $(inputs.output_name_h5parm)
       
+  - id: flagged_fraction_dict
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: $(inputs.jsonfilename)
+        outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
+
   - id: logfile
     type: File[]
     outputBinding:

lofar-cwl/steps/dpppconcat.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: dpppconcat
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: 'Directory[]'
@@ -118,12 +118,30 @@ inputs:
       shellQuote: false
       position: 0
     default: 12.205kHz
+  - id: save2json
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count.savetojson=True
+  - id: jsonfilename
+    type: string?
+    default: 'out.json'
+    inputBinding:
+      prefix: count.jsonfilename=
+      separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set
     type: Directory
     outputBinding:
       glob: $(inputs.msout_name)
+  - id: flagged_fraction_dict
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: $(inputs.jsonfilename)
+        outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
   - id: logfile
     type: 'File[]'
     outputBinding:

lofar-cwl/steps/filter_predict.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: predict
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory
@@ -29,7 +29,9 @@ inputs:
       prefix: msout.datacolumn=
       separate: false
   - id: sources_db
-    type: File
+    type:
+      - File
+      - Directory
     inputBinding:
       position: 0
       prefix: predict.sourcedb=

lofar-cwl/steps/gaincal.cwl

@@ -5,7 +5,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: gaincal
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory
@@ -34,7 +34,9 @@ inputs:
     doc: |
       The type of calibration that needs to be performed.
   - id: sourcedb
-    type: File
+    type:
+      - File
+      - Directory
     inputBinding:
       position: 0
       prefix: gaincal.sourcedb=
@@ -89,7 +91,7 @@ inputs:
       position: 0
       prefix: gaincal.parmdb=
       separate: false
-  - default: out.MS
+  - default: '.'
     id: msout_name
     type: string?
     inputBinding:

lofar-cwl/steps/interpolate_visibilities.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: interpolate_visibilities
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory

lofar-cwl/steps/predict.cwl

@@ -4,7 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: predict
 baseCommand:
-  - DPPP
+  - DP3
 inputs:
   - id: msin
     type: Directory
@@ -29,7 +29,9 @@ inputs:
       prefix: msout.datacolumn=
       separate: false
   - id: sources_db
-    type: File
+    type:
+      - File
+      - Directory
     inputBinding:
       position: 0
       prefix: predict.sourcedb=

lofar-cwl/steps/wsclean.cwl

@@ -15,8 +15,8 @@ inputs:
       valueFrom: $(concatenate_path_wsclean(self))
   - id: image_size
     default:
-      - 1000
-      - 1000
+      - 2500
+      - 2500
     type: 'int[]'
     inputBinding:
       position: 1
@@ -30,7 +30,7 @@ inputs:
       prefix: '-scale'
       shellQuote: false
   - id: niter
-    default: 10000
+    default: 1000
     type: int?
     inputBinding:
       position: 1

steps/createRMh5parm.cwl

@@ -113,7 +113,9 @@ hints:
    listing:
      - entry: $(inputs.h5parm)
        writable: true
-
+requirements:
+  - class: NetworkAccess
+    networkAccess: true
 stdout: createh5parm.log
 stderr: createh5parm_err.log
 $namespaces:

steps/dppp_make_parset_cal.cwl

@@ -75,7 +75,7 @@ requirements:
     listing:
       - entryname: input.parset
         entry: |+
-          steps                               =  [$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,avg]
+          steps                               =  [count,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,avg]
           #
           msout.storagemanager                =   "Dysco"
           msout.storagemanager.databitrate    =   0

steps/dppp_make_parset_target.cwl

@@ -93,11 +93,14 @@ requirements:
     listing:
       - entryname: input.parset
         entry: |+
-          steps                               =  [$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,applyPA,applybandpass,$(inputs.apply_clock_correction?'applyclock,':'')$(inputs.apply_beam_correction?'applybeam,':'')$(inputs.apply_rm_correction?'applyRM,':'')$(inputs.apply_tec_correction?'applytec,':'')$(inputs.apply_phase_correction?'applyphase,':'')avg]
+          steps                               =  [count1,$(inputs.raw_data?'flagedge,aoflag,':'')flagbaseline,flagelev,flagamp,$(inputs.demix?'demix,':'')filter,applyPA,applybandpass,$(inputs.apply_clock_correction?'applyclock,':'')$(inputs.apply_beam_correction?'applybeam,':'')$(inputs.apply_rm_correction?'applyRM,':'')$(inputs.apply_tec_correction?'applytec,':'')$(inputs.apply_phase_correction?'applyphase,':'')avg,count2]
           #
           msout.storagemanager                =   "Dysco"
           msout.storagemanager.databitrate    =   0
           #
+          count1.type                         =   counter
+          count2.type                         =   counter
+          #
           filter.type                         =   filter
           filter.baseline                     =   $(inputs.filter_baselines)
           filter.remove                       =   true

steps/dppp_prep_cal.cwl

@@ -75,17 +75,37 @@ inputs:
       prefix: msout.storagemanager.databitrate=
       separate: false
   - id: skymodel
-    type: File
+    type:
+      - File
+      - Directory
     inputBinding:
         position: 0
         prefix: demix.skymodel=
         separate: false
+  - id: save2json
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count.savetojson=True
+  - id: jsonfilename
+    type: string?
+    default: 'out.json'
+    inputBinding:
+      prefix: count.jsonfilename=
+      separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set
     type: Directory
     outputBinding:
       glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+  - id: flagged_fraction_dict
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: $(inputs.jsonfilename)
+        outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
   - id: logfile
     type: File[]
     outputBinding:

steps/dppp_prep_target.cwl

@@ -110,11 +110,37 @@ inputs:
       separate: false
       valueFrom: $(inputs.solutions.path)
   - id: skymodel
-    type: File
+    type:
+      - File
+      - Directory
     inputBinding:
         position: 0
         prefix: demix.skymodel=
         separate: false
+  - id: save2json1
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count1.savetojson=True
+  - id: jsonfilename1
+    type: string?
+    default: 'out1.json'
+    inputBinding:
+      prefix: count1.jsonfilename=
+      separate: false
+  - id: save2json2
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count2.savetojson=True
+  - id: jsonfilename2
+    type: string?
+    default: 'out2.json'
+    inputBinding:
+      prefix: count2.jsonfilename=
+      separate: false
 arguments:
   - applyPA.parmdb=$(inputs.solutions.path)
   - applybandpass.parmdb=$(inputs.solutions.path)
@@ -124,6 +150,18 @@ outputs:
     type: Directory
     outputBinding:
       glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+  - id: flagged_fraction_dict_initial
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: $(inputs.jsonfilename1)
+        outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
+  - id: flagged_fraction_dict_prep
+    type: string
+    outputBinding:
+        loadContents: true
+        glob: $(inputs.jsonfilename2)
+        outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
   - id: logfile
     type: File[]
     outputBinding:

steps/findRefAnt.cwl

@@ -2,45 +2,43 @@ class: CommandLineTool
 cwlVersion: v1.1
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
-id: findRefAnt
+id: dppp
 baseCommand:
-  - python3
+  - DP3
 inputs:
   - id: msin
     type: Directory
-      doc: MS to compare with
     inputBinding:
       position: 0
-
-label: findRefAnt.py
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - id: save2json
+    default: true
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: count.savetojson=True
+  - id: jsonfilename
+    type: string?
+    default: 'out.json'
+    inputBinding:
+      prefix: count.jsonfilename=
+      separate: false
 arguments:
-  - '-c'
-  - |
-    import sys
-    import json
-    from findRefAnt import main as findRefAnt
-    
-    ms = sys.argv[1]
-
-    flagged_fraction_dict = str(findRefAnt(ms))
-
-    cwl_output  = {"flagged_fraction_dict": flagged_fraction_dict}
-
-    with open('./out.json', 'w') as fp:
-        json.dump(cwl_output, fp)
-        
+  - steps=[count]
+  - msout=
 outputs:
   - id: flagged_fraction_dict
     type: string
     outputBinding:
         loadContents: true
-        glob: 'out.json'
+        glob: $(inputs.jsonfilename)
         outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
   - id: logfile
     type: File
     outputBinding:
       glob: findRefAnt.log
-        
 requirements:
   - class: InlineJavascriptRequirement
   - class: InitialWorkDirRequirement