Merge branch 'testing' into 'master'

Implemented new workflow for A-team clipping

See merge request !50

scripts/TargetListToCoords.py

@@ -32,7 +32,7 @@ def plugin_main(**kwargs):
         DEC_val = [DEC_val[0]]
         Source_id = Source_id[0]
         if isinstance(Source_id, str):
-            pass
+            Source_id = [Source_id]
         else:
             Source_id = ['S' + str(Source_id)]
 
@@ -45,6 +45,6 @@ def plugin_main(**kwargs):
     else:
         raise ValueError("Argument mode must be one of"
                         + " \"delay_calibration\", \"split_directions\""
-                        + f" but was {mode}.")
+                        + f" but was \"{mode}\".")
 
     return {'name' : ",".join(Source_id), 'coords' : ss}

steps/Ateamclipper.cwl

-class: CommandLineTool
-cwlVersion: v1.2
-id: Ateamclipper
-label: A-team clipper
-doc: |
-    Clipping of the A-team sources
-    using LINC`s Ateamclipper script.
-
-baseCommand:
-  - python3
-  - Ateamclipper.py
-
-inputs:
-  - id: msin
-    type:
-      - Directory
-      - type: array
-        items: Directory
-    inputBinding:
-      position: 0
-    doc: Input measurementSet.
-
-  - id: number_cores
-    type: int?
-    default: 12
-    doc: |
-     Number of cores to use per job for tasks with
-     high I/O or memory.
-
-  - id: linc_libraries
-    type: File[]
-    doc: |
-      Scripts and reference files from the
-      LOFAR INitial calibration pipeline.
-      Must contain `Ateamclipper.py`.
-
-outputs:
-  - id: msout
-    doc: Output MeasurementSet.
-    type: Directory
-    outputBinding:
-      glob: $(inputs.msin.basename)
-
-  - id: logfile
-    type: File[]
-    outputBinding:
-      glob: Ateamclipper.log
-    doc: The files containing the stdout from the step.
-
-  - id: output
-    type: File
-    outputBinding:
-      glob: Ateamclipper.txt
-    doc: A text file containing flagging fraction statistics.
-
-hints:
-  - class: InitialWorkDirRequirement
-    listing:
-      - entry: $(inputs.msin)
-        writable: true
-      - entry: $(inputs.linc_libraries)
-  - class: InplaceUpdateRequirement
-    inplaceUpdate: true
-  - class: DockerRequirement
-    dockerPull: vlbi-cwl
-  - class: InlineJavascriptRequirement
-  - class: ResourceRequirement
-    coresMin: $(inputs.number_cores)
-
-stdout: Ateamclipper.log
-stderr: Ateamclipper_err.log

steps/predict.cwl → steps/clipper.cwl

 class: CommandLineTool
 cwlVersion: v1.2
-id: predict_ateam
-label: DP3 predict A-team
+id: Ateamclipper
+label: clip A-team
 doc: |
-    Simulates data for the A-team sources
-    based off a skymodel. Writes the simulated
-    data to the MSOUT datacolumn of the given
-    MeasurementSet.
+    Simulates data for the A-team sources based off a skymodel,
+    and flags the visibilities of the input MeasuremenSet where
+    the model data exceeds the threshold for LBA (50 janskys)
+    or HBA (5 janksys).
 
 baseCommand: DP3
 arguments:
-  - steps=[filter,predict]
-  - predict.sourcedb=Ateam_LBA_CC.skymodel
-  - predict.usechannelfreq=False
-  - predict.operation=replace
-  - predict.beamproximitylimit=2000
+  - steps=[filter,clipper,counter]
+  - clipper.sourcedb=Ateam_LBA_CC.skymodel
+  - clipper.usechannelfreq=False
+  - clipper.operation=replace
+  - clipper.beamproximitylimit=2000
   - filter.baseline=[CR]S*&
   - filter.remove=False
+  - counter.savetojson=True
+  - counter.jsonfilename=Ateamclipper.json
   - msout=.
 
-
 inputs:
   - id: msin
     type: Directory
@@ -42,18 +43,6 @@ inputs:
         Data column of the MeasurementSet
         from which input data is read.
 
-  - id: msout_datacolumn
-    type: string?
-    default: MODEL_DATA
-    inputBinding:
-      position: 0
-      prefix: msout.datacolumn=
-      separate: false
-      shellQuote: false
-    doc: |
-        Data column of the MeasurementSet
-        into which output data is written.
-
   - id: linc_libraries
     type: File[]
     doc: |
@@ -70,7 +59,7 @@ inputs:
         - "TauAGG"
     inputBinding:
       position: 0
-      prefix: predict.sources=
+      prefix: clipper.sources=
       separate: false
       itemSeparator: ','
       valueFrom: '"$(self)"'
@@ -84,34 +73,11 @@ inputs:
     default: true
     inputBinding:
       position: 0
-      prefix: predict.usebeammodel=True
+      prefix: clipper.usebeammodel=True
       shellQuote: false
     doc: |
         Determines whether to use the beam model.
 
-  - id: storagemanager
-    type: string?
-    default: "dysco"
-    inputBinding:
-      prefix: msout.storagemanager=
-      separate: false
-      shellQuote: false
-    doc: |
-        String that specifies what storage manager
-        to use. By default uses `dysco` compression.
-
-  - id: databitrate
-    type: int?
-    default: 0
-    inputBinding:
-      prefix: msout.storagemanager.databitrate=
-      separate: false
-      shellQuote: false
-    doc: |
-        Number of bits per float used for columns
-        containing visibilities. Default compresses
-        weights only.
-
   - id: max_dp3_threads
     type: int?
     default: 5
@@ -121,6 +87,13 @@ inputs:
       shellQuote: false
     doc: The number of threads per DP3 process.
 
+  - id: number_cores
+    type: int?
+    default: 12
+    doc: |
+     Number of cores to use per job for tasks with
+     high I/O or memory.
+
 requirements:
   - class: InitialWorkDirRequirement
     listing:
@@ -140,16 +113,22 @@ outputs:
   - id: logfile
     type: File[]
     outputBinding:
-      glob: predict_ateam*.log
+      glob: clipper*.log
     doc: |
         The files containing the stdout
         and stderr from the step.
 
+  - id: output
+    type: File
+    outputBinding:
+      glob: Ateamclipper.json
+    doc: A text file containing flagging fraction statistics.
+
 hints:
   - class: ResourceRequirement
-    coresMin: 6
+    coresMin: $(inputs.number_cores)
   - class: DockerRequirement
     dockerPull: vlbi-cwl
 
-stdout: predict_ateam.log
-stderr: predict_ateam_err.log
+stdout: clipper.log
+stderr: clipper_err.log

workflows/subworkflows/clip_A-team.cwl

@@ -62,7 +62,7 @@ steps:
         - id: flag_statistics_before
         - id: flag_statistics_after
       run: ../../steps/dp3_prep_target.cwl
-    - id: predict
+    - id: Ateamclipper
       in:
         - id: msin
           source: dp3_prep_target/msout
@@ -70,25 +70,13 @@ steps:
           source: max_dp3_threads
         - id: linc_libraries
           source: linc_libraries
-      out:
-        - id: msout
-        - id: logfile
-      run: ../../steps/predict.cwl
-      label: predict
-    - id: Ateamcliptar
-      in:
-        - id: msin
-          source: predict/msout
         - id: number_cores
           source: number_cores
-        - id: linc_libraries
-          source: linc_libraries
       out:
         - id: msout
         - id: logfile
-        - id: output
-      run: ../../steps/Ateamclipper.cwl
-      label: Ateamcliptar
+        - id: output 
+      run: ../../steps/clipper.cwl
     - id: concat_logfiles_prep_targ
       label: concat_logfiles_prep_target
       in:
@@ -101,24 +89,13 @@ steps:
       out:
         - id: output
       run: ../../steps/concatenate_files.cwl
-    - id: concat_logfiles_predict
-      in:
-        - id: file_list
-          linkMerge: merge_flattened
-          source: predict/logfile
-        - id: file_prefix
-          default: predict
-      out:
-        - id: output
-      run: ../../steps/concatenate_files.cwl
-      label: concat_logfiles_predict
     - id: concat_logfiles_cliptar
       in:
         - id: file_list
           linkMerge: merge_flattened
-          source: Ateamcliptar/logfile
+          source: Ateamclipper/logfile
         - id: file_prefix
-          default: Ateamcliptar
+          default: Ateamclipper
       out:
         - id: output
       run: ../../steps/concatenate_files.cwl
@@ -129,7 +106,6 @@ steps:
           linkMerge: merge_flattened
           source:
             - concat_logfiles_prep_targ/output
-            - concat_logfiles_predict/output
             - concat_logfiles_cliptar/output
         - id: file_prefix
           default: Ateamcliptar
@@ -149,7 +125,7 @@ outputs:
 
     - id: msout
       outputSource:
-        - Ateamcliptar/msout
+        - Ateamclipper/msout
       type: Directory
       doc: |
         The input data in MeasurementSet format after