finish first version target pipeline

Former-commit-id: 7350e7bf
Former-commit-id: 1381a821

lofar-cwl/steps/applytarget.cwl

@@ -55,6 +55,13 @@ inputs:
       Type of correction to perform. When using H5Parm, this is for now the name
       of the soltab; the type will be deduced from the metadata in that soltab,
       except for full Jones, in which case correction should be 'fulljones'.
+  - default: 'sol000'
+    id: solset
+    type: string
+    inputBinding:
+      position: 0
+      prefix: applycal.solset=
+      separate: false
   - id: storagemanager
     type: string
     default: ""

lofar-cwl/steps/dpppconcat.cwl

@@ -9,11 +9,11 @@ inputs:
   - id: msin
     type: 'Directory[]'
     inputBinding:
+      position: 0
       prefix: msin=
       separate: false
       itemSeparator: ','
-      shellQuote: false
-      position: 0
+      valueFrom: $(concatenate_path(self))
     doc: Input Measurement Set
   - id: msout_name
     type: string
@@ -44,7 +44,7 @@ inputs:
     inputBinding:
       prefix: filter.baseline=
       separate: false
-      shellQuote: false
+      shellQuote: true
       position: 0
       valueFrom: $(self)
     default: null
@@ -96,7 +96,7 @@ inputs:
     inputBinding:
       prefix: msin.baseline=
       separate: false
-      shellQuote: false
+      shellQuote: true
       position: 0
     default: '*'
   - id: avg_timeresolution
@@ -128,9 +128,12 @@ outputs:
 arguments:
   - 'steps=[filter,avg]'
   - msout.orderms=False
+  - avg.type=average
 requirements:
   - class: ShellCommandRequirement
   - class: InlineJavascriptRequirement
+    expressionLib:
+      - { $include: 'utils.js' }
 hints:
   - class: DockerRequirement
     dockerPull: 'lofareosc/prefactor:HBAcalibrator'

steps/add_missing_stations.cwl

 class: CommandLineTool
 cwlVersion: v1.0
 $namespaces:
+  lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
   sbg: 'https://www.sevenbridges.com/'
 id: add_missing_stations
 baseCommand:
@@ -17,13 +18,14 @@ inputs:
     type: File
     inputBinding:
       position: 0
-      prefix: '--refh5parm'
+      prefix: '--refh5'
     doc: External H5parm from which the full list of antennas is used from.
   - id: solset
     type: string?
     inputBinding:
       position: 0
       prefix: '--solset'
+    default: 'sol000'
     doc: Input calibration solutions
   - id: refsolset
     type: string?
@@ -31,35 +33,41 @@ inputs:
       position: 0
       prefix: '--refsolset'
     doc: Input calibration solutions of the reference h5parm file
+    default: 'sol000'
   - id: soltab_in
     type: string?
     inputBinding:
       position: 0
       prefix: '--soltab_in'
     doc: Input solution table
+    default: 'phase000'
   - id: soltab_out
     type: string?
     inputBinding:
       position: 0
       prefix: '--soltab_out'
     doc: Output solution table (has to be different from input solution table)
+    default: 'GSMphase'
   - id: filter
     type: string?
     inputBinding:
       position: 0
       prefix: '--filter'
     doc: Filter these antenna string from the processing
+    default: '[CR]S*&'
   - id: bad_antennas
     type: string?
     inputBinding:
       position: 0
       prefix: '--bad_antennas'
     doc: Antenna string to be processed
+    default: '[CR]S*&'
 outputs:
   - id: outh5parm
     type: File
     outputBinding:
       glob: $(inputs.h5parm.basename)
+    format: 'lofar:#H5Parm'
   - id: log
     type: File[]
     outputBinding:
@@ -77,3 +85,5 @@ requirements:
     listing:
       - entry: $(inputs.h5parm)
         writable: true
+$schema:
+  - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'

steps/filter_ms_group.cwl

@@ -34,12 +34,13 @@ requirements:
        entry: |
         import sys
         import json
-        import os.path as path
+        import os
+
         group_id = "$(inputs.group_id)"
         json_file = sys.argv[1]
         ms_list = sys.argv[2:]
         
-        ms_by_name = { ms.split(path.sep)[-1]:
+        ms_by_name = { ms.split(os.path.sep)[-1]:
                        {'class':'Directory', 'path': ms} for ms in ms_list}
         
         output_file = 'selected_ms.json'
@@ -47,7 +48,7 @@ requirements:
         with open(json_file, 'r') as f_stream:
             selected_ms = json.load(f_stream)[group_id]
         print(selected_ms, ms_by_name)
-        selected_ms = [ms_by_name[ms_name] for ms_name in selected_ms]
+        selected_ms = [ms_by_name[os.path.basename(ms_name)] for ms_name in selected_ms]
             
         
         with open(output_file, 'w') as f_stream:

steps/find_skymodel_target.cwl

@@ -52,7 +52,7 @@ arguments:
     
     output = download_skymodel_target(mss, SkymodelPath, Radius, DoDownload, Source)
     
-    if SkymodelPath != "target.skymodel":
+    if "target.skymodel" not in SkymodelPath:
        shutil.copyfile(SkymodelPath, "target.skymodel")
    
 outputs:

steps/h5parmcat.cwl

@@ -90,7 +90,7 @@ requirements:
 hints:
   - class: DockerRequirement
     dockerPull: lofareosc/prefactor:HBAcalibrator
-stdout: parm_collector_output.log
-stderr: parm_collector_output_err.log
+stdout: h5parm_collector_output.log
+stderr: h5parm_collector_output_err.log
 $schema:
   - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'

steps/identify_bad_antennas.cwl

@@ -48,12 +48,12 @@ outputs:
         
 requirements:
   - class: InlineJavascriptRequirement
-
-hints:
   - class: InitialWorkDirRequirement
     listing:
       - entry: $(inputs.msin)
         writable: true
+hints:
   - class: DockerRequirement
     dockerPull: lofareosc/prefactor:HBAcalibrator
 stdout: identifyBadAntennas.log
+stderr: identifyBadAntennas_err.log
\ No newline at end of file

subworkflow/ndppp_prep_targ.cwl

@@ -472,3 +472,4 @@ steps:
 requirements:
   - class: SubworkflowFeatureRequirement
   - class: InlineJavascriptRequirement
+  - class: StepInputExpressionRequirement

workflows/prefactor_target/finalize.cwl

@@ -72,6 +72,7 @@ steps:
         source: insolutions
       - id: soltab_in
         source: gsmcal_step
+        valueFrom: $(self+'000')
       - id: soltab_out
         default: 'TGSSphase'
       - id: filter
@@ -171,3 +172,5 @@ steps:
 requirements:
   - class: SubworkflowFeatureRequirement
   - class: ScatterFeatureRequirement
+  - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement
\ No newline at end of file