Add blsmooth and refactored gitlab integration

.gitlab-ci.yml

@@ -25,24 +25,73 @@ validate_scripts:
   script:
     - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
 
-test_steps:
-  stage: test
+average_step_generator:
+  stage: test_steps
   allow_failure: true
   script:
     - cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
+    - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
+
+aoflagger_step_generator:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
+
+preflagger_step_generator:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
+
+parset_selector:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwl-runner steps/DP3.ParsetSelector.cwl test_jobs/parset_selector.json
+
+file_selector:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
-    - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH test_workflow/flag_average.cwl --msin=/data/L570745_SB001_uv_first10.MS
+
+NDPPP:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json 
+
+aoflagger:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
+
+find_skymodel_cal:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
+
+interpolate_visibilities:
+  stage: test_steps
+  allow_failure: true
+  script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
   
+blsmooth:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+
 
-integration_test: 
-  stage: test
+prefactor_calibrator: 
+  stage: integration_test
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json
+
+flag_average: 
+  stage: integration_test
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH test_workflow/flag_average.cwl --msin=/data/L570745_SB001_uv_first10.MS

steps/blsmooth.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+id: blsmooth
+label: BLsmooth
+arguments:
+  - '-S SMOOTH'
+baseCommand:
+  - BLsmooth.py
+inputs:
+  - id: msin
+    type: Directory
+    inputBinding:
+      position: 1
+    doc: Input measurement set
+  - default: 0.2
+    id: ionfactor
+    type: float
+    inputBinding:
+      position: 0
+      prefix: '-f'
+    doc: Gives an indication on how strong is the ionosphere
+  - default: 0.5
+    id: bscalefactor
+    type: float
+    inputBinding:
+      position: 0
+      prefix: '-s'
+    doc: Gives an indication on how the smoothing varies with
+  - default: DATA
+    id: in_column_name
+    type: string
+    inputBinding:
+      position: 0
+      prefix: '-i'
+    doc: Column name to smooth
+  - default: SMOOTHED_DATA
+    id: out_column
+    type: string
+    inputBinding:
+      position: 0
+      prefix: '-o'
+    doc: Output column
+  - default: false
+    id: weight
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: '-w'
+    doc: >-
+      Save the newly computed WEIGHT_SPECTRUM, this action permanently modify
+      the MS!
+  - default: false
+    id: restore
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: '-r'
+    doc: If WEIGHT_SPECTRUM_ORIG exists then restore it before smoothing
+  - default: false
+    id: nobackup
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: '-b'
+    doc: Do not backup the old WEIGHT_SPECTRUM in WEIGHT_SPECTRUM_ORIG
+  - default: false
+    id: onlyamp
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: '-a'
+    doc: Smooth only amplitudes
+outputs:
+  - id: msout
+    doc: MS set output
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msin.basename)
+
+requirements:
+- class: DockerRequirement
+  dockerPull: 'prefactor:latest'
+- class: InitialWorkDirRequirement
+  listing:
+    - entry: $(inputs.msin)
+      writable: true
+- class: InlineJavascriptRequirement

test_jobs/blsmooth.json

+{
+    "msin": {
+            "class": "Directory",
+            "path": "/data/L570745_SB003_uv_first10.MS"
+        }
+}

test_jobs/demo_prefactor.json

@@ -2,15 +2,15 @@
     "msin": [
         {
             "class": "Directory",
-            "path": "/home/mmancini/git/prefactor3-cwl/test_data/L570745_SB003_uv_first10.MS"
+            "path": "/data/L570745_SB003_uv_first10.MS"
         },
         {
             "class": "Directory",
-            "path": "/home/mmancini/git/prefactor3-cwl/test_data/L570745_SB005_uv_first10.MS"
+            "path": "/data/L570745_SB005_uv_first10.MS"
         },
         {
             "class": "Directory",
-            "path": "/home/mmancini/git/prefactor3-cwl/test_data/L570745_SB008_uv_first10.MS"
+            "path": "/data/L570745_SB008_uv_first10.MS"
         }
     ]
 }