Merge branch 'add_dp3_execute_step' into 'master'

Add dp3 execute step

See merge request eosc/prefactor3-cwl!14

.gitlab-ci.yml

@@ -15,7 +15,6 @@ before_script:
    # make toil workdir
    - mkdir workdir
    - mkdir logs
-   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 after_script:
   - echo "All done"
@@ -30,10 +29,12 @@ build_docker:
     
 push_docker_tags:
   image: docker:latest
+  allow_failure: true
   stage: push_docker
   only: 
     - tags
   script: 
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
     - docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
 
@@ -46,8 +47,6 @@ push_docker_master:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker push lofareosc/prefactor-ci:latest
 
-
-
 validate_scripts:
   stage: validate_cwl_scripts
   script:
@@ -126,6 +125,12 @@ NDPPP:
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
 
+DP3.Execute:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwltool --no-container --preserve-enviroment LD_LIBRARY_PATH --preserve-enviroment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json
+
 h5parm_collector:
   stage: test_steps
   allow_failure: true

Docker/Dockerfile

@@ -65,7 +65,7 @@ RUN cd /src/Dysco/ && \
     make install -j2
 
 #----------------------------------------------
-FROM kernsuite/base:4 as runner
+FROM kernsuite/base:5 as runner
 
 COPY --from=builder /usr/local/ /usr/local
 
@@ -85,13 +85,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libboost-system1.65.1 \
         libboost-signals1.65.1 \
         libboost-test1.65.1 \
-        libboost-python1.65-dev \ 
-        libcfitsio5 \ 
+        libboost-python1.65-dev \
+        libstationresponse2 \
+        libcasa-* \
+        libcfitsio5 \
         libgtkmm-3.0 \
         libfftw3-3 \
         libhdf5-cpp-100 \
         libpng16-16 \
-        libstationresponse0 \ 
         libxml2 \
         python3.7 \
         python3-numpy \

steps/DP3.Execute.cwl

+class: Workflow
+cwlVersion: v1.0
+id: dp3_execute
+label: DP3.Execute
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+inputs:
+  - id: msout_name
+    type: string
+    'sbg:x': -425.39886474609375
+    'sbg:y': 127
+  - id: msin
+    type: Directory
+    'sbg:x': -254.39886474609375
+    'sbg:y': 242
+  - id: steps
+    type: 'Any[]?'
+    'sbg:x': -518.3988647460938
+    'sbg:y': -88
+outputs:
+  - id: secondary_output_files
+    outputSource:
+      - dppp/secondary_output_files
+    type: Any
+    'sbg:x': 213.60113525390625
+    'sbg:y': -184
+  - id: secondary_output_directories
+    outputSource:
+      - dppp/secondary_output_directories
+    type: Any
+    'sbg:x': 248.60113525390625
+    'sbg:y': -47
+  - id: msout
+    outputSource:
+      - dppp/msout
+    type: Directory
+    'sbg:x': 224.60113525390625
+    'sbg:y': 130
+steps:
+  - id: generic_step
+    in:
+      - id: steps
+        source:
+          - steps
+    out:
+      - id: parset
+      - id: input_files
+      - id: input_directories
+      - id: output_file_names
+      - id: output_directory_names
+    run: ./DP3.ParsetGenerator.cwl
+    'sbg:x': -294
+    'sbg:y': -84
+  - id: dppp
+    in:
+      - id: parset
+        source: generic_step/parset
+      - id: msin
+        source: msin
+      - id: msout_name
+        source: msout_name
+      - id: secondary_files
+        source:
+          - generic_step/input_files
+      - id: secondary_directories
+        source:
+          - generic_step/input_directories
+      - id: output_file_names
+        source: generic_step/output_file_names
+      - id: output_directory_names
+        source: generic_step/output_directory_names
+    out:
+      - id: msout
+      - id: secondary_output_files
+      - id: secondary_output_directories
+    run: ./DPPP.cwl
+    'sbg:x': 26
+    'sbg:y': -46
+requirements: []

test_jobs/dp3_execute.json

+{ 
+  "msin": {"class": "Directory", "path":"/data/L570745_SB003_uv_first10.MS"},
+  "steps": [
+        {
+            "step_type": "averager",
+            "step_id": "average",
+            "parameters": {
+                "timestep": 1,
+                "freqstep": 1,
+                "minpoints": 0,
+                "minperc": 0,
+                "timeresolution": 0,
+                "freqresolution": 0
+            },
+            "output_files": {}
+        }
+    ],
+  "msout_name": "TEST" 
+}