Merge branch 'master' of git.astron.nl:eosc/prefactor3-cwl

.gitlab-ci.yml

@@ -9,15 +9,10 @@ stages:
   - test_steps
   - integration_test
 
-
 before_script:
-#  - python3 -V  # Print out python version for debugging
-#  - apt-get update && apt-get install -y nodejs python3-pip
-#  - python3 -m pip install cwltool cwl-runner
-#  - cd test_data
-#  - tar -xf ${TEST_DATASET_NAME}
-#  - cd $CI_PROJECT_DIR
-#  - source /opt/bashrc
+   # make toil workdir
+   - mkdir workdir
+   - mkdir logs
 
 after_script:
   - echo "All done"
@@ -50,7 +45,7 @@ parset_selector:
   stage: test_steps
   allow_failure: true
   script:
-    - cwl-runner steps/DP3.ParsetSelector.cwl test_jobs/parset_selector.json
+    - cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json
 
 file_selector:
   stage: test_steps
@@ -117,8 +112,12 @@ prefactor_calibrator:
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json
 
+toil:
+  stage: integration_test
+  script:
+    - toil-cwl-runner --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH --jobStore jobstore --writeLogs logs --workDir workdir prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json
+
 flag_average:
   stage: integration_test
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH test_workflow/flag_average.cwl test_jobs/flag_average.json
-

Docker/Dockerfile

@@ -88,12 +88,19 @@ RUN mkdir /home/lofaruser/ && \
     useradd --no-log-init -r -g lofaruser lofaruser && \
     chown lofaruser:lofaruser /home/lofaruser
 
+ARG TOIL_VERSION=3.20.0
+LABEL toil.version=${TOIL_VERSION}
+
 ARG LSMTool_TAG=master
+LABEL LSMTool.version.tag=${LSMTool_TAG}
+
+ARG LoSoTo_TAG=master
+LABEL LoSoTo.version.tag=${LoSoTo_TAG}
     
 RUN apt-get update && apt-get install -y nodejs python3-pip git wsclean
-RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" git+https://github.com/DataBiosphere/toil
+RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" toil[cwl]==${TOIL_VERSION}
 
-RUN python3 -m pip install git+https://github.com/revoltek/losoto.git
+RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo"
 
 ADD L570745_uv_first10.MS.tar.xz /data
 ADD skymodels.tar.gz /data/skymodels

Docker/build_docker.sh

@@ -5,20 +5,23 @@ git_clone_or_pull () {
     REPO=$1
     BRANCH=$2
     DIR=$3
-    
-    git clone -single-branch -b "${BRANCH}" "${REPO}" "${DIR}" 2> /dev/null || git -C "${DIR}" pull
+
+    git clone --single-branch -b "${BRANCH}" "${REPO}" "${DIR}" 2> /dev/null || git -C "${DIR}" pull
 }
 
 # SOFTWARE VERSIONS
 DP3_TAG=v4.1
 IDG_TAG=0.6
 LSMTool_TAG=v1.4.2
+LoSoTo_TAG=f0398b
+TOIL_VERSION=3.20.0
+
 
 # FETCHES THE IDG
 git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg
 
 # FETCHES DP3
-git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3 
+git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3
 
 # FETCHES THE SCRIPTS ONLY
 BRANCH=production
@@ -29,4 +32,9 @@ SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]})
 
 DOCKER_PATH=$(dirname ${SCRIPT_PATH})
 
-docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG} --build-arg=DP3_TAG=${DP3_TAG} --build-arg=LSMTool_TAG=${LSMTool_TAG} -t lofareosc/prefactor-ci
+docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\
+                            --build-arg=DP3_TAG=${DP3_TAG}\
+                            --build-arg=LSMTool_TAG=${LSMTool_TAG}\
+                            --build-arg=LoSoTo_TAG=${LoSoTo_TAG}\
+                            --build-arg=TOIL_VERSION=${TOIL_VERSION}\
+                            -t lofareosc/prefactor-ci

steps/DP3.AOFlaggerStepGenerator.cwl

@@ -5,7 +5,7 @@ cwlVersion: v1.0
 id: aoflagger_step_generator
 
 inputs:
-  - id: parameters_input
+  - id: steps
     type: Any[]?
     default: []
   - id: step_name
@@ -83,9 +83,9 @@ inputs:
     default: true
     doc: Write the quality statistics?
 outputs:
-  - id: parameters_out
+  - id: augmented_steps
     outputSource:
-      - DP3_GenericStep/parameters_out
+      - DP3_GenericStep/augmented_steps
     type: Any[]
 
 steps:
@@ -95,8 +95,8 @@ steps:
         default: 'aoflagger'
       - id: step_id
         source: step_name
-      - id: parameters_input
-        source: parameters_input
+      - id: steps
+        source: steps
       - id: parameters
         valueFrom: $(inputs)
       - id: count.save
@@ -124,7 +124,7 @@ steps:
       - id: keepstatistics
         source: keepstatistics
     out:
-      - parameters_out
+      - augmented_steps
     run: ../steps/DP3.GenericStep.cwl
 requirements:
   - class: StepInputExpressionRequirement

steps/DP3.AveragerStepGenerator.cwl

@@ -5,7 +5,7 @@ cwlVersion: v1.0
 id: average_step_generator
 
 inputs:
-  - id: input_parset
+  - id: steps
     type: Any[]?
     default: []
   - id: step_name
@@ -48,9 +48,9 @@ inputs:
       <step>.freqstep
     default: 0
 outputs:
-  - id: parameters_out
+  - id: augmented_steps
     outputSource:
-      - DP3_GenericStep/parameters_out
+      - DP3_GenericStep/augmented_steps
     type: Any[]
 
 steps:
@@ -60,8 +60,8 @@ steps:
         default: 'averager'
       - id: step_id
         source: step_name
-      - id: parameters_input
-        source: input_parset
+      - id: steps
+        source: steps
       - id: parameters
         valueFrom: $(inputs)
       - id: timestep
@@ -77,7 +77,7 @@ steps:
       - id: freqresolution
         source: freqresolution
     out:
-      - parameters_out
+      - augmented_steps
     run: ../steps/DP3.GenericStep.cwl
 requirements:
   - class: StepInputExpressionRequirement

steps/DP3.DemixerStepGenerator.cwl

@@ -7,7 +7,7 @@ $namespaces:
 id: demix_step_generator
 
 inputs:
-  - id: parameters_input
+  - id: steps
     type: Any[]?
     default: []
   - id: step_name
@@ -126,16 +126,16 @@ inputs:
     type: int
     doc: Maximum number of iterations used in the LM solve
 outputs:
-- id: parameters_out
+- id: augmented_steps
   outputSource:
-    - DP3_GenericStep/parameters_out
+    - DP3_GenericStep/augmented_steps
   type: Any[]
 
 steps:
   - id: DP3_GenericStep
     in:
-    - id: parameters_input
-      source: parameters_input
+    - id: steps
+      source: steps
     - id: output_files
       default: ["instrumentmodel"]
       
@@ -182,7 +182,7 @@ steps:
     - id: maxiter
       source: maxiter
     out:
-      - parameters_out
+      - augmented_steps
     run: ../steps/DP3.GenericStep.cwl
 requirements:
   - class: StepInputExpressionRequirement

steps/DP3.GenericStep.cwl

@@ -8,14 +8,14 @@ requirements:
   - class: InlineJavascriptRequirement
 
 inputs:
-  - id: parameters_input
+  - id: steps
     type: Any[]
 
   - id: parameters
     type: Any?
 
 outputs:
-  - id: parameters_out
+  - id: augmented_steps
     type: Any[]
 expression: |-
   ${
@@ -36,9 +36,9 @@ expression: |-
 
    delete inputs.parameters.step_type;
    delete inputs.parameters.step_id;
-   delete inputs.parameters.parameters_input;
+   delete inputs.parameters.steps;
    delete inputs.parameters.parameters;
-   delete inputs.parameters.parameters_input;
+   delete inputs.parameters.steps;
 
 
    var new_step = {'step_type': step_type,
@@ -47,7 +47,7 @@ expression: |-
                    'output_files': output_files};
 
 
-   var in_parameters = inputs.parameters_input;
+   var in_parameters = inputs.steps;
 
    // Checks if the step_id is duplicate if so raised an exception
    var step_ids = []
@@ -59,5 +59,5 @@ expression: |-
 
    var out_parameters =  in_parameters == null ? [new_step] : in_parameters.concat(new_step)
 
-   return {'parameters_out': out_parameters};
+   return {'augmented_steps': out_parameters};
   }

steps/DP3.ParsetGenerator.cwl

@@ -18,7 +18,7 @@ arguments:
         true = True
         false = False
         null = None
-        steps_list = $(inputs.parameters_input)
+        steps_list = $(inputs.steps)
         steps_id_list = list(map(lambda item: item['step_id'], steps_list))
 
         for step in steps_list:
@@ -50,15 +50,15 @@ arguments:
         last_step_line = 'steps=[%s]' % ', '.join(steps_id_list)
         print(last_step_line)
 inputs:
-  - id: parameters_input
+  - id: steps
     type: Any[]?
 
 outputs:
-  - id: output_parset
+  - id: parset
     doc: Parset output file
     type: File
     outputBinding:
-      glob: output_parset
+      glob: parset
   - id: input_files
     doc: additional input files
     type: File[]
@@ -66,8 +66,8 @@ outputs:
       outputEval: |-
         ${
         var outfiles = []
-        for(var step in inputs.parameters_input){
-          step = inputs.parameters_input[step]
+        for(var step in inputs.steps){
+          step = inputs.steps[step]
           for(var parameter in step['parameters']){
             parameter = step['parameters'][parameter]
 
@@ -85,8 +85,8 @@ outputs:
       outputEval: |
         ${
         var outdirs = []
-        for(var step in inputs.parameters_input){
-          step = inputs.parameters_input[step]
+        for(var step in inputs.steps){
+          step = inputs.steps[step]
           for(var parameter in step['parameters']){
             parameter = step['parameters'][parameter]
             if (parameter != null && parameter.class=='Directory'){
@@ -104,8 +104,8 @@ outputs:
       outputEval: |
         ${
         var out_names = {}
-        for(var step_idx in inputs.parameters_input){
-          var step = inputs.parameters_input[step_idx]
+        for(var step_idx in inputs.steps){
+          var step = inputs.steps[step_idx]
           var step_id = step['step_id']
           var step_dirs = {}
           if(!step.hasOwnProperty("output_files")) continue;
@@ -129,8 +129,8 @@ outputs:
       outputEval: |-
         ${
         var out_names = {}
-        for(var step_idx in inputs.parameters_input){
-          var step = inputs.parameters_input[step_idx]
+        for(var step_idx in inputs.steps){
+          var step = inputs.steps[step_idx]
           var step_id = step['step_id']
           var step_dirs = {}
           if(!step.hasOwnProperty("output_directories")) continue;
@@ -145,4 +145,4 @@ outputs:
         return out_names
         }
 stdout:
-  output_parset
+  parset

steps/DP3.PreflaggerStepGenerator.cwl

@@ -5,7 +5,7 @@ cwlVersion: v1.0
 id: preflag_step_generator
 
 inputs:
-  - id: input_parset
+  - id: steps
     type: Any[]?
     default: []
   - id: step_name
@@ -169,9 +169,9 @@ inputs:
     type: float
     doc: Correlation data with imaginary complex part > imagmax will match.
 outputs:
-- id: parameters_out
+- id: augmented_steps
   outputSource:
-    - DP3_GenericStep/parameters_out
+    - DP3_GenericStep/augmented_steps
   type: Any[]
 
 steps:
@@ -181,8 +181,8 @@ steps:
         default: 'preflagger'
       - id: step_id
         source: step_name
-      - id: parameters_input
-        source: input_parset
+      - id: steps
+        source: steps
       - id: parameters
         valueFrom: $(inputs)
       - id: count.save
@@ -238,7 +238,7 @@ steps:
       - id: imagmax
         source: imagmax
     out:
-      - parameters_out
+      - augmented_steps
     run: ../steps/DP3.GenericStep.cwl
 
 requirements:

steps/DP3.ParsetSelector.cwl → steps/DP3.Selector.cwl

 class: ExpressionTool
 cwlVersion: v1.0
-id: parsetselector
+id: selector
 inputs:
   - id: select_a
     type: boolean
-    doc: 'if true selects parset_a, files_a if false selects parset_b files_b'
-  - id: a_arguments
+    doc: 'if true selects a_steps if false selects b_steps'
+  - id: a_steps
     type: Any[]
     default: []
-  - id: b_arguments
+  - id: b_steps
     type: Any[]
     default: []
 outputs:
-  - id: output_arguments
+  - id: selected_steps
     type: Any
 expression: |
   $(
-    {'output_arguments': inputs.selector ? inputs.a_arguments : inputs.b_arguments}
+    {'selected_steps': inputs.selector ? inputs.a_steps : inputs.b_steps}
   )
-label: ParsetSelector
+label: Selector
 
 requirements:
   - class: InlineJavascriptRequirement

subworkflow/ndppp_prep_cal.cwl

@@ -7,16 +7,16 @@ $namespaces:
 inputs:
   - id: baselines_to_flag
     type: 'string[]'
-    'sbg:x': 707.7924194335938
-    'sbg:y': -366.86871337890625
+    'sbg:x': 732.2234497070312
+    'sbg:y': 79.11939239501953
   - id: elevation_to_flag
     type: string
-    'sbg:x': 847
-    'sbg:y': -501.7469482421875
+    'sbg:x': 815.8416137695312
+    'sbg:y': -281.40777587890625
   - id: min_amplitude_to_flag
     type: float
-    'sbg:x': 1129.3828125
-    'sbg:y': 338.9372863769531
+    'sbg:x': 964.3058471679688
+    'sbg:y': 191.110595703125
   - id: memoryperc
     type: int
     'sbg:x': 136.78256225585938
@@ -31,8 +31,8 @@ inputs:
     'sbg:y': -298.45196533203125
   - id: msin
     type: Directory?
-    'sbg:x': 2095.969482421875
-    'sbg:y': 563.7855224609375
+    'sbg:x': 2129.6337890625
+    'sbg:y': 149.5590362548828
 outputs:
   - id: msout
     outputSource:
@@ -49,76 +49,80 @@ outputs:
 steps:
   - id: flagbaseline
     in:
-      - id: input_parset
+      - id: steps
         source:
-          - process_raw_data_select/output_arguments
+          - process_raw_data_select/selected_steps
       - id: step_name
         default: flagbaseline
       - id: baseline
         source:
           - baselines_to_flag
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.PreflaggerStepGenerator.cwl
-    'sbg:x': 1160.802978515625
-    'sbg:y': -95.15213775634766
+    'sbg:x': 955.2450561523438
+    'sbg:y': -75.71149444580078
   - id: flagelev
     in:
-      - id: input_parset
+      - id: steps
         source:
-          - flagbaseline/parameters_out
+          - flagbaseline/augmented_steps
       - id: step_name
         default: flagelev
       - id: elevation
         source: elevation_to_flag
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.PreflaggerStepGenerator.cwl
-    'sbg:x': 1459.2421875
-    'sbg:y': -83.970458984375
+    'sbg:x': 1187.4771728515625
+    'sbg:y': -83
   - id: process_raw_data_select
     in:
       - id: select_a
         source: raw_data
-      - id: a_arguments
+      - id: a_steps
         source:
-          - aoflag/parameters_out
+          - aoflag/augmented_steps
+      - id: b_steps
+        default: []
     out:
-      - id: output_arguments
-    run: ../steps/DP3.ParsetSelector.cwl
-    label: ParsetSelector
+      - id: selected_steps
+    run: ../steps/DP3.Selector.cwl
+    label: Selector
     'sbg:x': 621.4892578125
     'sbg:y': -71.31844329833984
   - id: flagamp
     in:
-      - id: input_parset
+      - id: steps
         source:
-          - flagelev/parameters_out
+          - flagelev/augmented_steps
       - id: step_name
         default: flagamp
       - id: amplmin
         source: min_amplitude_to_flag
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.PreflaggerStepGenerator.cwl
-    'sbg:x': 1659.302001953125
-    'sbg:y': -23
+    'sbg:x': 1385.993408203125
+    'sbg:y': -42.28850555419922
   - id: flagedge
     in:
       - id: step_name
         default: flagedge
+      - id: steps
+        default: []
       - id: chan
         default: '[0..nchan/32-1,31*nchan/32..nchan-1]'
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.PreflaggerStepGenerator.cwl
-    'sbg:x': 53.59375762939453
-    'sbg:y': -119.96077728271484
+    'sbg:x': 122.25408935546875
+    'sbg:y': -54.06163024902344
   - id: aoflag
     in:
-      - id: parameters_input
+      - id: steps
         source:
-          - flagedge/parameters_out
+          - flagedge/augmented_steps
       - id: step_name
         default: aoflag
       - id: memoryperc
@@ -126,54 +130,54 @@ steps:
       - id: keepstatistics
         default: false
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.AOFlaggerStepGenerator.cwl
-    'sbg:x': 357.934326171875
-    'sbg:y': -129.337890625
+    'sbg:x': 369.9876708984375
+    'sbg:y': -38.086280822753906
   - id: demix_select
     in:
       - id: select_a
         source: demix
-      - id: a_arguments
+      - id: a_steps
         source:
-          - demixstepgenerator/parameters_out
-      - id: b_arguments
+          - demixstepgenerator/augmented_steps
+      - id: b_steps
         source:
-          - flagamp/parameters_out
+          - flagamp/augmented_steps
     out:
-      - id: output_arguments
-    run: ../steps/DP3.ParsetSelector.cwl
+      - id: selected_steps
+    run: ../steps/DP3.Selector.cwl
     label: demix?
     'sbg:x': 1985.5047607421875
     'sbg:y': -44.651588439941406
   - id: demixstepgenerator
     in:
-      - id: parameters_input
+      - id: steps
         source:
-          - flagamp/parameters_out
+          - flagamp/augmented_steps
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.DemixerStepGenerator.cwl
-    'sbg:x': 1828.623046875
-    'sbg:y': 156.3474578857422
+    'sbg:x': 1676.6593017578125
+    'sbg:y': 90.7158203125
   - id: parsetgenerator
     in:
-      - id: parameters_input
+      - id: steps
         source:
-          - demix_select/output_arguments
+          - demix_select/selected_steps
     out:
-      - id: output_parset
+      - id: parset
       - id: input_files
       - id: input_directories
       - id: output_file_names
       - id: output_directory_names
     run: ../steps/DP3.ParsetGenerator.cwl
-    'sbg:x': 2228.349365234375
-    'sbg:y': -53.74734115600586
+    'sbg:x': 2228
+    'sbg:y': -40.012325286865234
   - id: dppp
     in:
       - id: parset
-        source: parsetgenerator/output_parset
+        source: parsetgenerator/parset
       - id: msin
         source: msin
       - id: msout_name
@@ -215,3 +219,4 @@ requirements:
   - class: SubworkflowFeatureRequirement
   - class: StepInputExpressionRequirement
   - class: InlineJavascriptRequirement
+

test_jobs/flag_average.json

 {
   "msin": {
-    "class":"File",
+    "class": "Directory",
     "path": "/data/L570745_SB001_uv_first10.MS"
   }
 }

test_workflow/flag_average.cwl

@@ -20,27 +20,27 @@ steps:
   - id: aoflagger_step_generator
     in: []
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.AOFlaggerStepGenerator.cwl
     'sbg:x': -1357.640625
     'sbg:y': -729.5
   - id: average_step_generator
     in:
-      - id: input_parset
+      - id: steps
         source:
-          - aoflagger_step_generator/parameters_out
+          - aoflagger_step_generator/augmented_steps
     out:
-      - id: parameters_out
+      - id: augmented_steps
     run: ../steps/DP3.AveragerStepGenerator.cwl
     'sbg:x': -1125
     'sbg:y': -729
   - id: parset_generator
     in:
-      - id: parameters_input
+      - id: steps
         source:
-          - average_step_generator/parameters_out
+          - average_step_generator/augmented_steps
     out:
-      - id: output_parset
+      - id: parset
       - id: input_files
       - id: input_directories
       - id: output_file_names
@@ -51,7 +51,7 @@ steps:
   - id: dppp
     in:
       - id: parset
-        source: parset_generator/output_parset
+        source: parset_generator/parset
       - id: msin
         source: msin
       - id: secondary_files