diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 499eb4e827165079abd99a9c1bd0b3e3eb2a9f40..7d8b7a61cfaa9ded136ee1a0add495fbc08b53c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,15 +9,10 @@ stages: - test_steps - integration_test - before_script: -# - python3 -V # Print out python version for debugging -# - apt-get update && apt-get install -y nodejs python3-pip -# - python3 -m pip install cwltool cwl-runner -# - cd test_data -# - tar -xf ${TEST_DATASET_NAME} -# - cd $CI_PROJECT_DIR -# - source /opt/bashrc + # make toil workdir + - mkdir workdir + - mkdir logs after_script: - echo "All done" @@ -50,7 +45,7 @@ parset_selector: stage: test_steps allow_failure: true script: - - cwl-runner steps/DP3.ParsetSelector.cwl test_jobs/parset_selector.json + - cwl-runner steps/DP3.Selector.cwl test_jobs/selector.json file_selector: stage: test_steps @@ -117,8 +112,12 @@ prefactor_calibrator: script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json +toil: + stage: integration_test + script: + - toil-cwl-runner --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH --jobStore jobstore --writeLogs logs --workDir workdir prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json + flag_average: stage: integration_test script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH test_workflow/flag_average.cwl test_jobs/flag_average.json - diff --git a/Docker/Dockerfile b/Docker/Dockerfile index e5537483a8092fff03238a29a3287a75a7a1477e..e72582ecc14d3b22def4f77c2c8f95fb44c4f4c7 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -88,12 +88,19 @@ RUN mkdir /home/lofaruser/ && \ useradd --no-log-init -r -g lofaruser lofaruser && \ chown lofaruser:lofaruser /home/lofaruser +ARG TOIL_VERSION=3.20.0 +LABEL toil.version=${TOIL_VERSION} + ARG LSMTool_TAG=master +LABEL LSMTool.version.tag=${LSMTool_TAG} + +ARG LoSoTo_TAG=master +LABEL LoSoTo.version.tag=${LoSoTo_TAG} RUN apt-get update && apt-get install -y nodejs python3-pip git wsclean -RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" git+https://github.com/DataBiosphere/toil +RUN python3 -m pip install cwltool cwl-runner -e "git://github.com/darafferty/LSMTool.git@${LSMTool_TAG}#egg=LSMTool" toil[cwl]==${TOIL_VERSION} -RUN python3 -m pip install git+https://github.com/revoltek/losoto.git +RUN python3 -m pip install -e "git+https://github.com/revoltek/losoto.git@${LoSoTo_TAG}#egg=LoSoTo" ADD L570745_uv_first10.MS.tar.xz /data ADD skymodels.tar.gz /data/skymodels diff --git a/Docker/build_docker.sh b/Docker/build_docker.sh index 33efd28ebad48c96dcd4baddce0c7a881e14a1e3..e39c94963ca43d00b07b69aa10c0f4095c144eaf 100755 --- a/Docker/build_docker.sh +++ b/Docker/build_docker.sh @@ -5,20 +5,23 @@ git_clone_or_pull () { REPO=$1 BRANCH=$2 DIR=$3 - - git clone -single-branch -b "${BRANCH}" "${REPO}" "${DIR}" 2> /dev/null || git -C "${DIR}" pull + + git clone --single-branch -b "${BRANCH}" "${REPO}" "${DIR}" 2> /dev/null || git -C "${DIR}" pull } # SOFTWARE VERSIONS DP3_TAG=v4.1 IDG_TAG=0.6 LSMTool_TAG=v1.4.2 +LoSoTo_TAG=f0398b +TOIL_VERSION=3.20.0 + # FETCHES THE IDG git_clone_or_pull https://gitlab.com/astron-idg/idg/ ${IDG_TAG} idg # FETCHES DP3 -git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3 +git_clone_or_pull https://github.com/lofar-astron/DP3 ${DP3_TAG} DP3 # FETCHES THE SCRIPTS ONLY BRANCH=production @@ -29,4 +32,9 @@ SCRIPT_PATH=$(realpath ${BASH_SOURCE[0]}) DOCKER_PATH=$(dirname ${SCRIPT_PATH}) -docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG} --build-arg=DP3_TAG=${DP3_TAG} --build-arg=LSMTool_TAG=${LSMTool_TAG} -t lofareosc/prefactor-ci +docker build ${DOCKER_PATH} --build-arg=IDG_TAG=${IDG_TAG}\ + --build-arg=DP3_TAG=${DP3_TAG}\ + --build-arg=LSMTool_TAG=${LSMTool_TAG}\ + --build-arg=LoSoTo_TAG=${LoSoTo_TAG}\ + --build-arg=TOIL_VERSION=${TOIL_VERSION}\ + -t lofareosc/prefactor-ci diff --git a/steps/DP3.AOFlaggerStepGenerator.cwl b/steps/DP3.AOFlaggerStepGenerator.cwl index 5b7b6ce930220030f150b3762bcd97f50aa7aef8..2a70f602771d7e5e669ba46ee395550807bb379d 100755 --- a/steps/DP3.AOFlaggerStepGenerator.cwl +++ b/steps/DP3.AOFlaggerStepGenerator.cwl @@ -5,7 +5,7 @@ cwlVersion: v1.0 id: aoflagger_step_generator inputs: - - id: parameters_input + - id: steps type: Any[]? default: [] - id: step_name @@ -83,9 +83,9 @@ inputs: default: true doc: Write the quality statistics? outputs: - - id: parameters_out + - id: augmented_steps outputSource: - - DP3_GenericStep/parameters_out + - DP3_GenericStep/augmented_steps type: Any[] steps: @@ -95,8 +95,8 @@ steps: default: 'aoflagger' - id: step_id source: step_name - - id: parameters_input - source: parameters_input + - id: steps + source: steps - id: parameters valueFrom: $(inputs) - id: count.save @@ -124,7 +124,7 @@ steps: - id: keepstatistics source: keepstatistics out: - - parameters_out + - augmented_steps run: ../steps/DP3.GenericStep.cwl requirements: - class: StepInputExpressionRequirement diff --git a/steps/DP3.AveragerStepGenerator.cwl b/steps/DP3.AveragerStepGenerator.cwl index c75232c69bc9ac715387ad747bdb322b84b19c46..d5424919111a6c38f65053bfec5f2e299a98c076 100755 --- a/steps/DP3.AveragerStepGenerator.cwl +++ b/steps/DP3.AveragerStepGenerator.cwl @@ -5,7 +5,7 @@ cwlVersion: v1.0 id: average_step_generator inputs: - - id: input_parset + - id: steps type: Any[]? default: [] - id: step_name @@ -48,9 +48,9 @@ inputs: <step>.freqstep default: 0 outputs: - - id: parameters_out + - id: augmented_steps outputSource: - - DP3_GenericStep/parameters_out + - DP3_GenericStep/augmented_steps type: Any[] steps: @@ -60,8 +60,8 @@ steps: default: 'averager' - id: step_id source: step_name - - id: parameters_input - source: input_parset + - id: steps + source: steps - id: parameters valueFrom: $(inputs) - id: timestep @@ -77,7 +77,7 @@ steps: - id: freqresolution source: freqresolution out: - - parameters_out + - augmented_steps run: ../steps/DP3.GenericStep.cwl requirements: - class: StepInputExpressionRequirement diff --git a/steps/DP3.DemixerStepGenerator.cwl b/steps/DP3.DemixerStepGenerator.cwl index c11ced07db6f7bfe7a71ffbaf778380e184ff824..caaa14ce8004716f969fb9b88f8589ba2a0167b9 100755 --- a/steps/DP3.DemixerStepGenerator.cwl +++ b/steps/DP3.DemixerStepGenerator.cwl @@ -7,7 +7,7 @@ $namespaces: id: demix_step_generator inputs: - - id: parameters_input + - id: steps type: Any[]? default: [] - id: step_name @@ -126,16 +126,16 @@ inputs: type: int doc: Maximum number of iterations used in the LM solve outputs: -- id: parameters_out +- id: augmented_steps outputSource: - - DP3_GenericStep/parameters_out + - DP3_GenericStep/augmented_steps type: Any[] steps: - id: DP3_GenericStep in: - - id: parameters_input - source: parameters_input + - id: steps + source: steps - id: output_files default: ["instrumentmodel"] @@ -182,7 +182,7 @@ steps: - id: maxiter source: maxiter out: - - parameters_out + - augmented_steps run: ../steps/DP3.GenericStep.cwl requirements: - class: StepInputExpressionRequirement diff --git a/steps/DP3.GenericStep.cwl b/steps/DP3.GenericStep.cwl index 28934bb7f2c645bd59a4226570e2cdf1bb7fd67f..38829da42ed0deaac51bcf92c7c105e9139d4108 100755 --- a/steps/DP3.GenericStep.cwl +++ b/steps/DP3.GenericStep.cwl @@ -8,14 +8,14 @@ requirements: - class: InlineJavascriptRequirement inputs: - - id: parameters_input + - id: steps type: Any[] - id: parameters type: Any? outputs: - - id: parameters_out + - id: augmented_steps type: Any[] expression: |- ${ @@ -36,9 +36,9 @@ expression: |- delete inputs.parameters.step_type; delete inputs.parameters.step_id; - delete inputs.parameters.parameters_input; + delete inputs.parameters.steps; delete inputs.parameters.parameters; - delete inputs.parameters.parameters_input; + delete inputs.parameters.steps; var new_step = {'step_type': step_type, @@ -47,7 +47,7 @@ expression: |- 'output_files': output_files}; - var in_parameters = inputs.parameters_input; + var in_parameters = inputs.steps; // Checks if the step_id is duplicate if so raised an exception var step_ids = [] @@ -59,5 +59,5 @@ expression: |- var out_parameters = in_parameters == null ? [new_step] : in_parameters.concat(new_step) - return {'parameters_out': out_parameters}; + return {'augmented_steps': out_parameters}; } diff --git a/steps/DP3.ParsetGenerator.cwl b/steps/DP3.ParsetGenerator.cwl index c810ac8a343b59bd8b45f285b14051c7acb0e3c3..f5837e9a3528135180143bee7fb406ef736b1efb 100644 --- a/steps/DP3.ParsetGenerator.cwl +++ b/steps/DP3.ParsetGenerator.cwl @@ -18,7 +18,7 @@ arguments: true = True false = False null = None - steps_list = $(inputs.parameters_input) + steps_list = $(inputs.steps) steps_id_list = list(map(lambda item: item['step_id'], steps_list)) for step in steps_list: @@ -50,15 +50,15 @@ arguments: last_step_line = 'steps=[%s]' % ', '.join(steps_id_list) print(last_step_line) inputs: - - id: parameters_input + - id: steps type: Any[]? outputs: - - id: output_parset + - id: parset doc: Parset output file type: File outputBinding: - glob: output_parset + glob: parset - id: input_files doc: additional input files type: File[] @@ -66,8 +66,8 @@ outputs: outputEval: |- ${ var outfiles = [] - for(var step in inputs.parameters_input){ - step = inputs.parameters_input[step] + for(var step in inputs.steps){ + step = inputs.steps[step] for(var parameter in step['parameters']){ parameter = step['parameters'][parameter] @@ -85,8 +85,8 @@ outputs: outputEval: | ${ var outdirs = [] - for(var step in inputs.parameters_input){ - step = inputs.parameters_input[step] + for(var step in inputs.steps){ + step = inputs.steps[step] for(var parameter in step['parameters']){ parameter = step['parameters'][parameter] if (parameter != null && parameter.class=='Directory'){ @@ -104,8 +104,8 @@ outputs: outputEval: | ${ var out_names = {} - for(var step_idx in inputs.parameters_input){ - var step = inputs.parameters_input[step_idx] + for(var step_idx in inputs.steps){ + var step = inputs.steps[step_idx] var step_id = step['step_id'] var step_dirs = {} if(!step.hasOwnProperty("output_files")) continue; @@ -129,8 +129,8 @@ outputs: outputEval: |- ${ var out_names = {} - for(var step_idx in inputs.parameters_input){ - var step = inputs.parameters_input[step_idx] + for(var step_idx in inputs.steps){ + var step = inputs.steps[step_idx] var step_id = step['step_id'] var step_dirs = {} if(!step.hasOwnProperty("output_directories")) continue; @@ -145,4 +145,4 @@ outputs: return out_names } stdout: - output_parset + parset diff --git a/steps/DP3.ParsetSelector.cwl b/steps/DP3.ParsetSelector.cwl deleted file mode 100755 index f58d1420436a9f470c6d3106d31f260faee29da9..0000000000000000000000000000000000000000 --- a/steps/DP3.ParsetSelector.cwl +++ /dev/null @@ -1,24 +0,0 @@ -class: ExpressionTool -cwlVersion: v1.0 -id: parsetselector -inputs: - - id: select_a - type: boolean - doc: 'if true selects parset_a, files_a if false selects parset_b files_b' - - id: a_arguments - type: Any[] - default: [] - - id: b_arguments - type: Any[] - default: [] -outputs: - - id: output_arguments - type: Any -expression: | - $( - {'output_arguments': inputs.selector ? inputs.a_arguments : inputs.b_arguments} - ) -label: ParsetSelector - -requirements: - - class: InlineJavascriptRequirement diff --git a/steps/DP3.PreflaggerStepGenerator.cwl b/steps/DP3.PreflaggerStepGenerator.cwl index f7f5965469164912c0ac7a68cc772b84917ec505..7b29b255eb61e6d2d4e982b255f4adc0d2cb514a 100755 --- a/steps/DP3.PreflaggerStepGenerator.cwl +++ b/steps/DP3.PreflaggerStepGenerator.cwl @@ -5,7 +5,7 @@ cwlVersion: v1.0 id: preflag_step_generator inputs: - - id: input_parset + - id: steps type: Any[]? default: [] - id: step_name @@ -169,9 +169,9 @@ inputs: type: float doc: Correlation data with imaginary complex part > imagmax will match. outputs: -- id: parameters_out +- id: augmented_steps outputSource: - - DP3_GenericStep/parameters_out + - DP3_GenericStep/augmented_steps type: Any[] steps: @@ -181,8 +181,8 @@ steps: default: 'preflagger' - id: step_id source: step_name - - id: parameters_input - source: input_parset + - id: steps + source: steps - id: parameters valueFrom: $(inputs) - id: count.save @@ -238,7 +238,7 @@ steps: - id: imagmax source: imagmax out: - - parameters_out + - augmented_steps run: ../steps/DP3.GenericStep.cwl requirements: diff --git a/steps/DP3.Selector.cwl b/steps/DP3.Selector.cwl new file mode 100755 index 0000000000000000000000000000000000000000..28c5d866b55fc365774eeb593ebe5d807309ff8d --- /dev/null +++ b/steps/DP3.Selector.cwl @@ -0,0 +1,24 @@ +class: ExpressionTool +cwlVersion: v1.0 +id: selector +inputs: + - id: select_a + type: boolean + doc: 'if true selects a_steps if false selects b_steps' + - id: a_steps + type: Any[] + default: [] + - id: b_steps + type: Any[] + default: [] +outputs: + - id: selected_steps + type: Any +expression: | + $( + {'selected_steps': inputs.selector ? inputs.a_steps : inputs.b_steps} + ) +label: Selector + +requirements: + - class: InlineJavascriptRequirement diff --git a/subworkflow/ndppp_prep_cal.cwl b/subworkflow/ndppp_prep_cal.cwl index 8226f13af15327ea024f44426c2b93494dda5061..b495d85a1a3491b70e28d8e90dee66e70581674b 100644 --- a/subworkflow/ndppp_prep_cal.cwl +++ b/subworkflow/ndppp_prep_cal.cwl @@ -7,16 +7,16 @@ $namespaces: inputs: - id: baselines_to_flag type: 'string[]' - 'sbg:x': 707.7924194335938 - 'sbg:y': -366.86871337890625 + 'sbg:x': 732.2234497070312 + 'sbg:y': 79.11939239501953 - id: elevation_to_flag type: string - 'sbg:x': 847 - 'sbg:y': -501.7469482421875 + 'sbg:x': 815.8416137695312 + 'sbg:y': -281.40777587890625 - id: min_amplitude_to_flag type: float - 'sbg:x': 1129.3828125 - 'sbg:y': 338.9372863769531 + 'sbg:x': 964.3058471679688 + 'sbg:y': 191.110595703125 - id: memoryperc type: int 'sbg:x': 136.78256225585938 @@ -31,8 +31,8 @@ inputs: 'sbg:y': -298.45196533203125 - id: msin type: Directory? - 'sbg:x': 2095.969482421875 - 'sbg:y': 563.7855224609375 + 'sbg:x': 2129.6337890625 + 'sbg:y': 149.5590362548828 outputs: - id: msout outputSource: @@ -49,76 +49,80 @@ outputs: steps: - id: flagbaseline in: - - id: input_parset + - id: steps source: - - process_raw_data_select/output_arguments + - process_raw_data_select/selected_steps - id: step_name default: flagbaseline - id: baseline source: - baselines_to_flag out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.PreflaggerStepGenerator.cwl - 'sbg:x': 1160.802978515625 - 'sbg:y': -95.15213775634766 + 'sbg:x': 955.2450561523438 + 'sbg:y': -75.71149444580078 - id: flagelev in: - - id: input_parset + - id: steps source: - - flagbaseline/parameters_out + - flagbaseline/augmented_steps - id: step_name default: flagelev - id: elevation source: elevation_to_flag out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.PreflaggerStepGenerator.cwl - 'sbg:x': 1459.2421875 - 'sbg:y': -83.970458984375 + 'sbg:x': 1187.4771728515625 + 'sbg:y': -83 - id: process_raw_data_select in: - id: select_a source: raw_data - - id: a_arguments + - id: a_steps source: - - aoflag/parameters_out + - aoflag/augmented_steps + - id: b_steps + default: [] out: - - id: output_arguments - run: ../steps/DP3.ParsetSelector.cwl - label: ParsetSelector + - id: selected_steps + run: ../steps/DP3.Selector.cwl + label: Selector 'sbg:x': 621.4892578125 'sbg:y': -71.31844329833984 - id: flagamp in: - - id: input_parset + - id: steps source: - - flagelev/parameters_out + - flagelev/augmented_steps - id: step_name default: flagamp - id: amplmin source: min_amplitude_to_flag out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.PreflaggerStepGenerator.cwl - 'sbg:x': 1659.302001953125 - 'sbg:y': -23 + 'sbg:x': 1385.993408203125 + 'sbg:y': -42.28850555419922 - id: flagedge in: - id: step_name default: flagedge + - id: steps + default: [] - id: chan default: '[0..nchan/32-1,31*nchan/32..nchan-1]' out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.PreflaggerStepGenerator.cwl - 'sbg:x': 53.59375762939453 - 'sbg:y': -119.96077728271484 + 'sbg:x': 122.25408935546875 + 'sbg:y': -54.06163024902344 - id: aoflag in: - - id: parameters_input + - id: steps source: - - flagedge/parameters_out + - flagedge/augmented_steps - id: step_name default: aoflag - id: memoryperc @@ -126,54 +130,54 @@ steps: - id: keepstatistics default: false out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.AOFlaggerStepGenerator.cwl - 'sbg:x': 357.934326171875 - 'sbg:y': -129.337890625 + 'sbg:x': 369.9876708984375 + 'sbg:y': -38.086280822753906 - id: demix_select in: - id: select_a source: demix - - id: a_arguments + - id: a_steps source: - - demixstepgenerator/parameters_out - - id: b_arguments + - demixstepgenerator/augmented_steps + - id: b_steps source: - - flagamp/parameters_out + - flagamp/augmented_steps out: - - id: output_arguments - run: ../steps/DP3.ParsetSelector.cwl + - id: selected_steps + run: ../steps/DP3.Selector.cwl label: demix? 'sbg:x': 1985.5047607421875 'sbg:y': -44.651588439941406 - id: demixstepgenerator in: - - id: parameters_input + - id: steps source: - - flagamp/parameters_out + - flagamp/augmented_steps out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.DemixerStepGenerator.cwl - 'sbg:x': 1828.623046875 - 'sbg:y': 156.3474578857422 + 'sbg:x': 1676.6593017578125 + 'sbg:y': 90.7158203125 - id: parsetgenerator in: - - id: parameters_input + - id: steps source: - - demix_select/output_arguments + - demix_select/selected_steps out: - - id: output_parset + - id: parset - id: input_files - id: input_directories - id: output_file_names - id: output_directory_names run: ../steps/DP3.ParsetGenerator.cwl - 'sbg:x': 2228.349365234375 - 'sbg:y': -53.74734115600586 + 'sbg:x': 2228 + 'sbg:y': -40.012325286865234 - id: dppp in: - id: parset - source: parsetgenerator/output_parset + source: parsetgenerator/parset - id: msin source: msin - id: msout_name @@ -215,3 +219,4 @@ requirements: - class: SubworkflowFeatureRequirement - class: StepInputExpressionRequirement - class: InlineJavascriptRequirement + diff --git a/test_jobs/flag_average.json b/test_jobs/flag_average.json index cb33836d9da6570fb70adc9d023935106d546d89..36154f55330703582bc2a83c16cf666d27560148 100644 --- a/test_jobs/flag_average.json +++ b/test_jobs/flag_average.json @@ -1,6 +1,6 @@ { "msin": { - "class":"File", + "class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS" } } diff --git a/test_jobs/parset_selector.json b/test_jobs/selector.json similarity index 100% rename from test_jobs/parset_selector.json rename to test_jobs/selector.json diff --git a/test_workflow/flag_average.cwl b/test_workflow/flag_average.cwl index 09a0eb5fa16a8a70abaa2bc5eca0c057ddc77746..904477766ee3ca954245f4e394077e409a2e1188 100644 --- a/test_workflow/flag_average.cwl +++ b/test_workflow/flag_average.cwl @@ -20,27 +20,27 @@ steps: - id: aoflagger_step_generator in: [] out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.AOFlaggerStepGenerator.cwl 'sbg:x': -1357.640625 'sbg:y': -729.5 - id: average_step_generator in: - - id: input_parset + - id: steps source: - - aoflagger_step_generator/parameters_out + - aoflagger_step_generator/augmented_steps out: - - id: parameters_out + - id: augmented_steps run: ../steps/DP3.AveragerStepGenerator.cwl 'sbg:x': -1125 'sbg:y': -729 - id: parset_generator in: - - id: parameters_input + - id: steps source: - - average_step_generator/parameters_out + - average_step_generator/augmented_steps out: - - id: output_parset + - id: parset - id: input_files - id: input_directories - id: output_file_names @@ -51,7 +51,7 @@ steps: - id: dppp in: - id: parset - source: parset_generator/output_parset + source: parset_generator/parset - id: msin source: msin - id: secondary_files