diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aac584fffbf281180f7080ddd81bd644d672ccd8..e63379c1411f4de37ff7718b10fd1abbaa21e3f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,6 @@ before_script: # make toil workdir - mkdir workdir - mkdir logs - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY after_script: - echo "All done" @@ -30,10 +29,12 @@ build_docker: push_docker_tags: image: docker:latest + allow_failure: true stage: push_docker only: - tags script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME - docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME @@ -46,8 +47,6 @@ push_docker_master: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker push lofareosc/prefactor-ci:latest - - validate_scripts: stage: validate_cwl_scripts script: @@ -126,6 +125,12 @@ NDPPP: script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json +DP3.Execute: + stage: test_steps + allow_failure: true + script: + - cwltool --no-container --preserve-enviroment LD_LIBRARY_PATH --preserve-enviroment PATH steps/DP3.Execute.cwl test_jobs/dp3_execute.json + h5parm_collector: stage: test_steps allow_failure: true diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 44bcd9373b7bebd58f58fab823e4ea5ed518c590..3790b335be13cd417ff858ec0cd30de3c8f337c9 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -65,7 +65,7 @@ RUN cd /src/Dysco/ && \ make install -j2 #---------------------------------------------- -FROM kernsuite/base:4 as runner +FROM kernsuite/base:5 as runner COPY --from=builder /usr/local/ /usr/local @@ -85,13 +85,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libboost-system1.65.1 \ libboost-signals1.65.1 \ libboost-test1.65.1 \ - libboost-python1.65-dev \ - libcfitsio5 \ + libboost-python1.65-dev \ + libstationresponse2 \ + libcasa-* \ + libcfitsio5 \ libgtkmm-3.0 \ libfftw3-3 \ libhdf5-cpp-100 \ libpng16-16 \ - libstationresponse0 \ libxml2 \ python3.7 \ python3-numpy \ diff --git a/steps/DP3.Execute.cwl b/steps/DP3.Execute.cwl new file mode 100644 index 0000000000000000000000000000000000000000..bf15ea45d9ac61ab6d23521605ea0955324e7e45 --- /dev/null +++ b/steps/DP3.Execute.cwl @@ -0,0 +1,79 @@ +class: Workflow +cwlVersion: v1.0 +id: dp3_execute +label: DP3.Execute +$namespaces: + sbg: 'https://www.sevenbridges.com/' +inputs: + - id: msout_name + type: string + 'sbg:x': -425.39886474609375 + 'sbg:y': 127 + - id: msin + type: Directory + 'sbg:x': -254.39886474609375 + 'sbg:y': 242 + - id: steps + type: 'Any[]?' + 'sbg:x': -518.3988647460938 + 'sbg:y': -88 +outputs: + - id: secondary_output_files + outputSource: + - dppp/secondary_output_files + type: Any + 'sbg:x': 213.60113525390625 + 'sbg:y': -184 + - id: secondary_output_directories + outputSource: + - dppp/secondary_output_directories + type: Any + 'sbg:x': 248.60113525390625 + 'sbg:y': -47 + - id: msout + outputSource: + - dppp/msout + type: Directory + 'sbg:x': 224.60113525390625 + 'sbg:y': 130 +steps: + - id: generic_step + in: + - id: steps + source: + - steps + out: + - id: parset + - id: input_files + - id: input_directories + - id: output_file_names + - id: output_directory_names + run: ./DP3.ParsetGenerator.cwl + 'sbg:x': -294 + 'sbg:y': -84 + - id: dppp + in: + - id: parset + source: generic_step/parset + - id: msin + source: msin + - id: msout_name + source: msout_name + - id: secondary_files + source: + - generic_step/input_files + - id: secondary_directories + source: + - generic_step/input_directories + - id: output_file_names + source: generic_step/output_file_names + - id: output_directory_names + source: generic_step/output_directory_names + out: + - id: msout + - id: secondary_output_files + - id: secondary_output_directories + run: ./DPPP.cwl + 'sbg:x': 26 + 'sbg:y': -46 +requirements: [] diff --git a/test_jobs/dp3_execute.json b/test_jobs/dp3_execute.json new file mode 100644 index 0000000000000000000000000000000000000000..f21dff5b5c3fc6f6c7bd58edc1d6aee45ce5446b --- /dev/null +++ b/test_jobs/dp3_execute.json @@ -0,0 +1,19 @@ +{ + "msin": {"class": "Directory", "path":"/data/L570745_SB003_uv_first10.MS"}, + "steps": [ + { + "step_type": "averager", + "step_id": "average", + "parameters": { + "timestep": 1, + "freqstep": 1, + "minpoints": 0, + "minperc": 0, + "timeresolution": 0, + "freqresolution": 0 + }, + "output_files": {} + } + ], + "msout_name": "TEST" +}