diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f9c06ad3719097e8d1241a4682d7db5ed892670..f907828a949678a44bc34d55ee93583af40d3324 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,18 @@ stages: build-base: stage: prepare + image: docker:stable + services: + - docker:dind script: - - docker build --tag dppp_base:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_base . + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA -f ./docker/ubuntu_20_04_base . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA build-no-idg: stage: build needs: ["build-base"] - image: dppp_base:${CI_COMMIT_SHORT_SHA} + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA before_script: - rm -r /usr/lib/cmake/*idg* - rm -r /usr/lib/cmake/*IDGAPITargets* @@ -24,11 +29,14 @@ build-no-idg: - cd build - cmake -G Ninja .. - ninja + only: + variables: + - $CI_SERVER_HOST == "git.astron.nl" build-debug: stage: build needs: ["build-base"] - image: dppp_base:${CI_COMMIT_SHORT_SHA} + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA script: - mkdir build && cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-coverage" -DCMAKE_EXE_LINKER_FLAGS="-coverage" .. @@ -40,12 +48,16 @@ build-debug: build-lofar-release: stage: build + image: docker:stable + services: + - docker:dind needs: ["build-base"] script: - - docker build --build-arg BASE_TAG=${CI_COMMIT_SHORT_SHA} --tag dppp_lofar:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_lofar . + - docker build --build-arg BASE_TAG=$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA --tag dppp_lofar:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_lofar . build-doc: stage: build + image: docker:stable needs: [] before_script: - apk update @@ -59,11 +71,14 @@ build-doc: artifacts: # Only for master the docs are published; for branches it may be useful to browse the artifacts paths: - build/docs + only: + variables: + - $CI_SERVER_HOST == "git.astron.nl" linting: stage: linting needs: ["build-base"] - image: dppp_base:${CI_COMMIT_SHORT_SHA} + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA before_script: - pip3 install clang-format==9.0.0 script: @@ -72,18 +87,18 @@ linting: unit-test: stage: test needs: ["build-debug"] - image: dppp_base:${CI_COMMIT_SHORT_SHA} + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA before_script: - pip3 install gcovr script: - - export SRC=`pwd` - cd build - - ctest -j$(($(nproc) / 2)) -L unit |& tee ctest.out + - ninja # Needed when ran on different containers with different timestamps + - ctest -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -L unit |& tee ctest.out # Check if ctest found any tests. ctest >= 3.18 has a --no-tests=error # option. Older versions require a manual check. - if grep -q 'No tests were found' ctest.out; then exit 1; fi - - gcovr -j$(($(nproc) / 2)) -r $SRC -e '.*/external/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' -e '.*/Common/.*' - - gcovr -j$(($(nproc) / 2)) -r $SRC -e '.*/external/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' -e '.*/Common/.*' --json -o run-unit.json + - gcovr -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -r ../ -e '.*/external/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' -e '.*/Common/.*' + - gcovr -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -r ../ -e '.*/external/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' -e '.*/Common/.*' --json -o run-unit.json artifacts: paths: - build/run-unit.json @@ -95,23 +110,24 @@ unit-test: integration-test: stage: test needs: ["build-debug"] - image: dppp_base:${CI_COMMIT_SHORT_SHA} + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA before_script: - pip3 install gcovr h5py script: - - export SRC=`pwd` - cd build - - ctest -j$(($(nproc) / 2)) -L integration |& tee ctest.out + - ninja # Needed when ran on different containers with different timestamps + - ctest -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -L integration |& tee ctest.out # Check if ctest found any tests. ctest >= 3.18 has a --no-tests=error # option. Older versions require a manual check. - if grep -q 'No tests were found' ctest.out; then exit 1; fi - - gcovr -j$(($(nproc) / 2)) -r $SRC -e '.*/external/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' -e '.*/Common/.*' --json -o run-integration.json + - gcovr -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -r ../ -e '.*/external/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' -e '.*/Common/.*' --json -o run-integration.json artifacts: paths: - build/run-integration.json publish-doc: stage: publish + image: docker:stable needs: ["build-doc"] before_script: - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' @@ -130,18 +146,17 @@ publish-doc: pages: stage: pages + image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA needs: ["build-debug", "unit-test", "integration-test"] - image: dppp_base:${CI_COMMIT_SHORT_SHA} before_script: - apt-get update - apt-get -y install curl - pip3 install gcovr script: - - export SRC=`pwd` - mkdir -p .public/build/reports - cd .public - - gcovr -j$(($(nproc) / 2)) -r $SRC -a ../build/run-integration.json -a ../build/run-unit.json --xml -o build/reports/code-coverage.xml - - gcovr -j$(($(nproc) / 2)) -r $SRC -a ../build/run-integration.json -a ../build/run-unit.json --html --html-details -o index.html + - gcovr -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -r ../ -a ../build/run-integration.json -a ../build/run-unit.json --xml -o build/reports/code-coverage.xml + - gcovr -j$(($(nproc)/2 > 0 ? $(nproc)/2:1)) -r ../ -a ../build/run-integration.json -a ../build/run-unit.json --html --html-details -o index.html - cp ../build/unittests.xml build/reports/unit-tests.xml # Create and upload GitLab badges - chmod -R 700 ../CI diff --git a/CI/collect_metrics.py b/CI/collect_metrics.py index 04a26a9ade83f3c69a3504693861065c9ac31191..ca7a817cec852e95df3a038e48cce6867fa38f1c 100644 --- a/CI/collect_metrics.py +++ b/CI/collect_metrics.py @@ -1,4 +1,4 @@ -# Retrived from https://gitlab.com/ska-telescope/ci-metrics-utilities/ +# Retrieved from https://gitlab.com/ska-telescope/ci-metrics-utilities/ """Script to collect and parse CI metrics.""" import sys diff --git a/CI/create_badges.py b/CI/create_badges.py index 8c15b29e95895e79505015d417f0ca892e4b8975..d3953087a66e18058d42d1488cd0b40f14b1ea29 100644 --- a/CI/create_badges.py +++ b/CI/create_badges.py @@ -1,28 +1,5 @@ -# Retrived from https://gitlab.com/ska-telescope/ci-metrics-utilities/ -"""Create badges for GitLab metric collection. - -Run in .gitlab-ci.yml with: - ``` - create ci metrics: - stage: .post - when: always - tags: - - <tag> - script: - - apt-get -y update - - apt-get install -y curl --no-install-recommends - - curl -s https://gitlab.com/ska-telescope/ci-metrics-utilities/raw/master/scripts/ci-badges-func.sh | sh # pylint: disable=line-too-long - # Gitlab CI badges creation: END - allow_failure: true - artifacts: - paths: - - ./build - ``` -or - ``` - curl https://gitlab.com/ska-telescope/ci-metrics-utilities/raw/master/scripts/create_badges.py | python3 # pylint: disable=line-too-long - ``` -""" +# Retrieved from https://gitlab.com/ska-telescope/ci-metrics-utilities/ +"""Create badges for GitLab metric collection.""" import sys import json from datetime import datetime diff --git a/docker/ubuntu_20_04_lofar b/docker/ubuntu_20_04_lofar index d4bd89d5a4baf6067ea1b434e088c6141869cccf..8c98c43962ade0d893da1c4183689ee14d1ea7ba 100644 --- a/docker/ubuntu_20_04_lofar +++ b/docker/ubuntu_20_04_lofar @@ -1,12 +1,12 @@ ARG BASE_TAG=latest -FROM dppp_base:$BASE_TAG +FROM $BASE_TAG ADD . /src # Build DPPP RUN mkdir /build && cd /build \ && cmake ../src \ - && make -j$(($(nproc) / 2)) \ - && make install -j$(($(nproc) / 2)) \ + && make -j$(($(nproc) / 2 > 0 ? $(nproc) / 2:1)) \ + && make install -j$(($(nproc) / 2 > 0 ? $(nproc) / 2:1)) \ && DPPP \ && rm -r /src /build