diff --git a/.gitignore b/.gitignore index d890588e189903b1efc03a6e8f03a1d59d3735d1..672528bf6548090d3146968e5cd1736829df62d1 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,6 @@ charts/ska-tango-base/charts/tango-util-0.1.0.tgz charts/ska-tango-base/charts/* charts/ska-tango-base/Chart.lock charts/ska-tango-umbrella/charts +charts/ska-tango-umbrella/Chart.lock # we generate this values.yaml charts/ska-tango-base/values.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6bfb7f540852a5ea28560d2a94c341852e62a51c..a2914e4348f29c8b212babe26ce720dad0949e86 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,14 +94,15 @@ build_tango-db: build_tango-db-alpine: retry: 2 - stage: build_1 + stage: build_3 extends: - .image_builder_template variables: OCI_IMAGE: ska-tango-images-tango-db-alpine + BUILD_IMAGE: ${CI_REGISTRY}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/ska-tango-images-tango-cpp-alpine rules: - if: '$CI_COMMIT_TAG' - - if: '$FORCE_REBUILD == "yes" || $FORCE_REBUILD == "build_1"' + - if: '$FORCE_REBUILD == "yes" || $FORCE_REBUILD == "build_3"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' changes: - images/ska-tango-images-tango-db-alpine/* @@ -185,9 +186,28 @@ build_tango-rest: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' changes: - images/ska-tango-images-tango-dependencies/* - - images/ska-tango-images-tango-cpp/* + - images/ska-tango-images-tango-java/* - images/ska-tango-images-tango-rest/* +build_tango-rest-alpine: + retry: 2 + stage: build_3 + extends: + - .image_builder_template + variables: + OCI_IMAGE: ska-tango-images-tango-rest-alpine + BASE_IMAGE: ${CI_REGISTRY}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/ska-tango-images-tango-java-alpine + BUILD_IMAGE: ${CI_REGISTRY}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/ska-tango-images-tango-dependencies-alpine + rules: + - if: '$CI_COMMIT_TAG' + - if: '$FORCE_REBUILD == "yes" || $FORCE_REBUILD == "build_3"' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' + changes: + - images/ska-tango-images-tango-dependencies-alpine/* + - images/ska-tango-images-tango-java-alpine/* + - images/ska-tango-images-tango-rest-alpine/* + + build_pytango-builder: retry: 2 stage: build_3 diff --git a/Makefile b/Makefile index 427dfed5b5dec260ce095aa1db6ed1af318afba3..f1a0e245e753ba6d82bfc6954b2f553e80dd9690 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ BASE = $(shell pwd) HELM_CHARTS_TO_PUBLISH ?= ska-tango-util ska-tango-base -# OCI_IMAGES ?= ska-tango-images-tango-dependencies ska-tango-images-tango-dependencies-alpine ska-tango-images-tango-db ska-tango-images-tango-cpp ska-tango-images-tango-cpp-alpine ska-tango-images-tango-java ska-tango-images-tango-java-alpine ska-tango-images-tango-rest ska-tango-images-tango-rest-alpine ska-tango-images-pytango-builder ska-tango-images-pytango-builder-alpine ska-tango-images-tango-pogo ska-tango-images-tango-pogo-alpine ska-tango-images-tango-libtango ska-tango-images-tango-jive ska-tango-images-tango-jive-alpine ska-tango-images-pytango-runtime ska-tango-images-pytango-runtime-alpine ska-tango-images-tango-admin ska-tango-images-tango-databaseds ska-tango-images-tango-test ska-tango-images-tango-dsconfig ska-tango-images-tango-itango ska-tango-images-tango-vnc ska-tango-images-tango-vnc-alpine ska-tango-images-tango-pytango ska-tango-images-tango-panic ska-tango-images-tango-panic-gui -OCI_IMAGES ?= ska-tango-images-tango-dependencies ska-tango-images-tango-dependencies-alpine ska-tango-images-tango-db ska-tango-images-tango-cpp ska-tango-images-tango-cpp-alpine ska-tango-images-tango-java ska-tango-images-tango-java-alpine ska-tango-images-tango-rest ska-tango-images-pytango-builder ska-tango-images-pytango-builder-alpine ska-tango-images-tango-pogo ska-tango-images-tango-libtango ska-tango-images-tango-jive ska-tango-images-pytango-runtime ska-tango-images-pytango-runtime-alpine ska-tango-images-tango-admin ska-tango-images-tango-databaseds ska-tango-images-tango-test ska-tango-images-tango-dsconfig ska-tango-images-tango-itango ska-tango-images-tango-vnc ska-tango-images-tango-pytango ska-tango-images-tango-panic ska-tango-images-tango-panic-gui +OCI_IMAGES ?= ska-tango-images-tango-dependencies ska-tango-images-tango-dependencies-alpine ska-tango-images-tango-db ska-tango-images-tango-db-alpine ska-tango-images-tango-cpp ska-tango-images-tango-cpp-alpine ska-tango-images-tango-java ska-tango-images-tango-java-alpine ska-tango-images-tango-rest ska-tango-images-tango-rest-alpine ska-tango-images-pytango-builder ska-tango-images-pytango-builder-alpine ska-tango-images-tango-pogo ska-tango-images-tango-libtango ska-tango-images-tango-jive ska-tango-images-pytango-runtime ska-tango-images-pytango-runtime-alpine ska-tango-images-tango-admin ska-tango-images-tango-databaseds ska-tango-images-tango-test ska-tango-images-tango-dsconfig ska-tango-images-tango-dsconfig-alpine ska-tango-images-tango-itango ska-tango-images-tango-itango-alpine ska-tango-images-tango-vnc ska-tango-images-tango-pytango ska-tango-images-tango-panic ska-tango-images-tango-panic-gui OCI_IMAGES_TO_PUBLISH ?= $(OCI_IMAGES) KUBE_NAMESPACE ?= ska-tango-images#namespace to be used diff --git a/images/ska-tango-images-pytango-builder-alpine/Dockerfile b/images/ska-tango-images-pytango-builder-alpine/Dockerfile index 2ddbc2da58d19f4487d633e7f4d5b001e85d04d7..206002e6cd0bee06d3df6221cf33a6d4b09ddb9e 100644 --- a/images/ska-tango-images-pytango-builder-alpine/Dockerfile +++ b/images/ska-tango-images-pytango-builder-alpine/Dockerfile @@ -2,51 +2,50 @@ ARG CAR_OCI_REGISTRY_HOST ARG BASE_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-cpp-alpine:0.0.0" FROM $BASE_IMAGE - -LABEL \ - author="Piers Harding <Piers.Harding@skao.int>" \ - description="This image illustrates build dependencies" \ - license="Apache2.0" \ - int.skao.team="Systems Team" \ - int.skao.website="https://gitlab.com/ska-telescope/sdi/ska-ser-containerisation-and-orchestration" \ - int.skao.application="Tango Builder" - USER root +# pkgconfig needs to find these +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig +ENV LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib # Install build time dependencies -RUN apk add --update --no-cache curl libstdc++ g++\ - openssl ca-certificates libsodium-dev \ - boost py3-distutils-extra zlib-dev py3-pip boost-python3 boost-dev libzmq bash - -RUN apk add --no-cache --virtual build-dependencies \ - build-base alpine-sdk \ - linux-headers \ - git make - +# boost-dev and libstdc++ are fundamental dependencies +RUN apk --update add --no-cache libstdc++ \ + g++ \ + libsodium-dev \ + zlib-dev \ + boost-dev \ + python3-dev \ + py3-pip \ + py3-wheel\ + py3-cryptography \ + bash \ + make \ + ca-certificates \ + curl \ + git + +ENV PYTHONPATH=/usr/local/lib/python3.9/site-packages # setup pip ready for package installs COPY pip.conf /etc/pip.conf -RUN pip3 install setuptools \ - wheel - WORKDIR /app - -# pkgconfig needs to find these -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig +COPY requirements.txt requirements.txt # Install numpy manually before PyTango and other requirements to ensure we # build PyTango with numpy support. -RUN python3.9 -m pip install numpy==1.21.0 - +RUN python3.9 -m pip install --prefix=/usr/local numpy==1.21.0 \ # now install build requirements -COPY requirements.txt /requirements.txt -RUN python3.9 -m pip install -r /requirements.txt - -ENV LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib - -RUN mkdir -p /etc/ld.so.conf.d && \ - echo "include /etc/ld.so.conf.d/*.conf" > /etc/ld.so.conf && \ - echo "/usr/local/lib" > /etc/ld.so.conf.d/tango.conf && \ - echo "/usr/local/lib64" > /etc/ld.so.conf.d/tango.conf && \ - ldconfig /etc/ld.so.conf.d - -RUN mkdir /venv && ln -s /usr/* /venv/ && ln -s /usr/local/bin/itango3 /venv/bin/itango3 \ No newline at end of file + && python3.9 -m pip install --prefix=/usr/local -r requirements.txt \ +# this should be commented if we wish for the builder +# to keep the python cache + && python3.9 -m pip cache purge + +# do ldconfig stuff +RUN mkdir -p /etc/ld.so.conf.d \ + && echo "include /etc/ld.so.conf.d/*.conf" > /etc/ld.so.conf \ + && echo "/usr/local/lib" > /etc/ld.so.conf.d/tango.conf \ + && echo "/usr/local/lib64" > /etc/ld.so.conf.d/tango.conf \ + && ldconfig /etc/ld.so.conf.d \ +# this was on the debian version + && mkdir /venv \ + && ln -s /usr/* /venv/ \ + && ln -s /usr/local/bin/itango3 /venv/bin/itango3 diff --git a/images/ska-tango-images-pytango-runtime-alpine/Dockerfile b/images/ska-tango-images-pytango-runtime-alpine/Dockerfile index 5a8554b48a19faf32f6a8db9041d60409984fb15..679c6b6e65bcc618c13e50c6f0e17c675bd88d16 100644 --- a/images/ska-tango-images-pytango-runtime-alpine/Dockerfile +++ b/images/ska-tango-images-pytango-runtime-alpine/Dockerfile @@ -15,9 +15,10 @@ LABEL \ USER root # Install build time dependencies -RUN apk add --update --no-cache curl bash \ - openssl ca-certificates libstdc++ g++ libsodium-dev \ - boost py3-distutils-extra py3-pip boost-python3 make libzmq +# cpp already has libstdc++ libsodium-dev +RUN apk add --update --no-cache curl bash ca-certificates \ + boost py3-distutils-extra py3-pip make g++ python3-dev + # Copy across files that are used to help orchestrate container compositions # and test execution sequences @@ -37,8 +38,8 @@ RUN mkdir /venv && ln -s /usr/* /venv/ && ln -s /usr/bin/python3 /venv/bin/pytho USER tango ENV PATH="/home/tango/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}" - +ENV PYTHONPATH=/usr/local/lib/python3.9/site-packages ONBUILD COPY --chown=tango:tango . /app ONBUILD COPY --from=buildenv /usr/local/bin/ /usr/local/bin/ -ONBUILD WORKDIR /app \ No newline at end of file +ONBUILD WORKDIR /app diff --git a/images/ska-tango-images-tango-cpp-alpine/Dockerfile b/images/ska-tango-images-tango-cpp-alpine/Dockerfile index 897b3db3d6a8fa20e46425e0611e873f3510ac09..96d104027c448e5e6f9a2cc1d1522f64e78f2c02 100644 --- a/images/ska-tango-images-tango-cpp-alpine/Dockerfile +++ b/images/ska-tango-images-tango-cpp-alpine/Dockerfile @@ -1,14 +1,12 @@ ARG CAR_OCI_REGISTRY_HOST -ARG BASE_IMAGE="python:3.9.6-alpine3.14" +ARG BASE_IMAGE="alpine:3.14" ARG BUILD_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-dependencies-alpine:0.0.0" FROM $BUILD_IMAGE AS buildenv # Install build time dependencies -RUN apk add --update --no-cache make bash\ - libsodium-dev libstdc++ g++ mariadb-connector-c-dev - -RUN apk add --virtual build-dependencies \ - git cmake +RUN apk --update add --no-cache make bash\ + libsodium-dev libstdc++ g++ mariadb-connector-c-dev \ + git cmake python3-dev # build and install tangoidl RUN git clone --depth 1 https://gitlab.com/tango-controls/tango-idl.git /idl && \ @@ -53,14 +51,12 @@ LABEL \ # copy the built library dependencies from the builder stage COPY --from=buildenv /usr/local /usr/local -RUN apk add --update --no-cache sudo libstdc++ bash libsodium-dev mariadb-connector-c - -RUN ln -s /usr/local/lib/libtango.so.9.4.0 /usr/local/lib/libtango.so.9 - -# for some reason we are getting the wrong DataBaseds name so link it to the expected one +RUN apk --update add --no-cache sudo libstdc++ bash libsodium-dev mariadb-connector-c \ + && ln -s /usr/local/lib/libtango.so.9.4.0 /usr/local/lib/libtango.so.9 \ +# for some reason we are getting the wrong DataBaseds name + && ln -s /usr/local/bin/Databaseds /usr/local/bin/DataBaseds \ # also create the user tango and add it to sudoers -RUN ln -s /usr/local/bin/Databaseds /usr/local/bin/DataBaseds && \ - adduser -h /home/tango -s /bin/bash -D tango && \ - echo "tango ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/tango + && adduser -h /home/tango -s /bin/bash -D tango \ + && echo "tango ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/tango -USER tango +USER tango \ No newline at end of file diff --git a/images/ska-tango-images-tango-db-alpine/Dockerfile b/images/ska-tango-images-tango-db-alpine/Dockerfile index e121f954caae1aa6c9e46b8b0a098279fe6b1db6..9c5e5136ccff6a6866dd7b819ab44ed2d74dba99 100644 --- a/images/ska-tango-images-tango-db-alpine/Dockerfile +++ b/images/ska-tango-images-tango-db-alpine/Dockerfile @@ -1,88 +1,20 @@ - +ARG CAR_OCI_REGISTRY_HOST ARG BASE_IMAGE="alpine:3.14" -FROM $BASE_IMAGE AS builder - -### Install and Setup Mariadb - -ENV LC_ALL=en_GB.UTF-8 - -RUN mkdir /docker-entrypoint-initdb.d && \ - apk -U upgrade && \ - apk add --no-cache mariadb mariadb-client && \ - apk add --no-cache tzdata && \ - # clean up - rm -rf /var/cache/apk/* - -# comment out a few problematic configuration values -RUN sed -Ei 's/^(bind-address|log)/#&/' /etc/my.cnf && \ - sed -i 's/^skip-networking/#&/' /etc/my.cnf.d/mariadb-server.cnf && \ - # don't reverse lookup hostnames, they are usually another container - sed -i '/^\[mysqld]$/a skip-host-cache\nskip-name-resolve' /etc/my.cnf && \ - # always run as user mysql - sed -i '/^\[mysqld]$/a user=mysql' /etc/my.cnf && \ - # allow custom configurations - echo -e '\n!includedir /etc/mysql/conf.d/' >> /etc/my.cnf && \ - mkdir -p /etc/mysql/conf.d/ - -VOLUME /var/lib/mysql - -EXPOSE 3306 - -ENV TANGO_DOWNLOAD_URL=https://artefact.skao.int/repository/raw-internal/ska-tango-images/libraries/tango-9.3.4.tar.gz - -RUN apk add --update --no-cache curl \ - ca-certificates - -RUN mkdir -p /usr/src/tango - -WORKDIR /usr/src/tango - -RUN curl -fsSL "$TANGO_DOWNLOAD_URL" -o tango.tar.gz && \ - tar xf tango.tar.gz -C /usr/src/tango --strip-components=1 +ARG BUILD_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-cpp-alpine:0.0.0" +FROM $BUILD_IMAGE AS buildenv -# create dump from tango tar gz and move it into the folder docker-entrypoint-initdb.d -RUN mkdir -p dbinit/include \ - && cp cppserver/database/create_db.sql.in dbinit/create_db.sql \ - && cp cppserver/database/create_db_tables.sql.in dbinit/include/create_db_tables.sql \ - && cp cppserver/database/stored_proc.sql.in dbinit/include/stored_proc.sql \ - && sed -i "s|@TANGO_DB_NAME@|tango|g" dbinit/create_db.sql \ - && sed -i "s|@TANGO_DB_NAME@|tango|g" dbinit/include/create_db_tables.sql \ - && sed -i "s|@TANGO_DB_NAME@|tango|g" dbinit/include/stored_proc.sql \ - && sed -i "s|^source create_db_tables.sql$|source /docker-entrypoint-initdb.d/include/create_db_tables.sql|g" dbinit/create_db.sql \ - && sed -i "s|^source stored_proc.sql$|source /docker-entrypoint-initdb.d/include/stored_proc.sql|g" dbinit/create_db.sql \ - && sed -i "/CREATE DATABASE tango;/d" dbinit/create_db.sql \ - && cp -r dbinit/* /docker-entrypoint-initdb.d +USER root -RUN rm -r /usr/src/tango +# source is giving syntax error, so we just paste everything +RUN mkdir /docker-entrypoint-initdb.d && cd /usr/local/share/tango/db \ + && sed -i '/create_db_tables.sql/ r create_db_tables.sql' create_db.sql \ + && sed -i '/stored_proc.sql/ r stored_proc.sql' create_db.sql \ + && sed -i "s/^source/#source/g" create_db.sql \ + && sed -i "/CREATE DATABASE tango;/d" create_db.sql \ + && cp create_db.sql /docker-entrypoint-initdb.d FROM $BASE_IMAGE -### Install and Setup Mariadb - -ENV LC_ALL=en_GB.UTF-8 - -RUN mkdir /docker-entrypoint-initdb.d && \ - apk -U upgrade && \ - apk add --no-cache mariadb mariadb-client && \ - apk add --no-cache tzdata && \ - # clean up - rm -rf /var/cache/apk/* - -# comment out a few problematic configuration values -RUN sed -Ei 's/^(bind-address|log)/#&/' /etc/my.cnf && \ - sed -i 's/^skip-networking/#&/' /etc/my.cnf.d/mariadb-server.cnf && \ - # don't reverse lookup hostnames, they are usually another container - sed -i '/^\[mysqld]$/a skip-host-cache\nskip-name-resolve' /etc/my.cnf && \ - # always run as user mysql - sed -i '/^\[mysqld]$/a user=mysql' /etc/my.cnf && \ - # allow custom configurations - echo -e '\n!includedir /etc/mysql/conf.d/' >> /etc/my.cnf && \ - mkdir -p /etc/mysql/conf.d/ - -VOLUME /var/lib/mysql - -EXPOSE 3306 - LABEL \ author="Matteo Di Carlo <matteo.dicarlo@inaf.it>" \ description="This image is the tango db (MariaDB) from the TANGO-community" \ @@ -92,5 +24,20 @@ LABEL \ org.skatelescope.version="1.0.0" \ int.skao.application="Tango DB" -COPY --from=builder /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN apk --update add --no-cache pwgen bash coreutils alpine-baselayout +RUN addgroup --system mysql && adduser --system -s /bin/bash -G mysql mysql + +RUN apk --update add --no-cache mariadb mariadb-client pwgen + +COPY run.sh /scripts/run.sh +RUN mkdir /docker-entrypoint-initdb.d /etc/mysql/conf.d && \ + chmod -R 755 /scripts /etc/mysql/conf.d + +EXPOSE 3306 + +VOLUME ["/var/lib/mysql"] +COPY --from=buildenv /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d COPY sql_mode.cnf /etc/mysql/conf.d +RUN chmod 644 /etc/mysql/conf.d/sql_mode.cnf +ENTRYPOINT ["/scripts/run.sh"] diff --git a/images/ska-tango-images-tango-db-alpine/run.sh b/images/ska-tango-images-tango-db-alpine/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..e84709c7bebcb7bb48930b47dad448edc307be43 --- /dev/null +++ b/images/ska-tango-images-tango-db-alpine/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# execute any pre-init scripts +for i in /scripts/pre-init.d/*sh +do + if [ -e "${i}" ]; then + echo "[i] pre-init.d - processing $i" + . "${i}" + fi +done + +if [ -d "/run/mysqld" ]; then + echo "[i] mysqld already present, skipping creation" + chown -R mysql:mysql /run/mysqld +else + echo "[i] mysqld not found, creating...." + mkdir -p /run/mysqld + chown -R mysql:mysql /run/mysqld +fi + +if [ -d /var/lib/mysql/mysql ]; then + echo "[i] MySQL directory already present, skipping creation" + chown -R mysql:mysql /var/lib/mysql +else + echo "[i] MySQL data directory not found, creating initial DBs" + + chown -R mysql:mysql /var/lib/mysql + + mysql_install_db --user=mysql --ldata=/var/lib/mysql > /dev/null + + if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then + MYSQL_ROOT_PASSWORD=`pwgen 16 1` + echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD" + fi + + MYSQL_DATABASE=${MYSQL_DATABASE:-""} + MYSQL_USER=${MYSQL_USER:-""} + MYSQL_PASSWORD=${MYSQL_PASSWORD:-""} + + tfile=`mktemp` + if [ ! -f "$tfile" ]; then + return 1 + fi + + cat << EOF > $tfile +USE mysql; +FLUSH PRIVILEGES ; +GRANT ALL ON *.* TO 'root'@'%' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; +GRANT ALL ON *.* TO 'root'@'localhost' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; +SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ; +DROP DATABASE IF EXISTS test ; +FLUSH PRIVILEGES ; +EOF + + if [ "tango" != "" ]; then + echo "[i] Creating database: tango" + if [ "$MYSQL_CHARSET" != "" ] && [ "$MYSQL_COLLATION" != "" ]; then + echo "[i] with character set [$MYSQL_CHARSET] and collation [$MYSQL_COLLATION]" + echo "CREATE DATABASE IF NOT EXISTS \`tango\` CHARACTER SET $MYSQL_CHARSET COLLATE $MYSQL_COLLATION;" >> $tfile + else + echo "[i] with character set: 'utf8mb4' and collation: 'utf8mb4_unicode_ci'" + echo "CREATE DATABASE IF NOT EXISTS \`tango\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" >> $tfile + fi + + if [ "$MYSQL_USER" != "" ]; then + echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" + echo "GRANT ALL ON \`tango\`.* to '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile + fi + fi + + /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 < $tfile + rm -f $tfile + + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sql) echo "$0: running $f"; /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 < "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 < "$f"; echo ;; + *) echo "$0: ignoring or entrypoint initdb empty $f" ;; + esac + echo + done + + echo + echo 'MySQL init process done. Ready for start up.' + echo + + echo "exec /usr/bin/mysqld --user=mysql --console " "$@" +fi + +# execute any pre-exec scripts +for i in /scripts/pre-exec.d/*sh +do + if [ -e "${i}" ]; then + echo "[i] pre-exec.d - processing $i" + . ${i} + fi +done + +exec /usr/bin/mysqld --user=mysql --console $@ + + diff --git a/images/ska-tango-images-tango-db/Dockerfile b/images/ska-tango-images-tango-db/Dockerfile index efaa184e44d1e2cc9d54db27e1a097f898c951fe..c71afbe58a30889238c4b95e9118f24360b0c9d8 100644 --- a/images/ska-tango-images-tango-db/Dockerfile +++ b/images/ska-tango-images-tango-db/Dockerfile @@ -41,3 +41,4 @@ LABEL \ COPY --from=builder /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d COPY sql_mode.cnf /etc/mysql/conf.d +RUN chmod 644 /etc/mysql/conf.d/sql_mode.cnf \ No newline at end of file diff --git a/images/ska-tango-images-tango-dependencies-alpine/Dockerfile b/images/ska-tango-images-tango-dependencies-alpine/Dockerfile index d27bab3e7751a381cd4c28ee4d0ff45fc8ae2e77..bad4bbb57f5f91b5a9f6edc66b6287591b0877f2 100644 --- a/images/ska-tango-images-tango-dependencies-alpine/Dockerfile +++ b/images/ska-tango-images-tango-dependencies-alpine/Dockerfile @@ -3,27 +3,23 @@ # intermediate image, then creates a release image containing the compiled # binaries. # -ARG BASE_IMAGE="python:3.9.6-alpine3.14" +ARG BASE_IMAGE="alpine:3.14" FROM $BASE_IMAGE as buildenv - # Install build time dependencies -RUN apk add --update --no-cache bash curl \ - openssl ca-certificates \ - libstdc++ \ - g++ make libsodium-dev -RUN apk add --virtual build-dependencies \ - git cmake +RUN apk --update add --no-cache bash curl \ + openssl ca-certificates libstdc++ \ + g++ make libsodium-dev git cmake python3-dev # build and install libzmq at v4.3.4 RUN git clone -b v4.3.4 --depth 1 https://github.com/zeromq/libzmq /libzmq && \ cmake -B /libzmq/build -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_DRAFTS=OFF -DWITH_DOC=OFF -DZMQ_BUILD_TESTS=OFF /libzmq && \ - make -C /libzmq/build install + make -j$(nproc) -C /libzmq/build install # build and install cpp zmq at v4.8.1 RUN git clone -b v4.8.1 --depth 1 https://github.com/zeromq/cppzmq.git /cppzmq && \ cmake -B /cppzmq/build -DCPPZMQ_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=/usr/local /cppzmq && \ - make -C /cppzmq/build install + make -j$(nproc) -C /cppzmq/build install # build and install omniORB at 4.2.4 RUN curl -L https://sourceforge.net/projects/omniorb/files/omniORB/omniORB-4.2.4/omniORB-4.2.4.tar.bz2/download -o /omniORB.tar.bz2 && \ @@ -31,24 +27,25 @@ RUN curl -L https://sourceforge.net/projects/omniorb/files/omniORB/omniORB-4.2.4 tar xaf /omniORB.tar.bz2 --strip-components=1 -C /omniORB && \ cd /omniORB && \ ./configure --prefix=/usr/local/ && \ - make && \ + make -j$(nproc) && \ make install +# Copy across files that are used to help orchestrate container compositions +# and test execution sequences +COPY wait-for-it.sh /usr/local/bin/wait-for-it.sh +COPY retry.sh /usr/local/bin/retry + + FROM $BASE_IMAGE LABEL \ author="Matteo Di Carlo <matteo.dicarlo@inaf.it>" \ description="This image includes all the depenencies needed for installing the TANGO-controls framework" \ license="BSD-3-Clause" \ - registry="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-dependencies" \ + registry="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-dependencies-alpine" \ org.skatelescope.team="Systems Team" \ org.skatelescope.version="1.0.0" \ int.skao.application="Tango dependencies" -RUN apk add --update --no-cache bash libstdc++ - COPY --from=buildenv /usr/local /usr/local -# Copy across files that are used to help orchestrate container compositions -# and test execution sequences -COPY wait-for-it.sh /usr/local/bin/wait-for-it.sh -COPY retry.sh /usr/local/bin/retry + diff --git a/images/ska-tango-images-tango-dsconfig-alpine/ska-tango-images-tango-dsconfig/.release b/images/ska-tango-images-tango-dsconfig-alpine/ska-tango-images-tango-dsconfig/.release deleted file mode 100644 index 3705631d84e536fc69561ed277186621929180b2..0000000000000000000000000000000000000000 --- a/images/ska-tango-images-tango-dsconfig-alpine/ska-tango-images-tango-dsconfig/.release +++ /dev/null @@ -1,2 +0,0 @@ -release=1.5.3 -tag=1.5.3 diff --git a/images/ska-tango-images-tango-dsconfig-alpine/ska-tango-images-tango-dsconfig/Dockerfile b/images/ska-tango-images-tango-dsconfig-alpine/ska-tango-images-tango-dsconfig/Dockerfile deleted file mode 100644 index dbc1779c91355754685135cc08e6a3e0dcca45da..0000000000000000000000000000000000000000 --- a/images/ska-tango-images-tango-dsconfig-alpine/ska-tango-images-tango-dsconfig/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -ARG CAR_OCI_REGISTRY_HOST -ARG BUILD_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-pytango-builder:9.3.12" -ARG BASE_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-pytango-runtime:9.3.12" -FROM $BUILD_IMAGE as buildenv -FROM $BASE_IMAGE - -LABEL \ - author="Matteo Di Carlo <matteo.dicarlo@inaf.it>" \ - description="This image includes the MaxIV Dsconfig tool" \ - license="BSD-3-Clause" \ - registry="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-dsconfig" \ - org.skatelescope.team="Systems Team" \ - org.skatelescope.version="1.0.0" \ - int.skao.application="MaxIV Tango Dsconfig" - -RUN python3 -m pip install python-dsconfig - -# Existing Helm charts assume tango-dsconfig starts in the root directory -WORKDIR / diff --git a/images/ska-tango-images-tango-itango-alpine/Dockerfile b/images/ska-tango-images-tango-itango-alpine/Dockerfile index 5263d78f6a9ac473bd86889a32cc2aa0ad85e242..1f7d59af7a9b4599c1d1bd49119c3753cd430747 100644 --- a/images/ska-tango-images-tango-itango-alpine/Dockerfile +++ b/images/ska-tango-images-tango-itango-alpine/Dockerfile @@ -15,9 +15,13 @@ LABEL \ int.skao.application="ITango" USER root -RUN apk add --update --no-cache procps +RUN apk add --update --no-cache procps \ +# if used for tests it will require gnu tar not the busybox one + tar USER tango +ENV PYTHONPATH=/usr/local/lib/python3.9/site-packages + # create ipython profile to so that itango doesn't fail if ipython hasn't run yet RUN ipython profile create diff --git a/images/ska-tango-images-tango-itango-alpine/requirements.txt b/images/ska-tango-images-tango-itango-alpine/requirements.txt index e4666223138ac5725b957a48f546bd2529ba162c..813581a0212038d2fd71bdb33b66033f8dc68f21 100644 --- a/images/ska-tango-images-tango-itango-alpine/requirements.txt +++ b/images/ska-tango-images-tango-itango-alpine/requirements.txt @@ -1,21 +1,31 @@ +numpy==1.21.0 +pytango==9.3.3 +flake8 +flake8_formatter_junit_xml +ipython==7.27.0 +itango==0.1.8 +coverage +docutils +MarkupSafe +Pygments +pylint pytest -pyyaml +pytest-bdd pytest-cov +pytest-forked pytest-json-report -pycodestyle<2.7.0,>=2.6.0a1 -pytest-bdd==3.4.0 -pytest-ordering -pytest-timeout -elasticsearch -kubernetes -assertpy -mock -importlib -elasticsearch-dsl -aiohttp -aiojobs -pytest-asyncio -aiohttp_cors -flake8 -tango-simlib>=0.9.0 -tabulate +pytest-mock +pytest-pycodestyle +pytest-pydocstyle +pytest-pylint +pytest-runner +python-dotenv>=0.5.1 +ptvsd +Sphinx +sphinx_rtd_theme +sphinx-autobuild +sphinxcontrib-websupport +requests +tox +transitions + diff --git a/images/ska-tango-images-tango-java-alpine/Dockerfile b/images/ska-tango-images-tango-java-alpine/Dockerfile index f80d364877ccdd0083b1ea16bf0d57e510d31730..e5830b7ec2e046cf0088bc38cc0337b4f5e18391 100644 --- a/images/ska-tango-images-tango-java-alpine/Dockerfile +++ b/images/ska-tango-images-tango-java-alpine/Dockerfile @@ -1,65 +1,21 @@ + # # This Dockerfile builds Tango including Java apps and libraries in an # intermediate image, then creates a release image containing the compiled # binaries. # ARG CAR_OCI_REGISTRY_HOST -ARG BUILD_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-dependencies-alpine:0.0.0" -ARG BASE_IMAGE="alpine:3.14" -FROM $BUILD_IMAGE AS buildenv -FROM $BASE_IMAGE - -# Install build time dependencies -RUN apk add --update --no-cache make bash sudo \ - libsodium-dev libstdc++ mariadb-connector-c-dev \ - && apk add --no-cache --virtual mydeps git cmake \ - curl openssl ca-certificates \ - java-cacerts wget autoconf automake libtool g++ +ARG BASE_IMAGE=${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-dependencies-alpine:0.0.0 +FROM $BASE_IMAGE AS buildenv -ENV JAVA_HOME /opt/openjdk-18 -ENV PATH $JAVA_HOME/bin:$PATH -ENV JAVA_VERSION 18-ea+11 +FROM $BASE_IMAGE ENV LOG4J=https://artefact.skao.int/repository/raw-internal/ska-tango-images/libraries/log4j-1.2.17.tar.gz ENV TANGO_DOWNLOAD_URL=https://artefact.skao.int/repository/raw-internal/ska-tango-images/libraries/tango-9.3.4.tar.gz -RUN set -eux; \ - \ - arch="$(apk --print-arch)"; \ - case "$arch" in \ - 'x86_64') \ - downloadUrl='https://download.java.net/java/early_access/alpine/11/binaries/openjdk-18-ea+11_linux-x64-musl_bin.tar.gz'; \ - downloadSha256='86fad9069587a5e9dd003e7354a69b2f720a05c12706d2f2345a0c8d90e56c47'; \ - ;; \ - *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ - esac; \ - \ - wget -O openjdk.tgz "$downloadUrl"; \ - echo "$downloadSha256 *openjdk.tgz" | sha256sum -c -; \ - \ - mkdir -p "$JAVA_HOME"; \ - tar --extract \ - --file openjdk.tgz \ - --directory "$JAVA_HOME" \ - --strip-components 1 \ - --no-same-owner \ - ; \ - rm openjdk.tgz*; \ - \ - rm -rf "$JAVA_HOME/lib/security/cacerts"; \ -# see "java-cacerts" package installed above (which maintains "/etc/ssl/certs/java/cacerts" for us) - ln -sT /etc/ssl/certs/java/cacerts "$JAVA_HOME/lib/security/cacerts"; \ - \ -# https://github.com/docker-library/openjdk/issues/212#issuecomment-420979840 -# https://openjdk.java.net/jeps/341 - java -Xshare:dump; \ - \ -# basic smoke test - fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java; \ - javac --version; \ - java --version - -# "jshell" is an interactive REPL for Java (see https://en.wikipedia.org/wiki/JShell) -CMD ["jshell"] +# Install run time dependencies +RUN apk --update add --no-cache bash sudo \ + libsodium-dev libstdc++ mariadb-connector-c-dev \ + openjdk8-jre-base openssl # copy the built library dependencies from the builder stage COPY --from=buildenv /usr/local /usr/local @@ -67,8 +23,13 @@ COPY --from=buildenv /usr/local /usr/local RUN mkdir -p /usr/src/tango /usr/share/man/man1 /usr/java WORKDIR /usr/src/tango -## tar with tango libraries, version 9.3.4 -RUN curl -fsSL "$TANGO_DOWNLOAD_URL" -o tango.tar.gz \ +## build dependencies are installed with virtual and removed +## in the end +RUN apk --update add --no-cache --virtual mybuilddeps make \ + curl ca-certificates file python3-dev \ + java-cacerts wget autoconf automake libtool g++ \ +## we use tar with tango libraries, version 9.3.4 + && curl -fsSL "$TANGO_DOWNLOAD_URL" -o tango.tar.gz \ && tar xf tango.tar.gz -C /usr/src/tango --strip-components=1 \ ## this tango version is old and and we need to change ## a few thingies under alpine to make it work, none of the following sed commands @@ -94,22 +55,24 @@ RUN curl -fsSL "$TANGO_DOWNLOAD_URL" -o tango.tar.gz \ ## the configure.ac file, some issue with the check for sed && sed -i '/SED_AC_WORKING($SED)/,+3d' configure.ac \ && autoconf && ./configure --with-zmq=/usr/local --with-omni=/usr/local \ - --with-mysqlclient-prefix=/usr --enable-static=no \ + --with-mysqlclient-prefix=/usr --enable-static=no \ ## we can now make it && make -C /usr/src/tango -j$(nproc) \ && make -C /usr/src/tango install \ - && rm -rf /usr/src/tango +## remove buildpackages & docs + && rm -rf /usr/src/tango && apk del mybuilddeps && rm -rf /usr/local/share/doc ##RUN ln -s /usr/local/lib/libtango.so.9.4.0 /usr/local/lib/libtango.so.9 WORKDIR /usr/java -RUN wget --no-check-certificate "$LOG4J" -O log4j.tar.gz \ +RUN apk --update add --no-cache --virtual mybuilddeps wget \ + && wget --no-check-certificate "$LOG4J" -O log4j.tar.gz \ && tar zxvf log4j.tar.gz \ && mv apache-log4j-1.2.17/log4j-1.2.17.jar /usr/local/share/java/log4j-1.2.17.jar \ && rm /usr/java/log4j.tar.gz && rm -rf apache-log4j-1.2.17 \ # create the user tango and add it to sudoers && adduser -h /home/tango -s /bin/bash -D tango \ - && echo "tango ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/tango && apk del mydeps - + && echo "tango ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/tango && apk del mybuilddeps \ + && apk --update add --no-cache python3 py3-pip -USER tango \ No newline at end of file +USER tango diff --git a/images/ska-tango-images-tango-rest-alpine/.release b/images/ska-tango-images-tango-rest-alpine/.release new file mode 100644 index 0000000000000000000000000000000000000000..cbedebed7d19e6bf4e37934b5c47cde9fd5ca888 --- /dev/null +++ b/images/ska-tango-images-tango-rest-alpine/.release @@ -0,0 +1,2 @@ +release=1.14.6 +tag=1.14.6 diff --git a/images/ska-tango-images-tango-rest-alpine/Dockerfile b/images/ska-tango-images-tango-rest-alpine/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..37637a50c9cc2c2e8a6807f16dccb648400a786d --- /dev/null +++ b/images/ska-tango-images-tango-rest-alpine/Dockerfile @@ -0,0 +1,42 @@ +ARG CAR_OCI_REGISTRY_HOST +ARG BUILD_IMAGE=alpine:3.14 +ARG BASE_IMAGE="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-java-alpine:0.0.0" + +FROM $BUILD_IMAGE AS buildenv + +ENV MTANGOREST_DOWNLOAD_URL=https://github.com/tango-controls/rest-server/releases/download/rest-server-1.14/rest-server-1.14.jar + +RUN apk add --no-cache ca-certificates curl + +RUN mkdir -p /usr/local/lib/tango + +WORKDIR /usr/local/lib/tango + +RUN curl -fsSL "$MTANGOREST_DOWNLOAD_URL" -o mtangorest.jar + +FROM $BASE_IMAGE + +LABEL \ + author="Matteo Di Carlo <matteo.dicarlo@inaf.it>" \ + description="This image is the tango-rest application available from the TANGO-community" \ + license="BSD-3-Clause" \ + registry="${CAR_OCI_REGISTRY_HOST}/ska-tango-images-tango-rest" \ + org.skatelescope.team="Systems Team" \ + org.skatelescope.version="1.0.0" \ + int.skao.application="Tango REST" + +COPY --from=buildenv /usr/local/lib/tango/mtangorest.jar /usr/local/lib/tango/mtangorest.jar + +USER root + +RUN mkdir -p /usr/share/man/man1 \ + && apk add --no-cache tomcat-native expat-dev \ + && python3 -m pip install --no-cache-dir supervisor==4.2.2 + +COPY tango_register_device.sh /usr/local/bin/ + +# Configure supervisord. Ensure supervisord.conf contains entries for your device! +COPY supervisord.conf /etc/supervisor/ + +# Start supervisor as daemon +CMD ["/usr/local/bin/supervisord", "--configuration", "/etc/supervisor/supervisord.conf"] diff --git a/images/ska-tango-images-tango-rest-alpine/supervisord.conf b/images/ska-tango-images-tango-rest-alpine/supervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..22a7582929357256854a3ad06bea0d00a54215e5 --- /dev/null +++ b/images/ska-tango-images-tango-rest-alpine/supervisord.conf @@ -0,0 +1,26 @@ +[supervisord] +nodaemon=true + +[program:tango-rest] +command=/bin/bash -c ' + MYHOSTNAME=`hostname --short` && + JAVA_OPTS="-Xmx4G -Xshare:off -XX:+UseG1GC -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5009" && + /usr/local/bin/tango_register_device.sh TangoRestServer/$MYHOSTNAME TangoRestServer $MYHOSTNAME/rest/0 && + /usr/local/bin/tango_admin --add-property $MYHOSTNAME/rest/0 TANGO_DB tango://$TANGO_HOST/sys/database/2 && + /usr/local/bin/tango_admin --add-property $MYHOSTNAME/rest/0 TOMCAT_PORT 8080 && + /usr/local/bin/tango_admin --add-property $MYHOSTNAME/rest/0 TOMCAT_AUTH_METHOD plain && + exec -- /usr/bin/java -jar $JAVA_OPTS /usr/local/lib/tango/mtangorest.jar $MYHOSTNAME' +autorestart=true +# TODO investigate why this can't run as a non-privileged user +#user=tango +priority=1 + +[unix_http_server] +username = dummy +password = dummy +file = /var/tmp/supervisord.sock + +[supervisorctl] +username = dummy +password = dummy + diff --git a/images/ska-tango-images-tango-rest-alpine/tango_register_device.sh b/images/ska-tango-images-tango-rest-alpine/tango_register_device.sh new file mode 100755 index 0000000000000000000000000000000000000000..837c22895c635c130319b23150a119b764541e87 --- /dev/null +++ b/images/ska-tango-images-tango-rest-alpine/tango_register_device.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ "$#" -ne 3 ]; then + echo "usage: $0 <exec>/<inst> <class> <device> " + exit 1 +fi + +tango_admin --check-device $3 || \ +tango_admin --add-server $1 $2 $3 || \ +exit 1 diff --git a/tests/post-deployment/features/tango_tools.feature b/tests/post-deployment/features/tango_tools.feature index df4dbabdb76a7d5aa6270d21084fba27f903c0d7..d7d8535c8ad13b791963b0c7422237947f50e10f 100644 --- a/tests/post-deployment/features/tango_tools.feature +++ b/tests/post-deployment/features/tango_tools.feature @@ -5,7 +5,6 @@ Scenario: Connect to the tango database using tango_admin When I call the tango_admin command with parameter ping-database Then the return code is 0 -@skip Scenario: Test starting itango session Given the TANGO_HOST is defined in the environment When I call the itango3 command with parameter simple-prompt diff --git a/tests/post-deployment/tests/__test_tango_tools.py b/tests/post-deployment/tests/test_tango_tools.py similarity index 100% rename from tests/post-deployment/tests/__test_tango_tools.py rename to tests/post-deployment/tests/test_tango_tools.py