diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 06ce8dd754454f38b9cf38ba8ec9903a8aaedaac..c44e723cc163fd9ff7f075be3b8308d0d2d4736f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -565,7 +565,7 @@ regression_test_SCU: # DEPLOY TEST STAGE # -deploy-tmss-test: +deploy-tmss-test-old: stage: deploy-test variables: LOFAR_TARGET: "scu199.control.lofar" @@ -581,7 +581,31 @@ deploy-tmss-test: - ssh $LOFAR_USER@$LOFAR_TARGET 'docker run --rm --env-file /localhome/lofarsys/.lofar/.lofar_env_test tmss_django:latest bash -c "source lofarinit.sh; bin/tmss_manage_django migrate"' - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*" environment: - name: test + name: test-old + needs: + - job: dockerize_TMSS + artifacts: false + allow_failure: true + when: manual + +.deploy-tmss-docker-compose: + stage: deploy-test + before_script: + - *prepare_ssh + script: + - apk update + - apk add rsync + - cd SAS/TMSS/deploy + - cp ${TMSS_DEPLOY_LCS129_PRODUCTION} environment + - chmod u+x environment + - source environment + - sh -e ./generate-env.sh + - rm environment + - rsync -aAXv --chmod=700 ./ ${LOFAR_USER}@${LOFAR_TARGET}:~/.lofar/tmss + - rm .env; rm env; rm app/.env + - ssh $LOFAR_USER@$LOFAR_TARGET "cd .lofar/tmss; /localhome/lofarsys/bin/docker-compose build; /localhome/lofarsys/bin/docker-compose up -d; /localhome/lofarsys/bin/docker-compose logs db_migrate" + after_script: + - ssh $LOFAR_USER@$LOFAR_TARGET "cd .lofar/tmss; rm .env; rm env; rm app/.env" needs: - job: dockerize_TMSS artifacts: false @@ -917,7 +941,17 @@ deploy-SCU-prod: only: - tags -deploy-tmss-prod: +deploy-tmss-prod-lcs129: + stage: deploy-prod + extends: .deploy-tmss-docker-compose + variables: + LOFAR_USER: "lofarsys" + LOFAR_TARGET: "lcs129.control.lofar" + SOURCE_IMAGE: "${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA" + environment: + name: production-lcs129 + +deploy-tmss-prod-scu001: stage: deploy-prod variables: LOFAR_TARGET: "scu001.control.lofar" @@ -933,7 +967,7 @@ deploy-tmss-prod: - ssh $LOFAR_USER@$LOFAR_TARGET 'docker run --rm --env-file /localhome/lofarsys/.lofar/.lofar_env_test tmss_django:latest bash -c "source lofarinit.sh; bin/tmss_manage_django migrate"' - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*" environment: - name: production + name: production-scu001 needs: - job: dockerize_TMSS artifacts: false diff --git a/SAS/TMSS/README.md b/SAS/TMSS/README.md index 68bdaf35fba678f9f468d845a8f742956237386b..0359b3b53e4bb3e2041c2d00d30d7080b6c6cabb 100644 --- a/SAS/TMSS/README.md +++ b/SAS/TMSS/README.md @@ -4,6 +4,19 @@ The TMSS project is developing a new software application for the specification, See the [TMSS Software Design](https://support.astron.nl/confluence/display/TMSS/Software+Design) for more details. +## Index + +- [Getting Started](#getting-started) + - [Prerequisites](#prerequisites) + - [Installing and running TMSS](#installing-and-running-tmss) +- [Running TMSS with MAC Scheduler](#running-tmss-with-mac-scheduler) +- [Continuous Integration](#continuous-integration) + - [Unit Test](#unit-test) + - [Integration Test](#integration-test) +- [Continuous Delivery](#continuous-delivery) + - [Test Deployment](#test-deployment) + - [Production Deployment](deploy/README.md) + ## Getting Started To make changes in the Lofar Repo you need a JIRA-ticket, which are for TMSS typical identified as TMSS-[ID]. @@ -125,7 +138,7 @@ This pipeline consist of 6 main stages: - TMSS Test - TMSS Acceptance -### Unit Test +### Unit Test Unit Test can be run with the command `ctest` that's how the 'build' pipeline will execute it. For development convenience, it is also possible to run the TMSS python unittest in PyCharm IDE. @@ -137,7 +150,16 @@ See the [TMSS landscape overview](https://support.astron.nl/confluence/display/T Integration tests are python unit test with `@integration_test` decorator. The session authentication test is a typical integration test for TMSS. -### Deployment +## Continuous Delivery + +The Deployment can only be executed manually by pushing the button after the +gitlab pipeline has executed most relevant stages of the pipeline successfully. + +### Test Deployment + +Test environment deployment of TMSS consist of pushing the TMSS docker container +to Lofar Test environment `scu199.control.lofar` and to the Acceptance +environment `tmss-ua.control.lofar`. -The Deployment can only be executed manually by pushing the button after the gitlab pipeline has accomplished all stages of the pipeline successfully. -Deployment of TMSS consist of pushing the TMSS docker container to Lofar Test environment `scu199.control.lofar` and to the Acceptance environment `tmss-ua.control.lofar`. +These environments are outdated and the process of deploying to them needs to +be improved [2023-03-15]. diff --git a/SAS/TMSS/deploy/.gitignore b/SAS/TMSS/deploy/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b33d93b15a7bfb5bea7bf72a5ef9eba6086a6ddc --- /dev/null +++ b/SAS/TMSS/deploy/.gitignore @@ -0,0 +1,4 @@ +app/.env +env +.env +environment diff --git a/SAS/TMSS/deploy/README.md b/SAS/TMSS/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e01a771eb645b6a92bd93651c9e1c0552d999ad6 --- /dev/null +++ b/SAS/TMSS/deploy/README.md @@ -0,0 +1,42 @@ +# Production deployment files for TMSS + +Deployment is orchestrated by running docker-compose on the target host. Only +this `deploy` directory needs to be migrated to the target host prior to +deployment. + +Below is an overview of the deployment structure / strategy across various +services (2023-03-15): + + + +- [Security](#security) +- [Service names](#service-names) +- [Accessing log files](#accessing-log-files) + +## Security + +Configuration for services is populated through environment files and +variables. The docker images built with compose in directory should never be +published as they contain the configuration to reach / access systems. + +## Service names + +The TMSS containers use fixed names of the pattern `tmss_xxx` while the +docker-compose service names omit this `tmss_` prefix. + +## Accessing log files + +Log files are flushed to journald so we can ensure they are persistent when +docker services are upgraded. To access the logs of any particular service +use: `journalctl -u docker CONTAINER_TAG=tmss_xxx` + +```shell +journalctl -u docker CONTAINER_TAG=tmss_feedback +``` + +The advantange of journald is that the docker engine can still access the logs +through `docker logs` as well: + +```shell +docker logs tmss_feedback +``` \ No newline at end of file diff --git a/SAS/TMSS/deploy/app/Dockerfile b/SAS/TMSS/deploy/app/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bcf945d392c5d2400c56baed74d6cdfd8862cd99 --- /dev/null +++ b/SAS/TMSS/deploy/app/Dockerfile @@ -0,0 +1,15 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} + +RUN source /opt/lofar/lofarinit.sh; bin/tmss_manage_django collectstatic --no-input + +ARG HOME +COPY .env ./ +COPY generate-dbcredentials.sh ./ +USER root +RUN chmod 0755 ./generate-dbcredentials.sh +RUN sh -e -c "source ./.env; ./generate-dbcredentials.sh" +RUN chown -R lofarsys ${HOME} +RUN chgrp -R lofarsys ${HOME} + +USER lofarsys diff --git a/SAS/TMSS/deploy/app/generate-dbcredentials.sh b/SAS/TMSS/deploy/app/generate-dbcredentials.sh new file mode 100755 index 0000000000000000000000000000000000000000..4326a5041069424fc7d451034522852ebe5fc03a --- /dev/null +++ b/SAS/TMSS/deploy/app/generate-dbcredentials.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +mkdir -p ${HOME}/.lofar/dbcredentials + +echo "[database:LTACatalogue]" > ${HOME}/.lofar/dbcredentials/LTACatalogue.ini +echo "host=${LTA_HOST}" >> ${HOME}/.lofar/dbcredentials/LTACatalogue.ini +echo "user=${LTA_USER}" >> ${HOME}/.lofar/dbcredentials/LTACatalogue.ini +echo "password=${LTA_PASSWORD}" >> ${HOME}/.lofar/dbcredentials/LTACatalogue.ini +echo "type=${LTA_TYPE}" >> ${HOME}/.lofar/dbcredentials/LTACatalogue.ini +echo "port=${LTA_PORT}" >> ${HOME}/.lofar/dbcredentials/LTACatalogue.ini +echo "database=${LTA_DATABASE}" >> ${HOME}/.lofar/dbcredentials/LTACatalogue.ini + +echo "[database:RabbitMQ]" > ${HOME}/.lofar/dbcredentials/rabbitmq.ini +echo "user=${RABBITMQ_USER}" >> ${HOME}/.lofar/dbcredentials/rabbitmq.ini +echo "password=${RABBITMQ_PASSWORD}" >> ${HOME}/.lofar/dbcredentials/rabbitmq.ini + +echo "[database:TMSS]" > ${HOME}/.lofar/dbcredentials/TMSS.ini +echo "host=${TMSS_HOST}" >> ${HOME}/.lofar/dbcredentials/TMSS.ini +echo "user=${TMSS_USER}" >> ${HOME}/.lofar/dbcredentials/TMSS.ini +echo "password=${TMSS_PASSWORD}" >> ${HOME}/.lofar/dbcredentials/TMSS.ini +echo "type=${TMSS_TYPE}" >> ${HOME}/.lofar/dbcredentials/TMSS.ini +echo "port=${TMSS_PORT}" >> ${HOME}/.lofar/dbcredentials/TMSS.ini +echo "database=${TMSS_DATABASE}" >> ${HOME}/.lofar/dbcredentials/TMSS.ini + +echo "[database:TMSSClient]" > ${HOME}/.lofar/dbcredentials/TMSSClient.ini +echo "host=${TMSS_CLIENT_HOST}" >> ${HOME}/.lofar/dbcredentials/TMSSClient.ini +echo "user=${TMSS_CLIENT_USER}" >> ${HOME}/.lofar/dbcredentials/TMSSClient.ini +echo "password=${TMSS_CLIENT_PASSWORD}" >> ${HOME}/.lofar/dbcredentials/TMSSClient.ini +echo "type=${TMSS_CLIENT_TYPE}" >> ${HOME}/.lofar/dbcredentials/TMSSClient.ini +echo "port=${TMSS_CLIENT_PORT}" >> ${HOME}/.lofar/dbcredentials/TMSSClient.ini +echo "database=${TMSS_CLIENT_DATABASE}" >> ${HOME}/.lofar/dbcredentials/TMSSClient.ini + +echo "[database:TMSSSlack]" > ${HOME}/.lofar/dbcredentials/TMSSSlack.ini +echo "host=${TMSS_SLACK_HOST}" >> ${HOME}/.lofar/dbcredentials/TMSSSlack.ini +echo "user=${TMSS_SLACK_USER}" >> ${HOME}/.lofar/dbcredentials/TMSSSlack.ini +echo "password=${TMSS_SLACK_PASSWORD}" >> ${HOME}/.lofar/dbcredentials/TMSSSlack.ini +echo "type=${TMSS_SLACK_TYPE}" >> ${HOME}/.lofar/dbcredentials/TMSSSlack.ini +echo "port=${TMSS_SLACK_PORT}" >> ${HOME}/.lofar/dbcredentials/TMSSSlack.ini +echo "database=${TMSS_SLACK_DATABASE}" >> ${HOME}/.lofar/dbcredentials/TMSSSlack.ini \ No newline at end of file diff --git a/SAS/TMSS/deploy/docker-compose.yml b/SAS/TMSS/deploy/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..1b7fdf7ccae20d2652f772cadc15489bab387fe3 --- /dev/null +++ b/SAS/TMSS/deploy/docker-compose.yml @@ -0,0 +1,297 @@ +version: '3' + +services: + db_migrate: + container_name: tmss_db_migrate + image: tmss_db_migrate + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; bin/tmss_manage_django migrate' + logging: + driver: journald + options: + tag: tmss_db_migrate + app: + container_name: tmss_app + image: tmss_app + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; gunicorn lofar.sas.tmss.tmss.wsgi --worker-class=gevent --workers=20 --timeout 120 --bind=0.0.0.0:8001' + ports: + - 8001:8001 + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_app + nginx: + container_name: tmss_nginx + image: tmss_nginx + build: + context: ./nginx + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + restart: unless-stopped + env_file: + - env + ports: + - 8008:8008 + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_nginx + websock: + container_name: tmss_websock + image: tmss_websock + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_websocket_service' + ports: + - 5678:5678 + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_websock + postgres_listener: + container_name: tmss_postgres_listener + image: tmss_postgres_listener + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_postgres_listener_service' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_postgres_listener + lta_adapter: + container_name: tmss_lta_adapter + image: tmss_lta_adapter + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_lta_adapter' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_lta_adapter + ra_adapter: + container_name: tmss_ra_adapter + image: tmss_ra_adapter + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_ra_adapter' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_ra_adapter + ingest_adapter: + container_name: tmss_ingest_adapter + image: tmss_ingest_adapter + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec ingesttmssadapter' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_ingest_adapter + precalculations: + container_name: tmss_precalculations + image: tmss_precalculations + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_precalculations_service' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_precalculations + feedback: + container_name: tmss_feedback + image: tmss_feedback + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_feedback_handling_service' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_feedback + scheduling: + container_name: tmss_scheduling + image: tmss_scheduling + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_scheduling_service' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_scheduling + workflow: + container_name: tmss_workflow + image: tmss_workflow + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_workflow_service' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_workflow + slack_webhook: + container_name: tmss_slack_webhook + image: tmss_slack_webhook + build: + context: ./app + dockerfile: Dockerfile + args: + SOURCE_IMAGE: ${SOURCE_IMAGE} + HOME: "/localhome/lofarsys" + restart: unless-stopped + env_file: + - env + environment: + - USER=lofarsys + - HOME=/localhome/lofarsys + command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_slack_webhook_service' + depends_on: + db_migrate: + condition: service_completed_successfully + logging: + driver: journald + options: + tag: tmss_slack_webhook \ No newline at end of file diff --git a/SAS/TMSS/deploy/generate-env.sh b/SAS/TMSS/deploy/generate-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..c82c1b45faa2186c6e8fbab8875956675221aac6 --- /dev/null +++ b/SAS/TMSS/deploy/generate-env.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Base image for two-step docker-compose image generation +echo "SOURCE_IMAGE=${SOURCE_IMAGE}" > .env + +echo "export LTA_HOST=${LTA_HOST}" > app/.env +echo "export LTA_USER=${LTA_USER}" >> app/.env +echo "export LTA_PASSWORD=${LTA_PASSWORD}" >> app/.env +echo "export LTA_TYPE=${LTA_TYPE}" >> app/.env +echo "export LTA_PORT=${LTA_PORT}" >> app/.env +echo "export LTA_DATABASE=${LTA_DATABASE}" >> app/.env +echo "export RABBITMQ_USER=${RABBITMQ_USER}" >> app/.env +echo "export RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}" >> app/.env +echo "export TMSS_HOST=${TMSS_HOST}" >> app/.env +echo "export TMSS_USER=${TMSS_USER}" >> app/.env +echo "export TMSS_PASSWORD=${TMSS_PASSWORD}" >> app/.env +echo "export TMSS_TYPE=${TMSS_TYPE}" >> app/.env +echo "export TMSS_PORT=${TMSS_PORT}" >> app/.env +echo "export TMSS_DATABASE=${TMSS_DATABASE}" >> app/.env +echo "export TMSS_CLIENT_HOST=${TMSS_CLIENT_HOST}" >> app/.env +echo "export TMSS_CLIENT_USER=${TMSS_CLIENT_USER}" >> app/.env +echo "export TMSS_CLIENT_PASSWORD=${TMSS_CLIENT_PASSWORD}" >> app/.env +echo "export TMSS_CLIENT_TYPE=${TMSS_CLIENT_TYPE}" >> app/.env +echo "export TMSS_CLIENT_PORT=${TMSS_CLIENT_PORT}" >> app/.env +echo "export TMSS_CLIENT_DATABASE=${TMSS_CLIENT_DATABASE}" >> app/.env +echo "export TMSS_SLACK_HOST=${TMSS_SLACK_HOST}" >> app/.env +echo "export TMSS_SLACK_USER=${TMSS_SLACK_USER}" >> app/.env +echo "export TMSS_SLACK_PASSWORD=${TMSS_SLACK_PASSWORD}" >> app/.env +echo "export TMSS_SLACK_TYPE=${TMSS_SLACK_TYPE}" >> app/.env +echo "export TMSS_SLACK_PORT=${TMSS_SLACK_PORT}" >> app/.env +echo "export TMSS_SLACK_DATABASE=${TMSS_SLACK_DATABASE}" >> app/.env + +# Environment variables required by processes in containers +echo "LOFARENV=${LOFARENV}" >> env +# tmss django settings +echo "DEBUG=${DEBUG}" >> env +echo "ALLOWED_HOSTS=${ALLOWED_HOSTS}" >> env +echo "SECRET_KEY=${SECRET_KEY}" >> env +echo "TMSS_ENABLE_VIEWFLOW=${TMSS_ENABLE_VIEWFLOW}" >> env +echo "TMSS_LOGOUT_REDIRECT_URL=${TMSS_LOGOUT_REDIRECT_URL}" >> env + +# RabbitMQ and lofar usage of rabbitmq +echo "RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}" >> env +echo "RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}" >> env +echo "RABBITMQ_DEFAULT_PORT=${RABBITMQ_DEFAULT_PORT}" >> env +echo "LOFAR_DEFAULT_BROKER=${LOFAR_DEFAULT_BROKER}" >> env +echo "LOFAR_DEFAULT_EXCHANGE=${LOFAR_DEFAULT_EXCHANGE}" >> env + +# Keycloak user authentication +echo "OIDC_RP_CLIENT_ID=${OIDC_RP_CLIENT_ID}" >> env +echo "OIDC_RP_CLIENT_SECRET=${OIDC_RP_CLIENT_SECRET}" >> env +echo "OIDC_OP_JWKS_ENDPOINT=${OIDC_OP_JWKS_ENDPOINT}" >> env +echo "OIDC_OP_AUTHORIZATION_ENDPOINT=${OIDC_OP_AUTHORIZATION_ENDPOINT}" >> env +echo "OIDC_OP_TOKEN_ENDPOINT=${OIDC_OP_TOKEN_ENDPOINT}" >> env +echo "OIDC_OP_USER_ENDPOINT=${OIDC_OP_USER_ENDPOINT}" >> env +echo "OIDC_ENDPOINT_HOST=${OIDC_ENDPOINT_HOST}" >> env +echo "KEYCLOAK_TOKEN_URL=${KEYCLOAK_TOKEN_URL}" >> env +echo "KEYCLOAK_ADMIN_USER=${KEYCLOAK_ADMIN_USER}" >> env +echo "KEYCLOAK_ADMIN_PASSWORD=${KEYCLOAK_ADMIN_PASSWORD}" >> env +echo "KEYCLOAK_API_BASE_URL=${KEYCLOAK_API_BASE_URL}" >> env + +# fallback for non-keycloak users (like admin etc) +echo "LDAP_SERVER_URI=${LDAP_SERVER_URI}" >> env \ No newline at end of file diff --git a/SAS/TMSS/deploy/nginx/Dockerfile b/SAS/TMSS/deploy/nginx/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c38078be117a7763680339291cceae033f16a78e --- /dev/null +++ b/SAS/TMSS/deploy/nginx/Dockerfile @@ -0,0 +1,10 @@ +ARG SOURCE_IMAGE +FROM ${SOURCE_IMAGE} AS static-files-stage + +RUN source /opt/lofar/lofarinit.sh; bin/tmss_manage_django collectstatic --no-input + +FROM nginx + +COPY --from=static-files-stage /opt/lofar/staticfiles /opt/lofar/staticfiles + +COPY default.conf /etc/nginx/conf.d/ \ No newline at end of file diff --git a/SAS/TMSS/deploy/nginx/default.conf b/SAS/TMSS/deploy/nginx/default.conf new file mode 100644 index 0000000000000000000000000000000000000000..7f0ef561b3be74657b57f921bc118f5243f10803 --- /dev/null +++ b/SAS/TMSS/deploy/nginx/default.conf @@ -0,0 +1,60 @@ +upstream django { + server app:8001; +} + +# upstream websocket { +# server websocket:5678; +# } + +server { + listen 8008; + + add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload" always; + add_header Content-Security-Policy "default-src 'self' data: https://cdnjs.cloudflare.com; style-src 'self' 'unsafe-inline' https://stackpath.bootstrapcdn.com https://fonts.googleapis.com https://cdnjs.cloudflare.com; font-src 'self' data: https://stackpath.bootstrapcdn.com https://fonts.gstatic.com; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdnjs.cloudflare.com; connect-src 'self' ws://localhost:5678 ws://tmss.lofar.eu:5678"; + add_header Referrer-Policy 'strict-origin'; + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_pass http://django; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_redirect off; + proxy_set_header Host tmss.lofar.eu; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + proxy_set_header X-Forwarded-Proto https; + } + + location /ws/ { + try_files $uri @proxy_to_ws; + } + +# location @proxy_to_ws { +# proxy_pass http://websocket; +# +# proxy_http_version 1.1; +# proxy_set_header Upgrade $http_upgrade; +# proxy_set_header Connection upgrade; +# +# proxy_redirect off; +# proxy_set_header Host tmss.lofar.eu; +# proxy_set_header X-Real-IP $remote_addr; +# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +# proxy_set_header X-Forwarded-Host $server_name; +# +# # disables proxy cache completely. does not go well with websockets +# proxy_cache_bypass 1; +# proxy_no_cache 1; +# } + + location /static/ { + alias /opt/lofar/staticfiles/; + } +} diff --git a/SAS/TMSS/doc/img/lofar-deployment.drawio b/SAS/TMSS/doc/img/lofar-deployment.drawio new file mode 100644 index 0000000000000000000000000000000000000000..bcfedd7bb8f9d00dc1b725807371a8d6bb6d89bd --- /dev/null +++ b/SAS/TMSS/doc/img/lofar-deployment.drawio @@ -0,0 +1 @@ +<mxfile host="app.diagrams.net" modified="2023-02-28T15:41:47.199Z" agent="5.0 (X11)" etag="4_uaOqTLayVnjnyzqKXI" version="20.5.1" type="device"><diagram id="cFLyrm104SwQYmhol9yS" name="Page-1">5VzbcqM4EP0aV+0+OIUQNz86cZKdms1myp7szjxNYVBsarDxArl4vn4lQFxa2Ca2Bbj2yUgIIY5a3adbLQ/wzer9PrQ3y4fAJf5AVdz3AZ4MVFW1TIP+sJptVoNGZlqzCD03rUNFxcz7RbJKJat98VwSVRrGQeDH3qZa6QTrNXHiSp0dhsFbtdlz4FffurEXRKiYObYv1v7jufEyrbV0paj/g3iLJX8zUrI7K5s3ziqipe0Gb6UqfDvAN2EQxOnV6v2G+Aw9jkv63N2Ou/nAQrKOmzzw+ul5Hv5C95O/nenjdhTEnz9fD7GWdvNq+y/ZF2ejjbccgjB4WbuE9aIM8PXb0ovJbGM77O4bnXVat4xXPi0hepl1R8KYvO8cKMo/nwoOCVYkDre0SfYAxzYTmaGVld8K/JHCUV2WwTezSjub9EXed4ELvcig+QBMHJU+wYRGVZwQEnEylBqYsCINJizA9Ofj3XhKq0KyCS4XMlUWYqh/cpWrXg6SVrP+jLrlh6TJlYCSGzg/STh0gtUmiMjVlgJwsbhJEy5xNXYOko4Pg6RqdSDpslCqs4CGT1977Xqv9HLBLq8WXuzb86HjpbKWNqDvK7W5YHBlYavvWLiU5v34+jCbdQ8ZtAhmzaKt4xqGLMgMAbJJAtmzRzHqGi4VSljncI36p+OwWgVpqOniMqxjGfIMKO+4BJO3Yp4PI2YLL+LvuEjY5JEzkZ3FqyjqHikDIIVEpNpdhdbhVUjW7pj547Tk+HYUeU4VlyqIFIlw+40Vrkydl79nTZPC5L3cdLLlpXcv/la6/p70oWel4iFW4M+kYyWuEAkAE0K/J3gJHXLY4MV2uCDxQdESZ7g0g3rNBPK6kPh27L1Wx1s3q9kbvgQe/ZJcgDStKkAqdJzT78yeUksxBdgRMJ/CWkyBEDqikmBvS802rEG0Z8AgJsD1QCGzaY+FBOeYnrD8Gzj9H5RqLpxXql6ST7RXOvOlgMrrgHexZymcUay5Jjwo10aXYo0hO4H2tKlYY6vaEdSbZ5JqDMmnpbcg1aJX6JKNH7CBxySKO7duOlzrdX50q9YNiS5iDtkmDNzeQYZr4lotQ2bK053NVecZNaDR1LBrnRp2BQgCPtaww0VoyVGB6qhecneNS4cf+NH2WhvE4fx0+AjhLyh0lTe0SBsaLxrc6aIBtAEfzYYBbcCS2HDHi+Zge+5OSF1kag93Ls0qDj0I/KhiBJYylpg4MYWFVYeePfdJ9zGNY6GTFvxR6xiMsC9A1q9eGKxX7Ct3bAoIz5RAT+/NQ37rN3fuhMSlvXm2H/0uNujpzgMCe1/dR6TUXtvgNn13Lsj99t0hc4V8s7HrDmKjaKRLscEIKCzgu4s2W9nbXpKN7OHuiImhphd9VVyb3SPNSPIRNVb1UU/V8NHYyktBELdQ+hVtMmG2WdeGC6u7EetFsGkEbX3niIkcN83W6ByqozMZ5a1HMb1AQOk4WoRKpOjQhltpd+9Dm3uD87EiHgY/yIpwpzt1FozDHRvQGwn8Q05swgLaoZVYAJYWbb40qdaaSrXaqVSDVA8MbUNTqbYAC8eQI55JqkfwPW2EkXEP2bsF3avOd5o4xS0n6STEUolISL/1ciGTRgK0Blnnx8ZGzIvaoODuyWF9OeoVCzh2hwKyAFn5OhZIMEJt6EtN9Jw6X/wFvd93KqBdhVl3LECIdqSuZq5DexnuOB5ceaq1wa7U/0S1qg1Vq9YpFUUKVFWjI3UrUmA6lyZHuSIF+nJt+FiaGGWpqA3F9r3FOrlh/PvCDrVe++Q5LkpcbTCV4jnsTO+4dn/LXjFNsZ5Hm6SssJTrob3ZNGx/qKykie+ss6Lrc/Q7f/HY2eKzDPLUMjsOTfVqnH5meuamF+Nyi7MsydAS8O/KJ1x6MEi6TMvSuWdDt7HQt/8VJ6Mwe3ya3tz++PQwvqdaYpycfMBjqmwT76r7aTr5A788zr7eT29nP55mt9NUHAeqNjCvqzfMiQz6wwUnERd8TWXfvwn8IEy6wq5OLFdjyjIOg5+kdMdS59gw8tefxJ+GCLimqiLG8nFtwooGdlXPR6CkBaj59SmhvIIx1fMnMI2GYo+QWTeN6sRkB83Pu9HflHDpXRIuHYbSIBdvyrcM9YBTfCa6lR+e42xLb4NtNco+Gk/voaKuVVVpu6paEzT43fTxIVeBlT4LDSgmItVYxsPZSsKj06e/2CpwlkEyAm49gYae7FTPubVVcj1ftUqL/PadHzi2v6QzlFw/22G0pbN/d5Vcs5GWU7BomZ1TvvLWXjvJWL2wCzAhtmbfMq+rmAW4O3U+syDzhOGHVHy9OTH325NO7ULTLOxU53TmiMMziUf74fBMoiw3HJIndFJGFy0W/wWVNi/+Ugvf/gc=</diagram></mxfile> \ No newline at end of file diff --git a/SAS/TMSS/doc/img/lofar-deployment.png b/SAS/TMSS/doc/img/lofar-deployment.png new file mode 100644 index 0000000000000000000000000000000000000000..90f68aadb6031cd6124f554264b9dc39a2d9d3e8 Binary files /dev/null and b/SAS/TMSS/doc/img/lofar-deployment.png differ