diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d82b10cb6f5668ccde8e5dbe91e85cf876241840..b2961ebe3643660bfb54d01ab6661207635133ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,13 +94,19 @@ prepare_ci_mac_docker_image: build_SCU: stage: build image: ci_scu:$CI_COMMIT_SHORT_SHA + variables: + PACKAGE: SCU script: - echo "Building SCU & TMSSFrontend..." - mkdir -p build/gnucxx11_opt + - mkdir -p build/gnucxx11_opt/install - cd build/gnucxx11_opt - - cmake -DBUILD_PACKAGES="SCU TMSSFrontend" -DUSE_PYTHON=$USE_PYTHON -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../.. + - cmake -DBUILD_PACKAGES="SCU TMSSFrontend" -DUSE_PYTHON=$USE_PYTHON -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false -DCMAKE_INSTALL_PREFIX=/opt/lofar ../.. - make -j $(nproc) - - make install + - make DESTDIR=install install + - cd install/opt/lofar + - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar * + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"' interruptible: true needs: - prepare_ci_scu_docker_image @@ -333,16 +339,10 @@ dockerize_TMSS: allow_failure: true script: - cd build/gnucxx11_opt - - ls * - - docker build --build-arg TMSS_VERSION=$CI_COMMIT_SHORT_SHA -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss . - - cd ../.. - - cd SAS/TMSS/backend/test/oidc/docker-test-mozilla-django-oidc - - docker build -t tmss_testprovider:$CI_COMMIT_SHORT_SHA -f dockerfiles/oidc_testprovider . + - docker build --build-arg TMSS_VERSION=$CI_COMMIT_SHORT_SHA --build-arg TMSS_COPY_DIR=install/opt/lofar -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss . - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY - docker tag tmss_django:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/tmss_django:$CI_COMMIT_SHORT_SHA - docker push $CI_NEXUS_REGISTRY_LOCATION/tmss_django:$CI_COMMIT_SHORT_SHA - - docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/tmss_testprovider:$CI_COMMIT_SHORT_SHA - - docker push $CI_NEXUS_REGISTRY_LOCATION/tmss_testprovider:$CI_COMMIT_SHORT_SHA - docker logout $CI_NEXUS_REGISTRY interruptible: true needs: @@ -444,15 +444,14 @@ deploy-tmss-test: - *prepare_ssh script: - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop TMSS:*" - - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA" - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA" - - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest" - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest" - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest" #- ssh $LOFAR_USER@head.cep4.control.lofar "/data/home/lofarsys/tmss_clean_commissioning_data.sh || true" - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*" needs: - - dockerize_TMSS + - job: dockerize_TMSS + artifacts: false allow_failure: true when: manual @@ -466,9 +465,7 @@ deploy-tmss-ua: - cd SAS/TMSS - ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml down" - scp docker-compose-ua.yml lofarsys@tmss-ua.control.lofar:~/ - - ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA" - ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA" - - ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest" - ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest" - ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml up -d" needs: @@ -493,7 +490,8 @@ deploy-MCU_MAC-test: - *mac_install_package - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6" needs: - - unit_test_MCU_MAC + - job: unit_test_MCU_MAC + artifacts: false allow_failure: true when: manual @@ -510,7 +508,8 @@ deploy-CCU_MAC-test: - *mac_install_package - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6" needs: - - build_CCU_MAC + - job: build_CCU_MAC + artifacts: false allow_failure: true when: manual @@ -527,7 +526,8 @@ deploy-SCU-test: - *mac_install_package - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all" needs: - - integration_test_SCU + - job: integration_test_SCU + artifacts: false allow_failure: true when: manual @@ -566,7 +566,8 @@ deploy-MCU_MAC-prod: environment: name: production needs: - - unit_test_MCU_MAC + - job: unit_test_MCU_MAC + artifacts: false allow_failure: true when: manual only: @@ -587,7 +588,8 @@ deploy-CCU_MAC-prod: environment: name: production needs: - - build_CCU_MAC + - job: build_CCU_MAC + artifacts: false allow_failure: true when: manual @@ -627,7 +629,8 @@ deploy-SCU-prod: environment: name: production needs: - - integration_test_SCU + - job: integration_test_SCU + artifacts: false allow_failure: true when: manual only: @@ -642,16 +645,15 @@ deploy-tmss-prod: - *prepare_ssh script: - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop TMSS:*" - - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA" - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA" - - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest" - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest" - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest" - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*" environment: name: production needs: - - dockerize_TMSS + - job: dockerize_TMSS + artifacts: false allow_failure: true when: manual only: @@ -672,7 +674,8 @@ deploy-LTAIngest-prod: environment: name: production needs: - - integration_test_LTAIngest + - job: integration_test_LTAIngest + artifacts: false allow_failure: true when: manual only: @@ -693,7 +696,8 @@ deploy-LTAIngestTransfer-prod: environment: name: production needs: - - integration_test_LTAIngestTransfer + - job: integration_test_LTAIngestTransfer + artifacts: false allow_failure: true when: manual only: @@ -714,7 +718,8 @@ deploy-lcu-prod: environment: name: production needs: - - build_LCU_MAC + - job: build_LCU_MAC + artifacts: false allow_failure: true when: manual only: @@ -734,7 +739,8 @@ deploy-station-test-prod: environment: name: production needs: - - build_ST_MAC + - job: build_ST_MAC + artifacts: false allow_failure: true when: manual only: diff --git a/Docker/lofar-ci/Dockerfile_ci_scu b/Docker/lofar-ci/Dockerfile_ci_scu index cf83370443256abd8077c0c778a3570ca2a29619..2a5adfafca82ef7e384819c407bf62c42cbc29fd 100644 --- a/Docker/lofar-ci/Dockerfile_ci_scu +++ b/Docker/lofar-ci/Dockerfile_ci_scu @@ -24,7 +24,7 @@ RUN mkdir -p /opt/oracle && \ unzip instantclient-basic-linux.x64-21.1.0.0.0.zip ENV LD_LIBRARY_PATH /opt/oracle/instantclient_21_1:$LD_LIBRARY_PATH -RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil Django==3.0.9 djangorestframework==3.11.1 djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet pyxb==1.2.5 graphviz isodate astropy packaging django-debug-toolbar pymysql astroplan SimpleWebSocketServer websocket_client drf-flex-fields django-property-filter cx_Oracle cachetools +RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil Django==3.0.9 djangorestframework==3.11.1 djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet pyxb==1.2.5 graphviz isodate astropy packaging django-debug-toolbar pymysql astroplan SimpleWebSocketServer websocket_client drf-flex-fields django-property-filter cx_Oracle cachetools gunicorn gevent #Viewflow package RUN pip3 install django-material django-viewflow diff --git a/SAS/TMSS/backend/bin/CMakeLists.txt b/SAS/TMSS/backend/bin/CMakeLists.txt index abdd7f8deb540b41ef5653447082fd7b4cb4ee48..fd516f869005198c445eaece543da6262c4ff032 100644 --- a/SAS/TMSS/backend/bin/CMakeLists.txt +++ b/SAS/TMSS/backend/bin/CMakeLists.txt @@ -6,4 +6,4 @@ lofar_add_bin_scripts(tmss_manage_django) lofar_add_bin_scripts(tmss_simulate_scheduling_unit_run) # supervisord config files -lofar_add_sysconf_files(tmss.ini DESTINATION supervisord.d) +lofar_add_sysconf_files(tmss.ini tmss_nginx.ini DESTINATION supervisord.d) diff --git a/SAS/TMSS/backend/bin/tmss.ini b/SAS/TMSS/backend/bin/tmss.ini index ffe03e35834eb5ed4f08707d7d316c62ff885141..6d11d563d5931f55422120f44ba2b8baea5c34fd 100644 --- a/SAS/TMSS/backend/bin/tmss.ini +++ b/SAS/TMSS/backend/bin/tmss.ini @@ -1,5 +1,5 @@ [program:tmss] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_test_environment --host $TMSS_HOST --public_host $TMSS_HOST --port $TMSS_PORT --schemas --permissions --viewflow_app --DB_ID=TMSS --LDAP_ID=TMSS_LDAP --REST_CLIENT_ID=TMSSClient' +command=docker run --rm -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp:/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -v tmss_staticfiles:/opt/lofar/staticfiles --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys --net "tmss-net" --net-alias "app" nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source $LOFARROOT/lofarinit.sh; gunicorn lofar.sas.tmss.tmss.wsgi --worker-class=gevent --workers=4 --bind=0.0.0.0:8001' priority=100 user=lofarsys stopsignal=INT ; KeyboardInterrupt diff --git a/SAS/TMSS/backend/bin/tmss_collect_static b/SAS/TMSS/backend/bin/tmss_collect_static new file mode 100644 index 0000000000000000000000000000000000000000..3bf612f35c73895f3f9439254be94d8e7cdedf4f --- /dev/null +++ b/SAS/TMSS/backend/bin/tmss_collect_static @@ -0,0 +1 @@ + docker run -it --rm -u root -v tmss_staticfiles:/opt/lofar/staticfiles: nexus.cep4.control.lofar:18080/tmss_django:latest bash -c "source lofarinit.sh; bin/tmss_manage_django collectstatic;" \ No newline at end of file diff --git a/SAS/TMSS/backend/bin/tmss_migrate b/SAS/TMSS/backend/bin/tmss_migrate new file mode 100644 index 0000000000000000000000000000000000000000..0d16b49cab0678c633679832cb316c5ed8cb5415 --- /dev/null +++ b/SAS/TMSS/backend/bin/tmss_migrate @@ -0,0 +1 @@ +docker run -it --rm --env-file /localhome/lofarsys/.lofar/.lofar_env_test nexus.cep4.control.lofar:18080/tmss_django:latest bash -c "source lofarinit.sh; bin/tmss_manage_django migrate;" \ No newline at end of file diff --git a/SAS/TMSS/backend/bin/tmss_nginx.ini b/SAS/TMSS/backend/bin/tmss_nginx.ini new file mode 100644 index 0000000000000000000000000000000000000000..bc30f08f55a859fa4c42c805f567f7b273326cf0 --- /dev/null +++ b/SAS/TMSS/backend/bin/tmss_nginx.ini @@ -0,0 +1,10 @@ +[program:tmss-nginx] +command=docker run -v tmss_staticfiles:/opt/lofar/staticfiles:ro -v /localhome/lofarsys/.lofar/etc/nginx:/etc/nginx/conf.d --net "tmss-net" -p 8008:8008 nginx +priority=100 +user=lofarsys +stopsignal=INT ; KeyboardInterrupt +stopasgroup=true ; bash does not propagate signals +stdout_logfile=%(program_name)s.log +redirect_stderr=true +stderr_logfile=NONE +stdout_logfile_maxbytes=0 diff --git a/SAS/TMSS/backend/services/feedback_handling/bin/tmss_feedback_handling_service.ini b/SAS/TMSS/backend/services/feedback_handling/bin/tmss_feedback_handling_service.ini index 8e085409bf042024533d95c521ab1bf69e2e33a7..d81880fbf344e927b55e5145c61f58da25cf7df8 100644 --- a/SAS/TMSS/backend/services/feedback_handling/bin/tmss_feedback_handling_service.ini +++ b/SAS/TMSS/backend/services/feedback_handling/bin/tmss_feedback_handling_service.ini @@ -1,5 +1,5 @@ [program:tmss_feedback_handling_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_feedback_handling_service' +command=docker run --rm -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_feedback_handling_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/services/precalculations_service/bin/tmss_precalculations_service.ini b/SAS/TMSS/backend/services/precalculations_service/bin/tmss_precalculations_service.ini index 924ce072404b15d8f96bf70b102844af673fbcdc..ca72a7ce54b7c3458709a5e8f9f713513ec0f186 100644 --- a/SAS/TMSS/backend/services/precalculations_service/bin/tmss_precalculations_service.ini +++ b/SAS/TMSS/backend/services/precalculations_service/bin/tmss_precalculations_service.ini @@ -1,5 +1,5 @@ [program:tmss_precalculations_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_precalculations_service' +command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_precalculations_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/services/scheduling/bin/tmss_scheduling_service.ini b/SAS/TMSS/backend/services/scheduling/bin/tmss_scheduling_service.ini index dfbdda1a396c2217d591f9f86f803373b2fe9cb9..28b117e6ba52eaf0352781f64f7f335ca8e6851e 100644 --- a/SAS/TMSS/backend/services/scheduling/bin/tmss_scheduling_service.ini +++ b/SAS/TMSS/backend/services/scheduling/bin/tmss_scheduling_service.ini @@ -1,5 +1,5 @@ [program:tmss_scheduling_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_scheduling_service' +command=docker run --rm -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_scheduling_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/services/slackwebhook/bin/tmss_slack_webhook_service.ini b/SAS/TMSS/backend/services/slackwebhook/bin/tmss_slack_webhook_service.ini index 7aabaad94e0680bc3174d0ece81f34130ba57980..b8042adb041851a046cb062d19e5e2f241ec262b 100644 --- a/SAS/TMSS/backend/services/slackwebhook/bin/tmss_slack_webhook_service.ini +++ b/SAS/TMSS/backend/services/slackwebhook/bin/tmss_slack_webhook_service.ini @@ -1,5 +1,5 @@ [program:tmss_slack_webhook_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_slack_webhook_service' +command=docker run --rm -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_slack_webhook_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/services/tmss_postgres_listener/bin/tmss_postgres_listener_service.ini b/SAS/TMSS/backend/services/tmss_postgres_listener/bin/tmss_postgres_listener_service.ini index c4bd8384e80b3a4e24a2872e9edac2116420c595..dd03e24dd0a28d0265c63e7afc743d20fe281dcf 100644 --- a/SAS/TMSS/backend/services/tmss_postgres_listener/bin/tmss_postgres_listener_service.ini +++ b/SAS/TMSS/backend/services/tmss_postgres_listener/bin/tmss_postgres_listener_service.ini @@ -1,5 +1,5 @@ [program:tmss_postgres_listener_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys -e LOFARENV=TEST nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec tmss_postgres_listener_service' +command=docker run --rm -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys -e LOFARENV=TEST nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec tmss_postgres_listener_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/services/websocket/bin/tmss_websocket_service.ini b/SAS/TMSS/backend/services/websocket/bin/tmss_websocket_service.ini index e1e61b56fd81a333e10b560eb013edb8d065227d..57a1f7db13ca7bb283cf34b22651058d582abf00 100644 --- a/SAS/TMSS/backend/services/websocket/bin/tmss_websocket_service.ini +++ b/SAS/TMSS/backend/services/websocket/bin/tmss_websocket_service.ini @@ -1,5 +1,5 @@ [program:tmss_websocket_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_websocket_service' +command=docker run --rm -p 5678:5678 -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_websocket_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/services/workflow_service/bin/tmss_workflow_service.ini b/SAS/TMSS/backend/services/workflow_service/bin/tmss_workflow_service.ini index fefadd9e0e0b2985dafd3302512389392fc06abf..0054ad6d783dedf0b078ac348f2726730ed9d4aa 100644 --- a/SAS/TMSS/backend/services/workflow_service/bin/tmss_workflow_service.ini +++ b/SAS/TMSS/backend/services/workflow_service/bin/tmss_workflow_service.ini @@ -1,5 +1,5 @@ [program:tmss_workflow_service] -command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_workflow_service' +command=docker run --rm -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys --env-file /localhome/lofarsys/.lofar/.lofar_env_test -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_workflow_service' user=lofarsys stopsignal=INT ; KeyboardInterrupt stopasgroup=true ; bash does not propagate signals diff --git a/SAS/TMSS/backend/src/Dockerfile-tmss b/SAS/TMSS/backend/src/Dockerfile-tmss index f742be131dbadcd4eb4568af2c81ba506960463c..d471d73bac7fe77a42b2035b7c9016e495067614 100644 --- a/SAS/TMSS/backend/src/Dockerfile-tmss +++ b/SAS/TMSS/backend/src/Dockerfile-tmss @@ -8,8 +8,11 @@ # docker build [-t image_name:tag] -f docker/Dockerfile-tmss . # ARG TMSS_VERSION=latest + FROM ci_scu:$TMSS_VERSION +ARG TMSS_COPY_DIR="./installed" + USER lofarsys RUN mkdir -p /opt/lofar @@ -18,8 +21,9 @@ WORKDIR /opt/lofar ENV LOFARROOT=/opt/lofar +RUN echo "Copy from $TMSS_COPY_DIR" # Add the rest of the code -COPY --chown=lofarsys:lofarsys ./installed /opt/lofar +COPY --chown=lofarsys:lofarsys ${TMSS_COPY_DIR} /opt/lofar # docker only chowns the copied dir, not the full tree RUN chown -R lofarsys:lofarsys /opt/lofar diff --git a/SAS/TMSS/backend/src/tmss/settings.py b/SAS/TMSS/backend/src/tmss/settings.py index 5ac185475abc07af34fba570389d8c7e9ad4a044..13341bfb6bd73990c79f0486de2e81fde921dadf 100644 --- a/SAS/TMSS/backend/src/tmss/settings.py +++ b/SAS/TMSS/backend/src/tmss/settings.py @@ -252,7 +252,28 @@ if "TMSS_LDAPCREDENTIALS" in os.environ.keys(): } AUTHENTICATION_BACKENDS += ('django_auth_ldap.backend.LDAPBackend',) +elif "LDAP_SERVER_URI" in os.environ.keys(): + # plain LDAP + import ldap + logger.info('Authenticating against LDAP is enabled') + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append('rest_framework.authentication.BasicAuthentication') + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append('rest_framework.authentication.SessionAuthentication') + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append('rest_framework.authentication.TokenAuthentication') + REST_FRAMEWORK['DEFAULT_PERMISSION_CLASSES'].append('rest_framework.permissions.IsAuthenticated') + # LDAP + AUTH_LDAP_GLOBAL_OPTIONS = {ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER} # cert still expired? + AUTH_LDAP_CONNECTION_OPTIONS = {ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER} # cert still expired? + AUTH_LDAP_SERVER_URI = os.environ.get('LDAP_SERVER_URI', '') + AUTH_LDAP_USER_DN_TEMPLATE = "cn=%(user)s,ou=Users,o=lofar,c=eu" + + AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" + } + + AUTHENTICATION_BACKENDS += ('django_auth_ldap.backend.LDAPBackend',) if "OIDC_RP_CLIENT_ID" in os.environ.keys(): logger.info('Authenticating against Keycloak is enabled') @@ -322,7 +343,7 @@ USE_TZ = False # We don't want timezone support since everything is UTC anyway # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ STATIC_URL = '/static/' -STATIC_ROOT = '/staticfiles/' +STATIC_ROOT = '/opt/lofar/staticfiles/' diff --git a/SAS/TMSS/backend/src/tmss/wsgi.py b/SAS/TMSS/backend/src/tmss/wsgi.py index 56a1bba520d9ce43c0b31c68560a34fc102e3857..6012782503799a119a0f3581d60eca6e09d65c74 100644 --- a/SAS/TMSS/backend/src/tmss/wsgi.py +++ b/SAS/TMSS/backend/src/tmss/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tmss.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lofar.sas.tmss.tmss.settings") application = get_wsgi_application()