TMSS-143: Add build and deploy step for TMSS (retry from TMSS-143 branhc)

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
   - prepare
   - build
   - unit_test
+  - deploy
 #  - integration_test
 
 #
@@ -39,6 +40,42 @@ build_RAServices:
       - build/gnucxx11_opt
       - install/*.ztar
 
+build_TMSS:
+  stage: build
+  image: ci_raservices:latest
+  script:
+    - PACKAGE=TMSS
+    - echo "Building $PACKAGE..."
+    - mkdir install
+    - mkdir -p build/gnucxx11_opt
+    - cd build/gnucxx11_opt
+    - cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DCMAKE_INSTALL_PREFIX=/opt/lofar -DUSE_LOG4CPLUS=false ../..
+    - make -j 6
+    - make DESTDIR=../../install install
+  dependencies:
+    - prepare_RAServices
+  artifacts:
+    expire_in: 6 hours
+    paths:
+      - install
+
+build_TMSS_docker:
+  stage: build
+  script:
+    - cd install
+    - docker build -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss .
+    - cd ..
+    - cd SAS/TMSS/test/oidc/docker-test-mozilla-django-oidc
+    - docker build -t tmss_testprovider:$CI_COMMIT_SHORT_SHA -f dockerfiles/oidc_testprovider .
+    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
+    - docker tag tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
+    - docker push nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
+    - docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
+    - docker push nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
+    - docker logout $CI_NEXUS_REGISTRY  
+  dependencies:
+    - build_TMSS
+
 #
 # UNIT TEST STAGE
 #
@@ -83,4 +120,48 @@ unit_test_RAServices:
 #     paths:
 #       - build/gnucxx11_opt/Testing/Temporary/LastTest.log
 
+deploy-tmss-test:
+  stage: deploy
+  before_script:
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan scu199.control.lofar >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:   
+    - cd SAS/TMSS
+    - ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml down"
+    - scp docker-compose-scu199.yml lofarsys@scu199.control.lofar:~/
+    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
+    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_web:latest"
+    - ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml up -d"
+  dependencies:
+    - build_TMSS_docker
+  when: manual
 
+deploy-tmss-ua:
+  stage: deploy
+  before_script:
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan tmss-ua.control.lofar >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    - cd SAS/TMSS
+    - ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml down"
+    - scp docker-compose-ua.yml lofarsys@tmss-ua.control.lofar:~/
+    - ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
+    - ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA"
+    - ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
+    - ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
+    - ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml up -d"
+  dependencies:
+    - build_TMSS_docker
+  when: manual
+  only:
+    - "master"

CMake/LofarMacros.cmake

@@ -284,6 +284,46 @@ if(NOT DEFINED LOFAR_MACROS_INCLUDED)
   endmacro(lofar_add_data_files)
 
 
+  # --------------------------------------------------------------------------
+  # lofar_add_docker_files([name1 [name2 ..]])
+  #
+  # Add docker files (architecture-independent data) that need to be
+  # installed into the <prefix>/docker directory. Also create a symbolic link
+  # in <build-dir>/docker to each of these files. The file names may contain
+  # a relative(!) path.
+  #
+  # The mentioned files are installed in the same relative path as provided,
+  # that is:
+  #     lofar_add_docker_files(foo/bar)
+  # installs "docker/foo/bar". To override this behaviour use:
+  #     lofar_add_docker_files(foo/bar DESTINATION .)
+  # installs "docker/bar".
+  # --------------------------------------------------------------------------
+  macro(lofar_add_docker_files)
+    string(REGEX REPLACE ";?DESTINATION.*" "" _src_names "${ARGN}")
+    string(REGEX MATCH "DESTINATION;.*" _destination "${ARGN}")
+    string(REGEX REPLACE "^DESTINATION;" "" _destination "${_destination}")
+    string(TOLOWER ${PACKAGE_NAME} lower_package_name)
+    foreach(_src_name ${_src_names})
+      if(_destination MATCHES ".+")
+        get_filename_component(_src_filename ${_src_name} NAME)
+        set(_dest_name ${_destination}/${_src_filename})
+      else(_destination MATCHES ".+")
+        set(_dest_name ${_src_name})
+      endif(_destination MATCHES ".+")
+
+      get_filename_component(_abs_name ${_src_name} ABSOLUTE)
+      get_filename_component(_dest_path ${_dest_name} PATH)
+
+      file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/docker/${_dest_path})
+      execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
+        ${_abs_name} ${CMAKE_BINARY_DIR}/docker/${_dest_name})
+      install(FILES ${_src_name}
+              DESTINATION docker/${_dest_path}
+              COMPONENT ${lower_package_name})
+    endforeach(_src_name ${_src_names})
+  endmacro(lofar_add_docker_files)
+
   # --------------------------------------------------------------------------
   # lofar_add_test(name [source ...] [DEPENDS depend ...])
   #

SAS/TMSS/CMakeLists.txt

@@ -7,4 +7,4 @@ add_subdirectory(bin)
 add_subdirectory(test)
 add_subdirectory(frontend)
 
-
+lofar_add_docker_files(docker-compose-tmss.yml)

SAS/TMSS/docker-compose-scu199.yml

+version: "3"
+services:
+  web:
+    image: nexus.cep4.control.lofar:18080/tmss_django:latest
+    restart: on-failure
+    env_file:
+      - ./.env
+    command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
+    ports:
+      - "8008:8008"
+  testprovider:
+    image: nexus.cep4.control.lofar:18080/tmss_testprovider:latest
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    ports:
+      - "8088:8088"
\ No newline at end of file

SAS/TMSS/docker-compose-tmss.yml

+version: "3"
+services:
+  tmss_web:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile-tmss
+    restart: on-failure
+    env_file:
+      - ../.env
+    command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
+    volumes:
+      - ..:/opt/lofar
+    ports:
+      - "8008:8008"
+    depends_on:
+      - tmss_db
+  tmss_db:
+    image: "postgres:10.3-alpine"
+    restart: unless-stopped
+    env_file:
+      - ../.env
+    ports:
+      - "5435:5432"
+    volumes:
+      - ./postgres/data:/var/lib/postgresql/data
+  tmss_testprovider:
+    build:
+      context: ../
+      dockerfile: docker/oidc/docker-test-mozilla-django-oidc/dockerfiles/oidc_testprovider
+    restart: unless-stopped
+    env_file:
+      - ../.env
+    ports:
+      - "8088:8088"
+    volumes:
+      - ../../../SAS/TMSS/test/oidc/docker-test-mozilla-django-oidc/testprovider:/code

SAS/TMSS/docker-compose-ua.yml

+version: "3"
+services:
+  web:
+    image: nexus.cep4.control.lofar:18080/tmss_django:latest
+    restart: on-failure
+    env_file:
+      - ./.env
+    command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
+    ports:
+      - "8008:8008"
+  testprovider:
+    image: nexus.cep4.control.lofar:18080/tmss_testprovider:latest
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    ports:
+      - "8088:8088"

SAS/TMSS/src/CMakeLists.txt

@@ -33,4 +33,6 @@ python_install(${_py_files}
 
 lofar_add_bin_scripts(migrate_momdb_to_tmss.py)
 
+lofar_add_docker_files(Dockerfile-tmss)
+
 add_subdirectory(tmss)

SAS/TMSS/src/Dockerfile-tmss

+# Use an official Python runtime as a parent image
+FROM python:3.6
+
+RUN apt-get -y update && apt-get -y upgrade
+
+# LOFAR checkout and compile dependencies
+RUN apt-get -y update && apt-get -y install make cmake g++ subversion python3 git
+
+# LOFAR build dependencies
+RUN apt-get -y update && apt-get -y install liblog4cplus-dev python3-dev libldap2-dev libsasl2-dev
+RUN apt-get -y update && apt-get -y install python3-pip && pip3 install django djangorestframework django-filter django-auth-ldap coreapi python-ldap-test django-jsonforms django-json-widget "git+git://github.com/nnseva/django-jsoneditor.git" psycopg2-binary markdown ldap3 drf-yasg flex swagger-spec-validator mozilla_django_oidc
+
+# Adding backend directory to make absolute filepaths consistent across services
+WORKDIR /opt/lofar
+
+ENV LOFARROOT=/opt/lofar
+
+# Add the rest of the code
+COPY ./installed /opt/lofar
+
+# Make port 8000 available for the app
+EXPOSE 8000
+
+RUN sed -i "s/lfr_root=.*/lfr_root=\/opt\/lofar/g" /opt/lofar/lofarinit.sh
+
+# Be sure to use 0.0.0.0 for the host within the Docker container,
+# otherwise the browser won't be able to find it
+# CMD python3 manage.py runserver 0.0.0.0:8000
+# CMD bash -c 'source /opt/lofar/lofarinit.sh && tmss'
+CMD bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8000'
\ No newline at end of file

SAS/TMSS/src/tmss/settings.py

@@ -71,14 +71,12 @@ LOGGING = {
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = '-&$!kx$_0)u1x#zk9w^^81hfssaover2(8wdq_8n8n3u(8=-9n'       # todo: set something new here for production !!!
-
+SECRET_KEY = os.getenv('SECRET_KEY', '-&$!kx$_0)u1x#zk9w^^81hfssaover2(8wdq_8n8n3u(8=-9n')       # todo: set something new here for production !!!
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = []
+DEBUG = bool(int(os.getenv('DEBUG', True)))
 
+ALLOWED_HOSTS = os.getenv('ALLOWED_HOSTS').split(',') if os.getenv('ALLOWED_HOSTS') else []
 
 # Application definition
 
@@ -147,11 +145,36 @@ WSGI_APPLICATION = 'lofar.sas.tmss.tmss.wsgi.application'
 # port = 5432
 # user = <your_tmss_user_name>
 # password = <your_tmss_password>
+DATABASES = {
+    'default': {
+        # Postgres:
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': '',
+        'USER': '',
+        'PASSWORD': '',
+        'HOST': '',
+        'PORT': '',
+    }
+}
 
-creds_name = os.environ.get('TMSS_DBCREDENTIALS', 'tmss')
-django_db_credentials = dbcredentials.DBCredentials().get(creds_name)
-logger.debug("TMSS Django settings: Using dbcreds '%s' for django database: %s",
-            creds_name, django_db_credentials.stringWithHiddenPassword())
+if "POSTGRES_HOST" in  os.environ.keys():
+    DATABASES['default']['HOST'] = os.getenv('POSTGRES_HOST')
+    DATABASES['default']['PORT'] = os.getenv('POSTGRES_PORT')
+    DATABASES['default']['NAME'] = os.getenv('POSTGRES_DB')
+    DATABASES['default']['USER'] = os.getenv('POSTGRES_USER')
+    DATABASES['default']['PASSWORD'] = os.getenv('POSTGRES_PASSWORD')
+else:
+    from lofar.common import dbcredentials, isDevelopmentEnvironment
+
+    creds_name = os.environ.get('TMSS_DBCREDENTIALS', 'tmss')
+    django_db_credentials = dbcredentials.DBCredentials().get(creds_name)
+    logger.debug("TMSS Django settings: Using dbcreds '%s' for django database: %s",
+                 creds_name, django_db_credentials.stringWithHiddenPassword())
+    DATABASES['default']['HOST'] = django_db_credentials.host
+    DATABASES['default']['PORT'] = django_db_credentials.port
+    DATABASES['default']['NAME'] = django_db_credentials.database
+    DATABASES['default']['USER'] = django_db_credentials.user
+    DATABASES['default']['PASSWORD'] = django_db_credentials.password
 
 DATABASES = {
     'default': {
@@ -281,4 +304,4 @@ STATIC_URL = '/static/'
 
 # Setup support for proxy headers
 USE_X_FORWARDED_HOST = True
-SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
\ No newline at end of file

SAS/TMSS/test/CMakeLists.txt

@@ -29,4 +29,3 @@ if(BUILD_TESTING)
 
     # set_tests_properties(t_tmssapp_scheduling_functional PROPERTIES TIMEOUT 300)
 endif()
-