Commit 1cbe57dd authored by Auke L. Klazema's avatar Auke L. Klazema

TMSS-143: Add build and deploy step for TMSS (retry from TMSS-143 branhc)

parent a37a10ae
......@@ -2,6 +2,7 @@ stages:
- prepare
- build
- unit_test
- deploy
# - integration_test
#
......@@ -39,6 +40,42 @@ build_RAServices:
- build/gnucxx11_opt
- install/*.ztar
build_TMSS:
stage: build
image: ci_raservices:latest
script:
- PACKAGE=TMSS
- echo "Building $PACKAGE..."
- mkdir install
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DCMAKE_INSTALL_PREFIX=/opt/lofar -DUSE_LOG4CPLUS=false ../..
- make -j 6
- make DESTDIR=../../install install
dependencies:
- prepare_RAServices
artifacts:
expire_in: 6 hours
paths:
- install
build_TMSS_docker:
stage: build
script:
- cd install
- docker build -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss .
- cd ..
- cd SAS/TMSS/test/oidc/docker-test-mozilla-django-oidc
- docker build -t tmss_testprovider:$CI_COMMIT_SHORT_SHA -f dockerfiles/oidc_testprovider .
- docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
- docker tag tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
- docker push nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
- docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
- docker push nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
- docker logout $CI_NEXUS_REGISTRY
dependencies:
- build_TMSS
#
# UNIT TEST STAGE
#
......@@ -83,4 +120,48 @@ unit_test_RAServices:
# paths:
# - build/gnucxx11_opt/Testing/Temporary/LastTest.log
deploy-tmss-test:
stage: deploy
before_script:
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan scu199.control.lofar >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- cd SAS/TMSS
- ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml down"
- scp docker-compose-scu199.yml lofarsys@scu199.control.lofar:~/
- ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
- ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_web:latest"
- ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml up -d"
dependencies:
- build_TMSS_docker
when: manual
deploy-tmss-ua:
stage: deploy
before_script:
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan tmss-ua.control.lofar >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- cd SAS/TMSS
- ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml down"
- scp docker-compose-ua.yml lofarsys@tmss-ua.control.lofar:~/
- ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
- ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA"
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml up -d"
dependencies:
- build_TMSS_docker
when: manual
only:
- "master"
......@@ -284,6 +284,46 @@ if(NOT DEFINED LOFAR_MACROS_INCLUDED)
endmacro(lofar_add_data_files)
# --------------------------------------------------------------------------
# lofar_add_docker_files([name1 [name2 ..]])
#
# Add docker files (architecture-independent data) that need to be
# installed into the <prefix>/docker directory. Also create a symbolic link
# in <build-dir>/docker to each of these files. The file names may contain
# a relative(!) path.
#
# The mentioned files are installed in the same relative path as provided,
# that is:
# lofar_add_docker_files(foo/bar)
# installs "docker/foo/bar". To override this behaviour use:
# lofar_add_docker_files(foo/bar DESTINATION .)
# installs "docker/bar".
# --------------------------------------------------------------------------
macro(lofar_add_docker_files)
string(REGEX REPLACE ";?DESTINATION.*" "" _src_names "${ARGN}")
string(REGEX MATCH "DESTINATION;.*" _destination "${ARGN}")
string(REGEX REPLACE "^DESTINATION;" "" _destination "${_destination}")
string(TOLOWER ${PACKAGE_NAME} lower_package_name)
foreach(_src_name ${_src_names})
if(_destination MATCHES ".+")
get_filename_component(_src_filename ${_src_name} NAME)
set(_dest_name ${_destination}/${_src_filename})
else(_destination MATCHES ".+")
set(_dest_name ${_src_name})
endif(_destination MATCHES ".+")
get_filename_component(_abs_name ${_src_name} ABSOLUTE)
get_filename_component(_dest_path ${_dest_name} PATH)
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/docker/${_dest_path})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
${_abs_name} ${CMAKE_BINARY_DIR}/docker/${_dest_name})
install(FILES ${_src_name}
DESTINATION docker/${_dest_path}
COMPONENT ${lower_package_name})
endforeach(_src_name ${_src_names})
endmacro(lofar_add_docker_files)
# --------------------------------------------------------------------------
# lofar_add_test(name [source ...] [DEPENDS depend ...])
#
......
......@@ -7,4 +7,4 @@ add_subdirectory(bin)
add_subdirectory(test)
add_subdirectory(frontend)
lofar_add_docker_files(docker-compose-tmss.yml)
version: "3"
services:
web:
image: nexus.cep4.control.lofar:18080/tmss_django:latest
restart: on-failure
env_file:
- ./.env
command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
ports:
- "8008:8008"
testprovider:
image: nexus.cep4.control.lofar:18080/tmss_testprovider:latest
restart: unless-stopped
env_file:
- ./.env
ports:
- "8088:8088"
\ No newline at end of file
version: "3"
services:
tmss_web:
build:
context: ..
dockerfile: docker/Dockerfile-tmss
restart: on-failure
env_file:
- ../.env
command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
volumes:
- ..:/opt/lofar
ports:
- "8008:8008"
depends_on:
- tmss_db
tmss_db:
image: "postgres:10.3-alpine"
restart: unless-stopped
env_file:
- ../.env
ports:
- "5435:5432"
volumes:
- ./postgres/data:/var/lib/postgresql/data
tmss_testprovider:
build:
context: ../
dockerfile: docker/oidc/docker-test-mozilla-django-oidc/dockerfiles/oidc_testprovider
restart: unless-stopped
env_file:
- ../.env
ports:
- "8088:8088"
volumes:
- ../../../SAS/TMSS/test/oidc/docker-test-mozilla-django-oidc/testprovider:/code
version: "3"
services:
web:
image: nexus.cep4.control.lofar:18080/tmss_django:latest
restart: on-failure
env_file:
- ./.env
command: bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8008'
ports:
- "8008:8008"
testprovider:
image: nexus.cep4.control.lofar:18080/tmss_testprovider:latest
restart: unless-stopped
env_file:
- ./.env
ports:
- "8088:8088"
......@@ -33,4 +33,6 @@ python_install(${_py_files}
lofar_add_bin_scripts(migrate_momdb_to_tmss.py)
lofar_add_docker_files(Dockerfile-tmss)
add_subdirectory(tmss)
# Use an official Python runtime as a parent image
FROM python:3.6
RUN apt-get -y update && apt-get -y upgrade
# LOFAR checkout and compile dependencies
RUN apt-get -y update && apt-get -y install make cmake g++ subversion python3 git
# LOFAR build dependencies
RUN apt-get -y update && apt-get -y install liblog4cplus-dev python3-dev libldap2-dev libsasl2-dev
RUN apt-get -y update && apt-get -y install python3-pip && pip3 install django djangorestframework django-filter django-auth-ldap coreapi python-ldap-test django-jsonforms django-json-widget "git+git://github.com/nnseva/django-jsoneditor.git" psycopg2-binary markdown ldap3 drf-yasg flex swagger-spec-validator mozilla_django_oidc
# Adding backend directory to make absolute filepaths consistent across services
WORKDIR /opt/lofar
ENV LOFARROOT=/opt/lofar
# Add the rest of the code
COPY ./installed /opt/lofar
# Make port 8000 available for the app
EXPOSE 8000
RUN sed -i "s/lfr_root=.*/lfr_root=\/opt\/lofar/g" /opt/lofar/lofarinit.sh
# Be sure to use 0.0.0.0 for the host within the Docker container,
# otherwise the browser won't be able to find it
# CMD python3 manage.py runserver 0.0.0.0:8000
# CMD bash -c 'source /opt/lofar/lofarinit.sh && tmss'
CMD bash -c 'source /opt/lofar/lofarinit.sh && python3 lib/python3.6/site-packages/lofar/sas/tmss/manage.py runserver 0.0.0.0:8000'
\ No newline at end of file
......@@ -71,14 +71,12 @@ LOGGING = {
BASE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '-&$!kx$_0)u1x#zk9w^^81hfssaover2(8wdq_8n8n3u(8=-9n' # todo: set something new here for production !!!
SECRET_KEY = os.getenv('SECRET_KEY', '-&$!kx$_0)u1x#zk9w^^81hfssaover2(8wdq_8n8n3u(8=-9n') # todo: set something new here for production !!!
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
DEBUG = bool(int(os.getenv('DEBUG', True)))
ALLOWED_HOSTS = os.getenv('ALLOWED_HOSTS').split(',') if os.getenv('ALLOWED_HOSTS') else []
# Application definition
......@@ -147,11 +145,36 @@ WSGI_APPLICATION = 'lofar.sas.tmss.tmss.wsgi.application'
# port = 5432
# user = <your_tmss_user_name>
# password = <your_tmss_password>
DATABASES = {
'default': {
# Postgres:
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': '',
'USER': '',
'PASSWORD': '',
'HOST': '',
'PORT': '',
}
}
creds_name = os.environ.get('TMSS_DBCREDENTIALS', 'tmss')
django_db_credentials = dbcredentials.DBCredentials().get(creds_name)
logger.debug("TMSS Django settings: Using dbcreds '%s' for django database: %s",
creds_name, django_db_credentials.stringWithHiddenPassword())
if "POSTGRES_HOST" in os.environ.keys():
DATABASES['default']['HOST'] = os.getenv('POSTGRES_HOST')
DATABASES['default']['PORT'] = os.getenv('POSTGRES_PORT')
DATABASES['default']['NAME'] = os.getenv('POSTGRES_DB')
DATABASES['default']['USER'] = os.getenv('POSTGRES_USER')
DATABASES['default']['PASSWORD'] = os.getenv('POSTGRES_PASSWORD')
else:
from lofar.common import dbcredentials, isDevelopmentEnvironment
creds_name = os.environ.get('TMSS_DBCREDENTIALS', 'tmss')
django_db_credentials = dbcredentials.DBCredentials().get(creds_name)
logger.debug("TMSS Django settings: Using dbcreds '%s' for django database: %s",
creds_name, django_db_credentials.stringWithHiddenPassword())
DATABASES['default']['HOST'] = django_db_credentials.host
DATABASES['default']['PORT'] = django_db_credentials.port
DATABASES['default']['NAME'] = django_db_credentials.database
DATABASES['default']['USER'] = django_db_credentials.user
DATABASES['default']['PASSWORD'] = django_db_credentials.password
DATABASES = {
'default': {
......@@ -281,4 +304,4 @@ STATIC_URL = '/static/'
# Setup support for proxy headers
USE_X_FORWARDED_HOST = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
\ No newline at end of file
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
......@@ -29,4 +29,3 @@ if(BUILD_TESTING)
# set_tests_properties(t_tmssapp_scheduling_functional PROPERTIES TIMEOUT 300)
endif()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment