Commit b98bb860 authored by Jorrit Schaap's avatar Jorrit Schaap

Merge branch 'TMSS-163' into 'TMSS-146'

Resolve TMSS-163

See merge request ro/lofar!110
parents 95011e36 6307c8f2
stages:
- prepare
- build
- dockerize
- unit_test
- integration_test
- dockerize
- deploy
# - integration_test
#
# PREPARE STAGE
#
prepare_RAServices:
prepare_ci_sas_docker_image:
stage: prepare
script:
- docker build -t ci_raservices -f SubSystems/RAServices/Dockerfile .
- docker build -t ci_base -f Docker/lofar-ci/Dockerfile_ci_base .
- docker build -t ci_sas -f Docker/lofar-ci/Dockerfile_ci_sas .
prepare_ci_sas_docker_image:
prepare_ci_lta_docker_image:
stage: prepare
script:
- docker build -t ci_base -f Docker/lofar-ci/Dockerfile_ci_base .
- docker build -t ci_sas -f Docker/lofar-ci/Dockerfile_ci_sas .
- docker build -t ci_lta -f Docker/lofar-ci/Dockerfile_ci_lta .
#TODO: make proper MAC docker image with WinCC (rpm packages from mcu001)
#prepare_ci_mac_docker_image:
# stage: prepare
# script:
# - docker build -t ci_base -f Docker/lofar-ci/Dockerfile_ci_base .
# - docker build -t ci_mac -f Docker/lofar-ci/Dockerfile_ci_mac .
#
# BUILD STAGE
#
build_RAServices:
build_TMSS:
stage: build
image: ci_raservices:latest
image: ci_sas:latest
script:
- PACKAGE=RAServices
- PACKAGE=TMSS
- echo "Building $PACKAGE..."
- mkdir install
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DCMAKE_INSTALL_PREFIX=/opt/lofar -DUSE_LOG4CPLUS=false ../..
- make -j 6
- make DESTDIR=../../install install
- cd ../../install/opt/lofar
- tar --ignore-failed-read --exclude=include -czf ../../RAServices_$CI_COMMIT_SHORT_SHA.ztar *
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../..
- make -j 8
- make install
dependencies:
- prepare_RAServices
- prepare_ci_sas_docker_image
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
- install/*.ztar
build_TMSS:
build_RAServices:
stage: build
image: ci_sas:latest
script:
- PACKAGE=TMSS
- PACKAGE=RAServices
- echo "Building $PACKAGE..."
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../..
- make -j 6
- make -j 8
- make install
dependencies:
- prepare_ci_sas_docker_image
......@@ -65,6 +69,126 @@ build_TMSS:
paths:
- build/gnucxx11_opt
build_LTAIngest:
stage: build
image: ci_lta:latest
script:
- PACKAGE=LTAIngest
- echo "Building $PACKAGE..."
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DUSE_LOG4CPLUS=false ../..
- make -j 8
- make install
dependencies:
- prepare_ci_lta_docker_image
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
# TODO: enable when prepare_ci_mac_docker_image is fixed
#build_MCU_MAC:
# stage: build
# image: ci_mac:latest
# script:
# - PACKAGE=MCU_MAC
# - echo "Building $PACKAGE..."
# - mkdir -p build/gnucxx11_opt
# - cd build/gnucxx11_opt
# - cmake -DBUILD_PACKAGES=$PACKAGE -DUSE_LOG4CPLUS=false ../..
# - make -j 8
# - make install
# dependencies:
# - prepare_ci_mac_docker_image
# artifacts:
# expire_in: 6 hours
# paths:
# - build/gnucxx11_opt
#
# UNIT TEST STAGE
#
unit_test_TMSS:
stage: unit_test
image: ci_sas:latest
script:
- PACKAGE=TMSS
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
dependencies:
- build_TMSS
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
unit_test_RAServices:
stage: unit_test
image: ci_sas:latest
script:
- PACKAGE=RAServices
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_RAServices
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
unit_test_LTAIngest:
stage: unit_test
image: ci_lta:latest
script:
- PACKAGE=LTAIngest
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_LTAIngest
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
# TODO: enable when build_MCU_MAC is fixed
#unit_test_MCU_MAC:
# stage: unit_test
# image: ci_mac:latest
# script:
# - PACKAGE=MCU_MAC
# - echo "Testing $PACKAGE..."
# - cd build/gnucxx11_opt
# - SKIP_INTEGRATION_TESTS=true ctest
# dependencies:
# - build_MCU_MAC
# artifacts:
# name: unit-test-report
# when: always
# paths:
# - build/gnucxx11_opt/Testing/Temporary/LastTest.log
#
# DOCKERIZE
#
......@@ -86,51 +210,85 @@ dockerize_TMSS:
- docker logout $CI_NEXUS_REGISTRY
dependencies:
- build_TMSS
- unit_test_TMSS
- integration_test_TMSS
#
# UNIT TEST STAGE
# INTEGRATION TEST STAGE
#
unit_test_RAServices:
stage: unit_test
image: ci_raservices:latest
integration_test_TMSS:
stage: integration_test
image: ci_sas:latest
script:
- PACKAGE=TMSS
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_TMSS
artifacts:
name: integration-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
integration_test_RAServices:
stage: integration_test
image: ci_sas:latest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
script:
- PACKAGE=RAServices
- echo "Testing $PACKAGE..."
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
dependencies:
- build_RAServices
artifacts:
name: unit-test-report
name: integration-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
integration_test_LTAIngest:
stage: integration_test
image: ci_lta:latest
script:
- PACKAGE=LTAIngest
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_LTAIngest
artifacts:
name: integration-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
#
# INTEGRATION TEST STAGE
# DEPLOY STAGE
#
# integration_test_RAServices:
# stage: integration_test
# image: ci_raservices:latest
# services:
# - rabbitmq:latest
# variables:
# LOFAR_DEFAULT_BROKER: "rabbitmq"
# script:
# - PACKAGE=RAServices
# - echo "Testing $PACKAGE..."
# - cd build/gnucxx11_opt
# - SKIP_UNIT_TESTS=true ctest
# dependencies:
# - build_RAServices
# artifacts:
# name: integration-test-report
# when: always
# paths:
# - build/gnucxx11_opt/Testing/Temporary/LastTest.log
deploy-tmss-test:
stage: deploy
before_script:
......@@ -151,7 +309,7 @@ deploy-tmss-test:
- ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml up -d"
dependencies:
- build_TMSS
- integration_test_TMSS
when: manual
deploy-tmss-ua:
......@@ -174,7 +332,7 @@ deploy-tmss-ua:
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml up -d"
dependencies:
- build_TMSS
- integration_test_TMSS
when: manual
only:
- "master"
......@@ -177,7 +177,7 @@ function(npm_install NPM_PACKAGE_SPECIFICATION)
add_custom_command(
TARGET packing_javascript_files_${PACKAGE_NAME}
COMMAND npm run build
COMMAND CI=false npm run build
DEPENDS "${INSTALLED_SOURCE_FILES}" "${INSTALLED_PUBLIC_FILES}"
WORKING_DIRECTORY "${NPM_BINARY_DIR}"
COMMENT "Packing javascript files for ${PACKAGE_NAME} into ${NPM_BINARY_DIR}/build for deployment")
......
......@@ -3,10 +3,13 @@
#
# base
#
FROM centos:7
FROM centos:centos7.6.1810
RUN yum -y groupinstall 'Development Tools' && \
yum -y install epel-release && \
yum -y install cmake log4cplus-devel python3 python3-devel python3-pip
yum -y install cmake gcc git log4cplus-devel python3 python3-devel python3-pip which wget curl atop
RUN pip3 install kombu requests coverage python-qpid-proton
RUN adduser lofarsys
......@@ -5,6 +5,16 @@
#
FROM ci_base:latest
RUN echo "Installing packages for LTA..." && \
yum -y install postgresql-devel && \
pip3 install kombu requests pysimplesoap mysql-connector psycopg2 flask
RUN echo "Installing packages for LTA..."
# see https://www.postgresql.org/download/linux/redhat/ on how to install postgresql-server > 9.2 on centos7
RUN yum erase -y postgresql postgresql-server postgresql-devel && \
yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \
yum install -y postgresql96 postgresql96-server postgresql96-devel && \
cd /bin && ln -s /usr/pgsql-9.6/bin/initdb && ln -s /usr/pgsql-9.6/bin/postgres
ENV PATH /usr/pgsql-9.6/bin:$PATH
RUN pip3 install kombu requests pysimplesoap mysql-connector flask lxml jsonschema psycopg2 testing.postgresql
RUN adduser ingest
USER ingest
\ No newline at end of file
......@@ -5,7 +5,7 @@
#
FROM ci_base:latest
RUN echo "Installing packages for LCU..." && \
RUN echo "Installing packages for MAC..." && \
yum -y install readline-devel boost-python36-devel hdf5-devel blas-devel lapack-devel cfitsio-devel wcslib-devel autogen postgresql-devel cmake3 libpqxx-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools unittest-cpp-devel && \
pip3 install psycopg2 testing.postgresql lxml mock numpy kombu requests python-dateutil fabric
......@@ -14,14 +14,14 @@ RUN echo "Installing Casacore..." && \
mkdir /casacore/build/ && \
cd /casacore/build/ && \
cmake -DCMAKE_INSTALL_PREFIX=/opt/casacore -DBUILD_PYTHON3=ON -DBUILD_PYTHON=OFF -DPYTHON_EXECUTABLE=/usr/bin/python3 -DUSE_OPENMP=ON -DUSE_FFTW3=TRUE -DUSE_HDF5=ON -DCMAKE_BUILD_TYPE=Release .. && \
make && \
make -j 8 && \
make install
RUN echo "Installing Blitz++" && \
cd /
cd / && \
git clone --depth 1 https://github.com/blitzpp/blitz.git && \
mkdir -p /blitz/build && \
cd /blitz/build && \
cmake3 --prefix=/opt/blitz/ .. && \
make lib && \
cmake --prefix=/opt/blitz/ .. && \
make -j 8 lib && \
make install
\ No newline at end of file
......@@ -6,10 +6,20 @@
FROM ci_base:latest
RUN echo "Installing packages for SAS..." && \
yum -y install postgresql-devel openldap-devel readline-devel qpid-cpp-server qpid-cpp-client-devel qpid-tools libpqxx-devel java-devel qt-devel autogen boost-python36-devel nodejs npm && \
pip3 install kombu psycopg2 requests lxml xmljson pygcn python-dateutil django djangorestframework djangorestframework-xml django-auth-ldap mysql-connector testing.mysqld testing.postgresql
yum install -y log4cplus log4cplus-devel python3 python3-libs python3-devel boost readline-devel boost-devel binutils-devel boost-python36 boost-python36-devel gettext which openldap-devel npm nodejs git java-11-openjdk python-twisted-core
# see https://www.postgresql.org/download/linux/redhat/ on how to install postgresql-server > 9.2 on centos7
RUN yum erase -y postgresql postgresql-server postgresql-devel && \
yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \
yum install -y postgresql96 postgresql96-server postgresql96-devel && \
cd /bin && ln -s /usr/pgsql-9.6/bin/initdb && ln -s /usr/pgsql-9.6/bin/postgres
ENV PATH /usr/pgsql-9.6/bin:$PATH
RUN pip3 install cython kombu lxml requests pygcn xmljson mysql-connector-python python-dateutil django djangorestframework djangorestframework-xml ldap==1.0.2 flask fabric coverage python-qpid-proton PyGreSQL numpy h5py psycopg2 testing.postgresql Flask-Testing scipy Markdown django-filter python-ldap python-ldap-test ldap3 djangorestframework django-jsonforms django-json-widget django-jsoneditor drf-yasg flex swagger-spec-validator django-auth-ldap mozilla-django-oidc jsonschema comet
RUN npm install -g npx && \
npm install -g n && \
n stable && \
npm install -g serve
USER lofarsys
\ No newline at end of file
......@@ -19,8 +19,8 @@ DEFAULT_BROKER = "scu001.control.lofar" if isProductionEnvironment() else \
if 'LOFAR_DEFAULT_BROKER' in os.environ.keys():
DEFAULT_BROKER = os.environ.get('LOFAR_DEFAULT_BROKER')
DEFAULT_USER = "guest"
DEFAULT_PASSWORD = "guest"
DEFAULT_USER = os.environ.get('RABBITMQ_DEFAULT_USER', 'guest')
DEFAULT_PASSWORD = os.environ.get('RABBITMQ_DEFAULT_PASS', 'guest')
if isProductionEnvironment() or isTestEnvironment():
# import the user and password from RabbitMQ 'db'credentials
......@@ -35,12 +35,15 @@ if isProductionEnvironment() or isTestEnvironment():
# dynamically determine port where RabbitMQ server runs by trying to connect
DEFAULT_PORT = -1
def broker_url(hostname: str=DEFAULT_BROKER, port: int=DEFAULT_PORT, userid: str=DEFAULT_USER, password :str=DEFAULT_PASSWORD) -> str:
return 'amqp://%s:%s@%s:%d//' % (userid, password, hostname, port)
for port in [5672, 5675]:
try:
logger.debug("trying to connect to broker: hostname=%s port=%s userid=%s password=***",
DEFAULT_BROKER, port, DEFAULT_USER)
with kombu.Connection(hostname=DEFAULT_BROKER, port=port, userid=DEFAULT_USER, password=DEFAULT_PASSWORD,
max_retries=0, connect_timeout=1) as connection:
with kombu.Connection(broker_url(port=port), max_retries=0, connect_timeout=1, ) as connection:
connection.connect()
DEFAULT_PORT = port
logger.info("detected rabbitmq broker to which we can connect with hostname=%s port=%s userid=%s password=***",
......
......@@ -204,7 +204,7 @@ logger = logging.getLogger(__name__)
from lofar.messaging.exceptions import *
from lofar.messaging import adaptNameToEnvironment
from lofar.messaging.messages import *
from lofar.messaging.config import DEFAULT_BROKER, DEFAULT_BUSNAME, DEFAULT_PORT, DEFAULT_USER, DEFAULT_PASSWORD
from lofar.messaging.config import DEFAULT_BROKER, DEFAULT_BUSNAME, DEFAULT_PORT, DEFAULT_USER, DEFAULT_PASSWORD, broker_url
from lofar.common.threading_utils import TimeoutLock
from lofar.common.util import program_name
from lofar.common.util import is_empty_function
......@@ -222,7 +222,7 @@ def can_connect_to_broker(broker: str=DEFAULT_BROKER, port: int=DEFAULT_PORT) ->
try:
logger.debug("trying to connect to broker: hostname=%s port=%s userid=%s password=***",
broker, port, DEFAULT_USER)
with kombu.Connection(hostname=broker, port=port, userid=DEFAULT_USER, password=DEFAULT_PASSWORD,
with kombu.Connection(broker_url(hostname=broker, port=port, userid=DEFAULT_USER, password=DEFAULT_PASSWORD),
max_retries=0, connect_timeout=1) as connection:
connection.connect()
logger.debug("can connect to broker with hostname=%s port=%s userid=%s password=***",
......@@ -244,7 +244,7 @@ def create_exchange(name: str, durable: bool=True, broker: str=DEFAULT_BROKER, l
:return True if created, False if not-created (because it already exists)
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
exchange = kombu.Exchange(name, durable=durable, type='topic')
try:
exchange.declare(channel=connection.default_channel, passive=True)
......@@ -266,7 +266,7 @@ def delete_exchange(name: str, broker: str=DEFAULT_BROKER, log_level=logging.DEB
:return True if deleted, False if not-deleted (because it does not exist)
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
exchange = kombu.Exchange(name, channel=connection)
try:
exchange.declare(channel=connection.default_channel, passive=True)
......@@ -286,7 +286,7 @@ def exchange_exists(name: str, broker: str=DEFAULT_BROKER) -> bool:
:return True if it exists, False if not.
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
exchange = kombu.Exchange(name, channel=connection)
try:
exchange.declare(channel=connection.default_channel, passive=True)
......@@ -309,7 +309,7 @@ def create_queue(name: str, durable: bool=True, broker: str=DEFAULT_BROKER, log_
:return True if created, False if not-created (because it already exists)
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
queue = kombu.Queue(name,
durable=durable,
auto_delete=auto_delete,
......@@ -335,7 +335,7 @@ def delete_queue(name: str, broker: str=DEFAULT_BROKER, log_level=logging.DEBUG)
:return True if deleted, False if not-deleted (because it does not exist)
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
queue = kombu.Queue(name, no_declare=True, channel=connection)
try:
queue.queue_declare(channel=connection.default_channel, passive=True)
......@@ -355,7 +355,7 @@ def queue_exists(name: str, broker: str=DEFAULT_BROKER) -> bool:
:return True if it exists, False if not.
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
queue = kombu.Queue(name, no_declare=True, channel=connection)
try:
queue.queue_declare(channel=connection.default_channel, passive=True)
......@@ -389,7 +389,7 @@ def create_binding(exchange: str, queue: str, routing_key: str='#', durable: boo
:param log_level: optional logging level (to add/reduce spamming)
"""
try:
with kombu.Connection(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD) as connection:
with kombu.Connection(broker_url(hostname=broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)) as connection:
kombu_exchange = kombu.Exchange(exchange, durable=durable, type='topic', no_declare=True)
kombu_queue = kombu.Queue(queue, exchange=kombu_exchange, routing_key=routing_key, durable=durable, no_declare=True)
if not kombu_queue.is_bound:
......@@ -483,7 +483,7 @@ class _AbstractBus:
return
logger.debug("[%s] Connecting to broker: %s", self.__class__.__name__, self.broker)
self._connection = kombu.Connection(hostname=self.broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD)
self._connection = kombu.Connection(broker_url(hostname=self.broker, port=DEFAULT_PORT, userid=DEFAULT_USER, password=DEFAULT_PASSWORD))
self._connection.connect()
logger.debug("[%s] Connected to broker: %s (%s)", self.__class__.__name__, self.broker, self.connection_name)
......
......@@ -374,7 +374,7 @@ class FromBusInitFailed(unittest.TestCase):
Connecting to broker on wrong port must raise MessageBusError
"""
with self.assertRaisesRegex(MessageBusError, ".*failed to resolve broker hostname"):
with FromBus("fake" + self.test_queue.address, broker="localhost:4"):
with FromBus("fake" + self.test_queue.address, broker="fdjsafhdjlahflaieoruieow"):
pass
......@@ -445,7 +445,7 @@ class ToBusInitFailed(unittest.TestCase):
Connecting to broker on wrong port must raise MessageBusError
"""
with self.assertRaisesRegex(MessageBusError, ".*failed to resolve broker hostname"):
with ToBus(self.test_exchange.address, broker="localhost:4"):
with ToBus(self.test_exchange.address, broker="fhjlahfowuefohwaueif"):
pass
......@@ -886,8 +886,17 @@ class ReconnectOnConnectionLossTests(unittest.TestCase):
self.tmp_exchange.close()
self.assertFalse(exchange_exists(tmp_exchange_address))
def _can_connect_to_rabbitmq_admin_site(self, hostname: str):
try:
url = 'http://%s:15672/api' % (hostname,)
return requests.get(url, auth=(DEFAULT_USER, DEFAULT_PASSWORD)).status_code in [200, 202]
except requests.ConnectionError:
return False
def _close_connection_of_bus_on_broker(self, bus: _AbstractBus):
if not self._can_connect_to_rabbitmq_admin_site(bus.broker):
raise unittest.SkipTest("Cannot connect tot RabbitMQ admin server to close connection %s" % (bus.connection_name))
# use the http REST API using request to forcefully close the connection on the broker-side
url = "http://%s:15672/api/connections/%s" % (bus.broker, bus.connection_name)
......
......@@ -6,6 +6,7 @@ lofar_find_package(Python 3.4 REQUIRED)
include(PythonInstall)
include(FindPythonModule)
find_python_module(jsonschema)
find_python_module(psycopg2)
set(_py_files
......
......@@ -27,7 +27,14 @@ IF(BUILD_TESTING)
lofar_add_test(t_methodtrigger)
lofar_add_test(t_util)
lofar_add_test(t_test_utils)
lofar_add_test(t_json_utils)
lofar_add_test(t_cep4_utils)
lofar_add_test(t_postgres)
IF(PYTHON_JSONSCHEMA)
lofar_add_test(t_json_utils)
ENDIF()
IF(PYTHON_PSYCOPG2 AND PYTHON_TESTING.POSTGRESQL)
lofar_add_test(t_postgres)
ENDIF()
ENDIF()
\ No newline at end of file
......@@ -19,5 +19,6 @@
include(LofarCTest)
lofar_add_test(t_hdf5_io)
set_tests_properties(t_hdf5_io PROPERTIES TIMEOUT 300)
......@@ -157,10 +157,11 @@ class SubtaskSerializerJSONeditorOnline(RelationalHyperlinkedModelSerializer):
import json
try:
schema = self.instance.specifications_template.schema
self.fields['specifications_doc'] = serializers.JSONField(
style={'template': 'josdejong_jsoneditor_widget.html',
'schema': json.dumps(schema)})
if isinstance(self.instance, models.Subtask):
schema = self.instance.specifications_template.schema
self.fields['specifications_doc'] = serializers.JSONField(
style={'template': 'josdejong_jsoneditor_widget.html',
'schema': json.dumps(schema)})
except Exception as e:
# todo: Shall we use one of the default templates for the init?
logger.exception('Could not determine schema, hence no fancy JSON form. Expected for list view.')
......