Skip to content
Snippets Groups Projects
Commit e0ff9295 authored by Stefano Di Frischia's avatar Stefano Di Frischia
Browse files

L2SS-970: replace ELK with Loki in lofar_logging

parent 21d14b35
Branches
Tags
1 merge request!447Resolve L2SS-970 "Add loki instance"
...@@ -56,7 +56,7 @@ class LogSuppressErrorSpam(logging.Formatter): ...@@ -56,7 +56,7 @@ class LogSuppressErrorSpam(logging.Formatter):
self.error_suppress_interval = error_suppress_interval self.error_suppress_interval = error_suppress_interval
def is_error_to_suppress(self, record): def is_error_to_suppress(self, record):
# Errors occuring by not being able to connect to the ELK stack, f.e. because it is down. # Errors occuring by not being able to connect to the log processing container, f.e. because it is down.
return record.name == "LogProcessingWorker" and record.msg == "An error occurred while sending events: %s" return record.name == "LogProcessingWorker" and record.msg == "An error occurred while sending events: %s"
def filter(self, record): def filter(self, record):
...@@ -105,7 +105,7 @@ class LogAnnotator(logging.Formatter): ...@@ -105,7 +105,7 @@ class LogAnnotator(logging.Formatter):
def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False): def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
""" """
Configure the given logger (or root if None) to: Configure the given logger (or root if None) to:
- send logs to the ELK stack - send logs to Loki through Logstash
- send logs to Tango - send logs to Tango
- send logs to stdout - send logs to stdout
""" """
...@@ -128,7 +128,7 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False): ...@@ -128,7 +128,7 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
# don't spam debug messages when fetching URLs # don't spam debug messages when fetching URLs
logging.getLogger("urllib3").setLevel(logging.INFO) logging.getLogger("urllib3").setLevel(logging.INFO)
# don't spam error messages when having trouble connecting to ELK # don't spam error messages when having connection troubles
logging.getLogger("LogProcessingWorker").setLevel(logging.CRITICAL) logging.getLogger("LogProcessingWorker").setLevel(logging.CRITICAL)
# for now, also log to stderr # for now, also log to stderr
...@@ -151,12 +151,12 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False): ...@@ -151,12 +151,12 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
if debug: if debug:
return logger return logger
# Log to ELK stack # Log to Logstash-Loki
try: try:
from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
# log to the tcp_input of logstash in our ELK stack # log to the tcp_input of logstash in our logstash-loki container
handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/lofar_pending_log_messages.db') handler = AsynchronousLogstashHandler("logstash-loki", 5959, database_path='/tmp/lofar_pending_log_messages.db')
# configure log messages # configure log messages
formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar"]) formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar"])
...@@ -167,9 +167,9 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False): ...@@ -167,9 +167,9 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
# install the handler # install the handler
logger.addHandler(handler) logger.addHandler(handler)
except ImportError: except ImportError:
logger.exception("Cannot forward logs to ELK: logstash_async module not found.") logger.exception("Cannot forward logs to Logstash-Loki: logstash_async module not found.")
except Exception: except Exception:
logger.exception("Cannot forward logs to ELK.") logger.exception("Cannot forward logs to Logstash-Loki.")
# Don't log to Tango to reduce log spam # Don't log to Tango to reduce log spam
""" """
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment