From 5befa504db74e7ad6e41a17c04b7a48b124c8cef Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Wed, 13 Oct 2021 13:43:57 +0200
Subject: [PATCH] Allow logs to also be send to a logstash server (f.e. the
LCU)
---
logconfig.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
pypcc2.py | 16 ++++++++------
requirements.txt | 1 +
3 files changed, 65 insertions(+), 6 deletions(-)
create mode 100644 logconfig.py
diff --git a/logconfig.py b/logconfig.py
new file mode 100644
index 0000000..b0d7182
--- /dev/null
+++ b/logconfig.py
@@ -0,0 +1,54 @@
+import logging
+
+logger = logging.getLogger()
+
+def configure_logger(logstash_host: str = "", level = logging.DEBUG, log_extra: dict = None):
+ """
+ Configure the output of Python's logging module.
+
+ logstash_host: Logstash server to forward a copy of the logs to (f.e. the LCU).
+ level: Configure to use this log level (can be a string, or integer)
+ log_extra: Extra dict to annotate log lines with, or None.
+ """
+
+ # By default we want to know everything
+ if isinstance(level, str):
+ level_nr = getattr(logging, level.upper(), None)
+ if not isinstance(level_nr, int):
+ raise ValueError('Invalid log level: %s' % level)
+
+ logger.setLevel(level_nr)
+ else:
+ logger.setLevel(level)
+
+ # remove spam from various libraries
+ logging.getLogger("asyncua").setLevel(logging.WARN)
+ logging.getLogger("opcua").setLevel(logging.WARN)
+ logging.getLogger("git").setLevel(logging.WARN)
+
+ # log to stderr, in a way that it can be understood by a human reader
+ handler = logging.StreamHandler()
+ formatter = logging.Formatter(fmt = '%(asctime)s [%(levelname)-8s,%(filename)-20s:%(lineno)-3d] %(message)s', datefmt = '%Y-%m-%dT%H:%M:%S')
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ # Log to ELK stack
+ if logstash_host:
+ try:
+ from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
+
+ logger.info(f"Sending logs to Logstash at {logstash_host}")
+
+ # log to the tcp_input of logstash, cache pending messages if we cannot reach the host
+ handler = AsynchronousLogstashHandler(logstash_host, 5959, database_path='pending_log_messages.db')
+
+ # configure log messages
+ formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar", "pypcc"])
+ handler.setFormatter(formatter)
+
+ # install the handler
+ logger.addHandler(handler)
+ except ImportError:
+ logger.exception("Cannot send logs to Logstash: logstash_async module not found.")
+ except Exception:
+ logger.exception(f"Cannot send logs to Logstash at {logstash_host}")
diff --git a/pypcc2.py b/pypcc2.py
index 0fb7bef..3bf0333 100644
--- a/pypcc2.py
+++ b/pypcc2.py
@@ -16,15 +16,19 @@ parser.add_argument("-t", "--test", help="Do not start OPC-UA server.", action="
parser.add_argument("-p", "--port", help="Port number to listen on [%(default)s].", type=int, default=4842)
parser.add_argument("-l", "--loglevel", help="Log level [%(default)s].", type=str, choices=["DEBUG","INFO","WARNING","ERROR"], default="INFO")
parser.add_argument("-c", "--config", help="YAML config files, comma seperated [%(default)s]",type=str, default='RCU')
+parser.add_argument("--loghost", help="Logstash host to which to forward logs [%(default)s]",type=str, default='')
args = parser.parse_args()
-# set log level
-loglevel_nr = getattr(logging, args.loglevel.upper(), None)
-if not isinstance(loglevel_nr, int):
- raise ValueError('Invalid log level: %s' % args.loglevel)
-#logging.basicConfig(level=loglevel_nr, format="%(asctime)s [%(levelname)8s] %(message)s")
+from logconfig import configure_logger
+log_extra = {
+ "simulator": args.simulator,
+ "test": args.test,
+ "port": args.port,
+ "config": args.config,
+ "lofar_id": f"pypcc - {args.config}",
+}
+configure_logger(logstash_host=args.loghost,level=args.loglevel, log_extra=log_extra)
-logging.basicConfig(level=loglevel_nr,format='%(asctime)s [%(levelname)-8s,%(filename)-20s:%(lineno)-3d] %(message)s')
RunTimer=True;
def signal_handler(sig, frame):
logging.warn('Stop signal received!')
diff --git a/requirements.txt b/requirements.txt
index 4a48e3d..01abf7a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ opcua
numpy
recordclass
pyyaml
+python-logstash-async
--
GitLab