Allow logs to also be send to a logstash server (f.e. the LCU)

logconfig.py

+import logging
+
+logger = logging.getLogger()
+
+def configure_logger(logstash_host: str = "", level = logging.DEBUG, log_extra: dict = None):
+    """
+      Configure the output of Python's logging module.
+
+      logstash_host: Logstash server to forward a copy of the logs to (f.e. the LCU).
+      level:         Configure to use this log level (can be a string, or integer)
+      log_extra:     Extra dict to annotate log lines with, or None.
+    """
+
+    # By default we want to know everything
+    if isinstance(level, str):
+        level_nr = getattr(logging, level.upper(), None)
+        if not isinstance(level_nr, int):
+            raise ValueError('Invalid log level: %s' % level)
+
+        logger.setLevel(level_nr)
+    else:
+        logger.setLevel(level)
+
+    # remove spam from various libraries
+    logging.getLogger("asyncua").setLevel(logging.WARN)
+    logging.getLogger("opcua").setLevel(logging.WARN)
+    logging.getLogger("git").setLevel(logging.WARN)
+
+    # log to stderr, in a way that it can be understood by a human reader
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter(fmt = '%(asctime)s [%(levelname)-8s,%(filename)-20s:%(lineno)-3d] %(message)s', datefmt = '%Y-%m-%dT%H:%M:%S')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    # Log to ELK stack
+    if logstash_host:
+        try:
+            from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
+
+            logger.info(f"Sending logs to Logstash at {logstash_host}")
+
+            # log to the tcp_input of logstash, cache pending messages if we cannot reach the host
+            handler = AsynchronousLogstashHandler(logstash_host, 5959, database_path='pending_log_messages.db')
+
+            # configure log messages
+            formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar", "pypcc"])
+            handler.setFormatter(formatter)
+
+            # install the handler
+            logger.addHandler(handler)
+        except ImportError:
+            logger.exception("Cannot send logs to Logstash: logstash_async module not found.")
+        except Exception:
+            logger.exception(f"Cannot send logs to Logstash at {logstash_host}")

pypcc2.py

@@ -16,15 +16,19 @@ parser.add_argument("-t", "--test", help="Do not start OPC-UA server.", action="
 parser.add_argument("-p", "--port", help="Port number to listen on [%(default)s].", type=int, default=4842)
 parser.add_argument("-l", "--loglevel", help="Log level [%(default)s].", type=str, choices=["DEBUG","INFO","WARNING","ERROR"], default="INFO")
 parser.add_argument("-c", "--config", help="YAML config files, comma seperated [%(default)s]",type=str, default='RCU')
+parser.add_argument("--loghost", help="Logstash host to which to forward logs [%(default)s]",type=str, default='')
 args = parser.parse_args()
 
-# set log level
-loglevel_nr = getattr(logging, args.loglevel.upper(), None)
-if not isinstance(loglevel_nr, int):
-    raise ValueError('Invalid log level: %s' % args.loglevel)
-#logging.basicConfig(level=loglevel_nr, format="%(asctime)s [%(levelname)8s] %(message)s")
+from logconfig import configure_logger
+log_extra = {
+    "simulator": args.simulator, 
+    "test": args.test,
+    "port": args.port,
+    "config": args.config,
+    "lofar_id": f"pypcc - {args.config}",
+}
+configure_logger(logstash_host=args.loghost,level=args.loglevel, log_extra=log_extra)
 
-logging.basicConfig(level=loglevel_nr,format='%(asctime)s [%(levelname)-8s,%(filename)-20s:%(lineno)-3d] %(message)s')
 RunTimer=True;
 def signal_handler(sig, frame):
     logging.warn('Stop signal received!')

requirements.txt

@@ -3,3 +3,4 @@ opcua
 numpy
 recordclass
 pyyaml
+python-logstash-async