Log to LCU

Add --loghost command-line argument to duplicate logs towards a logstash server, f.e. the one running on the LCU.

logconfig.py

+import logging
+
+logger = logging.getLogger()
+
+def configure_logger(logstash_host: str = "", level = logging.DEBUG, log_extra: dict = None):
+    """
+      Configure the output of Python's logging module.
+
+      logstash_host: Logstash server to forward a copy of the logs to (f.e. the LCU).
+      level:         Configure to use this log level (can be a string, or integer)
+      log_extra:     Extra dict to annotate log lines with, or None.
+    """
+
+    # By default we want to know everything
+    if isinstance(level, str):
+        level_nr = getattr(logging, level.upper(), None)
+        if not isinstance(level_nr, int):
+            raise ValueError('Invalid log level: %s' % level)
+
+        logger.setLevel(level_nr)
+    else:
+        logger.setLevel(level)
+
+    # remove spam from various libraries
+    logging.getLogger("asyncua").setLevel(logging.WARN)
+    logging.getLogger("opcua").setLevel(logging.WARN)
+    logging.getLogger("git").setLevel(logging.WARN)
+
+    # log to stderr, in a way that it can be understood by a human reader
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter(fmt = '%(asctime)s [%(levelname)-8s,%(filename)-20s:%(lineno)-3d] %(message)s', datefmt = '%Y-%m-%dT%H:%M:%S')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    # Log to ELK stack
+    if logstash_host:
+        try:
+            from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
+
+            logger.info(f"Sending logs to Logstash at {logstash_host}")
+
+            # log to the tcp_input of logstash, cache pending messages if we cannot reach the host
+            handler = AsynchronousLogstashHandler(logstash_host, 5959, database_path='pending_log_messages.db')
+
+            # configure log messages
+            formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar", "pypcc"])
+            handler.setFormatter(formatter)
+
+            # install the handler
+            logger.addHandler(handler)
+        except ImportError:
+            logger.exception("Cannot send logs to Logstash: logstash_async module not found.")
+        except Exception:
+            logger.exception(f"Cannot send logs to Logstash at {logstash_host}")