From af3d99b629a8f48add8890d5f3e8acc1c7e7ef1a Mon Sep 17 00:00:00 2001
From: Jan David Mol <mol@astron.nl>
Date: Fri, 15 Sep 2023 14:30:43 +0000
Subject: [PATCH] Fix parsing of MariaDB timestamps
---
docker-compose/logstash/Dockerfile | 5 +++--
docker-compose/logstash/{loki.conf => logstash.conf} | 5 +++--
docker-compose/logstash/patterns/mariadb.txt | 2 ++
docker-compose/tango.yml | 1 +
4 files changed, 9 insertions(+), 4 deletions(-)
rename docker-compose/logstash/{loki.conf => logstash.conf} (95%)
create mode 100644 docker-compose/logstash/patterns/mariadb.txt
diff --git a/docker-compose/logstash/Dockerfile b/docker-compose/logstash/Dockerfile
index 94fa5ab4b..c455b2192 100644
--- a/docker-compose/logstash/Dockerfile
+++ b/docker-compose/logstash/Dockerfile
@@ -5,6 +5,7 @@ FROM ${SOURCE_IMAGE}
ENV ELASTIC_CONTAINER=false
# Provide our logstash config
-COPY loki.conf /home/logstash/
+COPY logstash.conf /home/logstash/
COPY logstash.yml /usr/share/logstash/config/logstash.yml
-COPY loki.conf /usr/share/logstash/pipeline/logstash.conf
+COPY logstash.conf /usr/share/logstash/pipeline/logstash.conf
+COPY patterns /usr/share/logstash/patterns
diff --git a/docker-compose/logstash/loki.conf b/docker-compose/logstash/logstash.conf
similarity index 95%
rename from docker-compose/logstash/loki.conf
rename to docker-compose/logstash/logstash.conf
index 177ea72c8..71afa03e9 100644
--- a/docker-compose/logstash/loki.conf
+++ b/docker-compose/logstash/logstash.conf
@@ -97,10 +97,11 @@ filter {
if [program] == "tangodb" {
grok {
id => "grok_tangodb"
+ patterns_dir => [ "/usr/share/logstash/patterns/" ]
match => {
"message" => [
- "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
- "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
+ "%{TIMESTAMP_ISO8601_MARIADB:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601_MARIADB:timestamp} %{NONNEGINT} .%{WORD:level}. %{GREEDYDATA:message}"
]
}
"overwrite" => [ "timestamp", "level", "message" ]
diff --git a/docker-compose/logstash/patterns/mariadb.txt b/docker-compose/logstash/patterns/mariadb.txt
new file mode 100644
index 000000000..42d981acb
--- /dev/null
+++ b/docker-compose/logstash/patterns/mariadb.txt
@@ -0,0 +1,2 @@
+# MariaDB uses ' 2' instead of '02' for hours. The following format accepts both.
+TIMESTAMP_ISO8601_MARIADB %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ] ?%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
diff --git a/docker-compose/tango.yml b/docker-compose/tango.yml
index 58dfc1915..9de7d9e1d 100644
--- a/docker-compose/tango.yml
+++ b/docker-compose/tango.yml
@@ -27,6 +27,7 @@ services:
- MYSQL_DATABASE=tango
- MYSQL_USER=tango
- MYSQL_PASSWORD=tango
+ - MARIADB_AUTO_UPGRADE=1
volumes:
- tangodb:/var/lib/mysql
ports:
--
GitLab