From: Jakub Czajka <jakub@ekhem.eu.org>
Date: Sun, 14 Jan 2024 17:40:22 +0000 (+0100)
Subject: [postgres][rsyslog] Insert website logs to a database.
X-Git-Url: https://git.ekhem.eu.org/?a=commitdiff_plain;h=a56032e076d7cb67df3f0c07d1f6bd5253972e5e;p=server.git

[postgres][rsyslog] Insert website logs to a database.
---

diff --git a/databases/analytics/analytics_db_create.sql b/databases/analytics/analytics_db_create.sql
new file mode 100644
index 0000000..227b795
--- /dev/null
+++ b/databases/analytics/analytics_db_create.sql
@@ -0,0 +1,24 @@
+-- Copyright (c) 2024 Jakub Czajka <jakub@ekhem.eu.org>
+-- License: 0BSD.
+
+CREATE TABLE IF NOT EXISTS access_logs (
+   access_log TEXT NOT NULL,
+   created_at DATE NOT NULL
+);
+
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT * FROM pg_user WHERE usename = 'rsyslog')
+    THEN
+        CREATE ROLE rsyslog LOGIN;
+
+        GRANT INSERT
+        ON access_logs
+        TO rsyslog;
+
+        /* Execute for the current database. */
+        EXECUTE FORMAT('GRANT CONNECT
+                        ON DATABASE %I
+                        TO rsyslog', current_database());
+   END IF;
+END$$;
diff --git a/databases/analytics/analytics_db_drop.sql b/databases/analytics/analytics_db_drop.sql
new file mode 100644
index 0000000..0be26c4
--- /dev/null
+++ b/databases/analytics/analytics_db_drop.sql
@@ -0,0 +1,21 @@
+-- Copyright (c) 2024 Jakub Czajka <jakub@ekhem.eu.org>
+-- License: 0BSD.
+
+DO $$
+BEGIN
+    IF EXISTS (SELECT * FROM pg_user WHERE usename = 'rsyslog')
+    THEN
+        REVOKE INSERT
+        ON access_logs
+        FROM rsyslog;
+
+        EXECUTE
+            FORMAT('REVOKE CONNECT
+                    ON DATABASE %I
+                    FROM rsyslog;', current_database());
+
+        DROP ROLE rsyslog;
+    END IF;
+END$$;
+
+DROP TABLE IF EXISTS access_logs;
diff --git a/postgres/pg_hba.conf b/postgres/pg_hba.conf
index ae9be94..82e47e9 100755
--- a/postgres/pg_hba.conf
+++ b/postgres/pg_hba.conf
@@ -94,6 +94,7 @@ local   all             postgres                                trust
 local   mail_db         dovecot                                 trust
 host    mail_db         dovecot         ::1/128                 trust
 host    mail_db         dkim            ::1/128                 trust
+host    analytics_db    rsyslog         ::1/128                 trust
 local   gym_db          gym_tracker                             trust
 local   matrix_db       synapse_user                            trust
 local   metrics_db      metrics_tracker                         trust
diff --git a/rsyslog/51-website.conf b/rsyslog/51-website.conf
new file mode 100644
index 0000000..b7a5e97
--- /dev/null
+++ b/rsyslog/51-website.conf
@@ -0,0 +1,31 @@
+# Copyright (c) 2024 Jakub Czajka <jakub@ekhem.eu.org>
+# License: GPL-3.0 or later.
+
+# Based on https://www.rsyslog.com/doc/configuration/modules/ompgsql.html and
+# https://blog.supersetinc.com/2018/04/09/high-performance-logging-nginx-postgres-using-rsyslog.
+
+module(load="imfile")
+
+input(type="imfile"
+      File="${website_log_file}"
+      Tag="website:")
+
+template(name="insert_access_log" type="list" option.sql="on") {
+  constant(value="INSERT INTO access_logs (access_log, created_at) values ('")
+  property(name="msg")
+  constant(value="','")
+  property(name="timereported" dateformat="pgsql" date.inUTC="on")
+  constant(value="')")
+}
+
+# Load the ompgsql output module
+module(load="ompgsql")
+
+if ( ${dollar}syslogtag == 'website:' ) then {
+  action(type="ompgsql"
+         server="localhost"
+         user="rsyslog"
+         db="${analytics_db}"
+         template="insert_access_log"
+         queue.type="linkedList")
+}
diff --git a/rsyslog/README b/rsyslog/README
new file mode 100644
index 0000000..0b32f05
--- /dev/null
+++ b/rsyslog/README
@@ -0,0 +1,14 @@
+rsyslog
+========
+
+Create database `analytics_db`.
+
+```
+CREATE DATABASE analytics_db;
+```
+
+Execute `analytics_db_create.sql`.
+
+```
+psql --dbname=mail_db --file=databases/analytics/analytics_db_create.sql
+```