From 2caa67529a2ec2ec714765b3afc21bf1ad021313 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Sat, 1 Nov 2025 21:09:05 -0400 Subject: [PATCH] set up syslog-ng for nginx access log This sets up the infrastructure for moving from storing nginx access logs in journald to plain text files written by syslog-ng and rotated by logrotate. This works around the poor performance, poor space efficiency and lack of archived log compression for journald. Unlike writing access logs directly with nginx, this continues avoiding blocking writes in the event loop and sticks to asynchronous sends through a socket. Since nginx only supports syslog via the RFC 3164 protocol rather than the more modern RFC 5424 protocol, this leaves formatting timestamps up to nginx rather than using the ones provided via the syslog protocol. --- deploy-web | 5 +++- etc/logrotate.d/nginx | 11 +++++++ etc/syslog-ng/syslog-ng.conf | 30 +++++++++++++++++++ .../system/logrotate.timer.d/override.conf | 7 +++++ .../system/nginx.service.d/override.conf | 3 ++ packages/0.grapheneos.network | 1 + packages/0.grapheneos.org | 1 + packages/0.ns1.grapheneos.org | 1 + packages/0.ns2.grapheneos.org | 1 + packages/0.releases.grapheneos.org | 1 + packages/1.grapheneos.network | 1 + packages/1.grapheneos.org | 1 + packages/1.ns1.grapheneos.org | 1 + packages/1.ns2.grapheneos.org | 1 + packages/1.releases.grapheneos.org | 1 + packages/2.grapheneos.network | 1 + packages/2.grapheneos.org | 1 + packages/2.ns1.grapheneos.org | 1 + packages/2.ns2.grapheneos.org | 1 + packages/3.grapheneos.network | 1 + packages/3.grapheneos.org | 1 + packages/3.ns1.grapheneos.org | 1 + packages/3.releases.grapheneos.org | 1 + packages/attestation.app | 1 + packages/discuss.grapheneos.org | 1 + packages/grapheneos.social | 1 + packages/mail.grapheneos.org | 1 + packages/matrix.grapheneos.org | 1 + packages/ns1.staging.grapheneos.org | 1 + packages/staging.attestation.app | 1 + packages/staging.grapheneos.org | 1 + 31 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 etc/logrotate.d/nginx create mode 100644 etc/syslog-ng/syslog-ng.conf create mode 100644 etc/systemd/system/logrotate.timer.d/override.conf diff --git a/deploy-web b/deploy-web index 59fb72c..7c51916 100755 --- a/deploy-web +++ b/deploy-web @@ -15,10 +15,13 @@ for host in ${hosts_web[@]}; do rsync etc/systemd/system/{session-ticket-keys-create.service,session-ticket-keys-rotate.service,session-ticket-keys-rotate.timer} $remote:/etc/systemd/system/ rsync --chmod=755 session-ticket-keys-create session-ticket-keys-rotate $remote:/usr/local/bin/ rsync -r --delete etc/systemd/system/nginx.service.d/ $remote:/etc/systemd/system/nginx.service.d + rsync etc/syslog-ng/syslog-ng.conf $remote:/etc/syslog-ng/syslog-ng.conf + rsync etc/logrotate.d/nginx $remote:/etc/logrotate.d/nginx ssh $remote "mkdir -pm755 /var/cache/nginx groupadd -fg 2100 tls mkdir -p -m 750 /etc/session-ticket-keys && chgrp tls /etc/session-ticket-keys systemctl daemon-reload && -systemctl enable --now session-ticket-keys-create.service session-ticket-keys-rotate.timer nginx.service" +systemctl enable --now session-ticket-keys-create.service session-ticket-keys-rotate.timer syslog-ng@default.service nginx.service +syslog-ng-ctl reload" done diff --git a/etc/logrotate.d/nginx b/etc/logrotate.d/nginx new file mode 100644 index 0000000..05667c1 --- /dev/null +++ b/etc/logrotate.d/nginx @@ -0,0 +1,11 @@ +/var/log/nginx/*log { + missingok + notifempty + create 600 root root + sharedscripts + compress + maxsize 2G + postrotate + syslog-ng-ctl reopen >/dev/null + endscript +} diff --git a/etc/syslog-ng/syslog-ng.conf b/etc/syslog-ng/syslog-ng.conf new file mode 100644 index 0000000..050df08 --- /dev/null +++ b/etc/syslog-ng/syslog-ng.conf @@ -0,0 +1,30 @@ +@version: 4.10 + +source s_internal { + internal(); +}; +source s_nginx_access_log { + unix-dgram("/run/nginx-access-log" group("http") perm(0660)); +}; + +destination d_journald { + unix-dgram("/dev/log"); +}; +destination d_nginx { + file("/var/log/nginx/access.log" template("${MESSAGE}\n")); +}; + +log { + source(s_internal); + destination(d_journald); +}; +log { + source(s_nginx_access_log); + destination(d_nginx); +}; + +options { + keep-hostname(yes); + stats(freq(0)); + use-dns(no); +}; diff --git a/etc/systemd/system/logrotate.timer.d/override.conf b/etc/systemd/system/logrotate.timer.d/override.conf new file mode 100644 index 0000000..c331099 --- /dev/null +++ b/etc/systemd/system/logrotate.timer.d/override.conf @@ -0,0 +1,7 @@ +[Unit] +Description=Rotate log files every 5 minutes + +[Timer] +AccuracySec=1us +OnCalendar=*:0/5 +RandomizedDelaySec=0 diff --git a/etc/systemd/system/nginx.service.d/override.conf b/etc/systemd/system/nginx.service.d/override.conf index 8e4339e..6e73377 100644 --- a/etc/systemd/system/nginx.service.d/override.conf +++ b/etc/systemd/system/nginx.service.d/override.conf @@ -1,3 +1,6 @@ +[Unit] +After=syslog-ng.service + [Service] CapabilityBoundingSet=CAP_CHOWN CAP_DAC_OVERRIDE CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID LockPersonality=true diff --git a/packages/0.grapheneos.network b/packages/0.grapheneos.network index adaa432..118629e 100644 --- a/packages/0.grapheneos.network +++ b/packages/0.grapheneos.network @@ -27,6 +27,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/0.grapheneos.org b/packages/0.grapheneos.org index 98b2377..9dfb786 100644 --- a/packages/0.grapheneos.org +++ b/packages/0.grapheneos.org @@ -27,6 +27,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/0.ns1.grapheneos.org b/packages/0.ns1.grapheneos.org index 9250e5a..64f921d 100644 --- a/packages/0.ns1.grapheneos.org +++ b/packages/0.ns1.grapheneos.org @@ -34,6 +34,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/0.ns2.grapheneos.org b/packages/0.ns2.grapheneos.org index 493059f..3d73df2 100644 --- a/packages/0.ns2.grapheneos.org +++ b/packages/0.ns2.grapheneos.org @@ -33,6 +33,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/0.releases.grapheneos.org b/packages/0.releases.grapheneos.org index 322f5fe..1ff138e 100644 --- a/packages/0.releases.grapheneos.org +++ b/packages/0.releases.grapheneos.org @@ -34,6 +34,7 @@ rsync smartmontools strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/1.grapheneos.network b/packages/1.grapheneos.network index 885f90f..8f0b780 100644 --- a/packages/1.grapheneos.network +++ b/packages/1.grapheneos.network @@ -26,6 +26,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/1.grapheneos.org b/packages/1.grapheneos.org index d19c2ff..7e3ab42 100644 --- a/packages/1.grapheneos.org +++ b/packages/1.grapheneos.org @@ -26,6 +26,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/1.ns1.grapheneos.org b/packages/1.ns1.grapheneos.org index c7223d1..9a5446e 100644 --- a/packages/1.ns1.grapheneos.org +++ b/packages/1.ns1.grapheneos.org @@ -33,6 +33,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/1.ns2.grapheneos.org b/packages/1.ns2.grapheneos.org index 81d0975..e6ba1cd 100644 --- a/packages/1.ns2.grapheneos.org +++ b/packages/1.ns2.grapheneos.org @@ -32,6 +32,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/1.releases.grapheneos.org b/packages/1.releases.grapheneos.org index fa586f6..8f3bf5b 100644 --- a/packages/1.releases.grapheneos.org +++ b/packages/1.releases.grapheneos.org @@ -33,6 +33,7 @@ rsync smartmontools strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/2.grapheneos.network b/packages/2.grapheneos.network index 885f90f..8f0b780 100644 --- a/packages/2.grapheneos.network +++ b/packages/2.grapheneos.network @@ -26,6 +26,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/2.grapheneos.org b/packages/2.grapheneos.org index d19c2ff..7e3ab42 100644 --- a/packages/2.grapheneos.org +++ b/packages/2.grapheneos.org @@ -26,6 +26,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/2.ns1.grapheneos.org b/packages/2.ns1.grapheneos.org index c7223d1..9a5446e 100644 --- a/packages/2.ns1.grapheneos.org +++ b/packages/2.ns1.grapheneos.org @@ -33,6 +33,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/2.ns2.grapheneos.org b/packages/2.ns2.grapheneos.org index 81d0975..e6ba1cd 100644 --- a/packages/2.ns2.grapheneos.org +++ b/packages/2.ns2.grapheneos.org @@ -32,6 +32,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/3.grapheneos.network b/packages/3.grapheneos.network index 885f90f..8f0b780 100644 --- a/packages/3.grapheneos.network +++ b/packages/3.grapheneos.network @@ -26,6 +26,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/3.grapheneos.org b/packages/3.grapheneos.org index d19c2ff..7e3ab42 100644 --- a/packages/3.grapheneos.org +++ b/packages/3.grapheneos.org @@ -26,6 +26,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/3.ns1.grapheneos.org b/packages/3.ns1.grapheneos.org index c7223d1..9a5446e 100644 --- a/packages/3.ns1.grapheneos.org +++ b/packages/3.ns1.grapheneos.org @@ -33,6 +33,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/3.releases.grapheneos.org b/packages/3.releases.grapheneos.org index efe29dc..bb8cac9 100644 --- a/packages/3.releases.grapheneos.org +++ b/packages/3.releases.grapheneos.org @@ -34,6 +34,7 @@ rsync smartmontools strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/attestation.app b/packages/attestation.app index 7af7e53..187e080 100644 --- a/packages/attestation.app +++ b/packages/attestation.app @@ -33,6 +33,7 @@ rsync sqlite-analyzer strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/discuss.grapheneos.org b/packages/discuss.grapheneos.org index 87a21d7..de8a5c4 100644 --- a/packages/discuss.grapheneos.org +++ b/packages/discuss.grapheneos.org @@ -43,6 +43,7 @@ python-swiftclient rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/grapheneos.social b/packages/grapheneos.social index 4866899..b669406 100644 --- a/packages/grapheneos.social +++ b/packages/grapheneos.social @@ -34,6 +34,7 @@ python-swiftclient rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/mail.grapheneos.org b/packages/mail.grapheneos.org index 573207f..e706696 100644 --- a/packages/mail.grapheneos.org +++ b/packages/mail.grapheneos.org @@ -40,6 +40,7 @@ rsync s-nail strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/matrix.grapheneos.org b/packages/matrix.grapheneos.org index c431121..eb53d98 100644 --- a/packages/matrix.grapheneos.org +++ b/packages/matrix.grapheneos.org @@ -42,6 +42,7 @@ python-swiftclient rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/ns1.staging.grapheneos.org b/packages/ns1.staging.grapheneos.org index 9250e5a..64f921d 100644 --- a/packages/ns1.staging.grapheneos.org +++ b/packages/ns1.staging.grapheneos.org @@ -34,6 +34,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/staging.attestation.app b/packages/staging.attestation.app index 7af7e53..187e080 100644 --- a/packages/staging.attestation.app +++ b/packages/staging.attestation.app @@ -33,6 +33,7 @@ rsync sqlite-analyzer strace stress +syslog-ng sysstat tinyxxd tree diff --git a/packages/staging.grapheneos.org b/packages/staging.grapheneos.org index 98b2377..9dfb786 100644 --- a/packages/staging.grapheneos.org +++ b/packages/staging.grapheneos.org @@ -27,6 +27,7 @@ pv rsync strace stress +syslog-ng sysstat tinyxxd tree