set up syslog-ng for nginx access log

This sets up the infrastructure for moving from storing nginx access
logs in journald to plain text files written by syslog-ng and rotated by
logrotate. This works around the poor performance, poor space efficiency
and lack of archived log compression for journald. Unlike writing access
logs directly with nginx, this continues avoiding blocking writes in the
event loop and sticks to asynchronous sends through a socket.

Since nginx only supports syslog via the RFC 3164 protocol rather than
the more modern RFC 5424 protocol, this leaves formatting timestamps up
to nginx rather than using the ones provided via the syslog protocol.

deploy-web

@@ -15,10 +15,13 @@ for host in ${hosts_web[@]}; do
     rsync etc/systemd/system/{session-ticket-keys-create.service,session-ticket-keys-rotate.service,session-ticket-keys-rotate.timer} $remote:/etc/systemd/system/
     rsync --chmod=755 session-ticket-keys-create session-ticket-keys-rotate $remote:/usr/local/bin/
     rsync -r --delete etc/systemd/system/nginx.service.d/ $remote:/etc/systemd/system/nginx.service.d
+    rsync etc/syslog-ng/syslog-ng.conf $remote:/etc/syslog-ng/syslog-ng.conf
+    rsync etc/logrotate.d/nginx $remote:/etc/logrotate.d/nginx
 
     ssh $remote "mkdir -pm755 /var/cache/nginx
 groupadd -fg 2100 tls
 mkdir -p -m 750 /etc/session-ticket-keys && chgrp tls /etc/session-ticket-keys
 systemctl daemon-reload &&
-systemctl enable --now session-ticket-keys-create.service session-ticket-keys-rotate.timer nginx.service"
+systemctl enable --now session-ticket-keys-create.service session-ticket-keys-rotate.timer syslog-ng@default.service nginx.service
+syslog-ng-ctl reload"
 done

etc/logrotate.d/nginx

@@ -0,0 +1,11 @@
+/var/log/nginx/*log {
+    missingok
+    notifempty
+    create 600 root root
+    sharedscripts
+    compress
+    maxsize 2G
+    postrotate
+        syslog-ng-ctl reopen >/dev/null
+    endscript
+}

etc/syslog-ng/syslog-ng.conf

@@ -0,0 +1,30 @@
+@version: 4.10
+
+source s_internal {
+    internal();
+};
+source s_nginx_access_log {
+    unix-dgram("/run/nginx-access-log" group("http") perm(0660));
+};
+
+destination d_journald {
+    unix-dgram("/dev/log");
+};
+destination d_nginx {
+    file("/var/log/nginx/access.log" template("${MESSAGE}\n"));
+};
+
+log {
+    source(s_internal);
+    destination(d_journald);
+};
+log {
+    source(s_nginx_access_log);
+    destination(d_nginx);
+};
+
+options {
+    keep-hostname(yes);
+    stats(freq(0));
+    use-dns(no);
+};

etc/systemd/system/logrotate.timer.d/override.conf

@@ -0,0 +1,7 @@
+[Unit]
+Description=Rotate log files every 5 minutes
+
+[Timer]
+AccuracySec=1us
+OnCalendar=*:0/5
+RandomizedDelaySec=0

etc/systemd/system/nginx.service.d/override.conf

@@ -1,3 +1,6 @@
+[Unit]
+After=syslog-ng.service
+
 [Service]
 CapabilityBoundingSet=CAP_CHOWN CAP_DAC_OVERRIDE CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
 LockPersonality=true

packages/0.grapheneos.network

@@ -27,6 +27,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/0.grapheneos.org

@@ -27,6 +27,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/0.ns1.grapheneos.org

@@ -34,6 +34,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/0.ns2.grapheneos.org

@@ -33,6 +33,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/0.releases.grapheneos.org

@@ -34,6 +34,7 @@ rsync
 smartmontools
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/1.grapheneos.network

@@ -26,6 +26,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/1.grapheneos.org

@@ -26,6 +26,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/1.ns1.grapheneos.org

@@ -33,6 +33,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/1.ns2.grapheneos.org

@@ -32,6 +32,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/1.releases.grapheneos.org

@@ -33,6 +33,7 @@ rsync
 smartmontools
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/2.grapheneos.network

@@ -26,6 +26,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/2.grapheneos.org

@@ -26,6 +26,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/2.ns1.grapheneos.org

@@ -33,6 +33,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/2.ns2.grapheneos.org

@@ -32,6 +32,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/3.grapheneos.network

@@ -26,6 +26,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/3.grapheneos.org

@@ -26,6 +26,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/3.ns1.grapheneos.org

@@ -33,6 +33,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/3.releases.grapheneos.org

@@ -34,6 +34,7 @@ rsync
 smartmontools
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/attestation.app

@@ -33,6 +33,7 @@ rsync
 sqlite-analyzer
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/discuss.grapheneos.org

@@ -43,6 +43,7 @@ python-swiftclient
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/grapheneos.social

@@ -34,6 +34,7 @@ python-swiftclient
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/mail.grapheneos.org

@@ -40,6 +40,7 @@ rsync
 s-nail
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/matrix.grapheneos.org

@@ -42,6 +42,7 @@ python-swiftclient
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/ns1.staging.grapheneos.org

@@ -34,6 +34,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/staging.attestation.app

@@ -33,6 +33,7 @@ rsync
 sqlite-analyzer
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree

packages/staging.grapheneos.org

@@ -27,6 +27,7 @@ pv
 rsync
 strace
 stress
+syslog-ng
 sysstat
 tinyxxd
 tree