move IP-based SSH connection limits to nftables

We use synproxy for establishing all new connections to the SSH port and
enforce a connection limit between synproxy and the standard network
stack. Once the connection limit is reached, it's also enforced for new
connections at the synproxy layer. This avoids creating conntrack and
connection limit set entries until connections are already established
to avoid packets with spoofed source addresses exhausting these limited
size tables. Primary servers using SSH to mirror TLS certificates to
their replicas are allowlisted.

nftables-attestation.conf

@@ -3,6 +3,24 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -14,7 +32,8 @@ table inet filter {
         # drop packets to address not configured on incoming interface (strong host model)
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
-        tcp dport { 22, 80, 443 } notrack accept
+        tcp dport { 80, 443 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -22,7 +41,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 80, 443 } notrack accept
+        tcp sport { 80, 443 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -30,15 +49,17 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 80, 443 } accept
+        tcp dport { 80, 443 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-discuss.conf

@@ -3,6 +3,24 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -17,7 +35,8 @@ table inet filter {
         # IPv6 interacts badly with IP-based spam filtering
         meta nfproto ipv6 tcp dport { 80, 443 } reject with tcp reset
 
-        tcp dport { 22, 80, 443 } notrack accept
+        tcp dport { 80, 443 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -25,7 +44,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 80, 443 } notrack accept
+        tcp sport { 80, 443 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -33,15 +52,17 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 80, 443 } accept
+        tcp dport { 80, 443 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-mail.conf

@@ -3,6 +3,24 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -14,7 +32,8 @@ table inet filter {
         # drop packets to address not configured on incoming interface (strong host model)
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
-        tcp dport { 22, 25, 80, 443, 465, 993 } notrack accept
+        tcp dport { 25, 80, 443, 465, 993 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -22,7 +41,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 25, 80, 443, 465, 993 } notrack accept
+        tcp sport { 25, 80, 443, 465, 993 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -30,15 +49,17 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 25, 80, 443, 465, 993 } accept
+        tcp dport { 25, 80, 443, 465, 993 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-matrix.conf

@@ -3,6 +3,24 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -14,7 +32,8 @@ table inet filter {
         # drop packets to address not configured on incoming interface (strong host model)
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
-        tcp dport { 22, 80, 443 } notrack accept
+        tcp dport { 80, 443 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -22,7 +41,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 80, 443 } notrack accept
+        tcp sport { 80, 443 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -30,15 +49,17 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 80, 443 } accept
+        tcp dport { 80, 443 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-network.conf

@@ -3,6 +3,26 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+        51.222.159.116, # 0.grapheneos.network
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+        2607:5300:205:200::2584, # 0.grapheneos.network
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -14,7 +34,8 @@ table inet filter {
         # drop packets to address not configured on incoming interface (strong host model)
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
-        tcp dport { 22, 80, 443, 7275 } notrack accept
+        tcp dport { 80, 443, 7275 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         udp dport 123 notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
@@ -23,7 +44,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 80, 443, 7275 } notrack accept
+        tcp sport { 80, 443, 7275 } notrack accept
         udp sport 123 notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
@@ -32,16 +53,18 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 80, 443, 7275 } accept
+        tcp dport { 80, 443, 7275 } accept
         udp dport 123 accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-ns1.conf

@@ -3,6 +3,24 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -15,7 +33,8 @@ table inet filter {
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
         udp dport 53 notrack accept
-        tcp dport { 22, 53, 80, 443, 853 } notrack accept
+        tcp dport { 53, 80, 443, 853 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -24,7 +43,7 @@ table inet filter {
 
         oif lo notrack accept
         udp sport 53 notrack accept
-        tcp sport { 22, 53, 80, 443, 853 } notrack accept
+        tcp sport { 53, 80, 443, 853 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -32,16 +51,18 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
         udp dport 53 accept
-        tcp dport { 22, 53, 80, 443, 853 } accept
+        tcp dport { 53, 80, 443, 853 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-ns2.conf

@@ -3,6 +3,26 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+        198.98.53.141, # 0.ns2.grapheneos.org
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+        2605:6400:10:102e:95bc:89ef:2e7f:49bb, # 0.ns2.grapheneos.org
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -18,7 +38,8 @@ table inet filter {
         tcp dport 22 ip daddr 198.251.90.93 reject with tcp reset
 
         udp dport 53 notrack accept
-        tcp dport { 22, 53, 80, 443, 853 } notrack accept
+        tcp dport { 53, 80, 443, 853 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -27,7 +48,7 @@ table inet filter {
 
         oif lo notrack accept
         udp sport 53 notrack accept
-        tcp sport { 22, 53, 80, 443, 853 } notrack accept
+        tcp sport { 53, 80, 443, 853 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -35,16 +56,18 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
         udp dport 53 accept
-        tcp dport { 22, 53, 80, 443, 853 } accept
+        tcp dport { 53, 80, 443, 853 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-social.conf

@@ -3,6 +3,24 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -14,7 +32,8 @@ table inet filter {
         # drop packets to address not configured on incoming interface (strong host model)
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
-        tcp dport { 22, 80, 443 } notrack accept
+        tcp dport { 80, 443 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -22,7 +41,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 80, 443 } notrack accept
+        tcp sport { 80, 443 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -30,15 +49,17 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 80, 443 } accept
+        tcp dport { 80, 443 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

nftables-web.conf

@@ -3,6 +3,28 @@
 flush ruleset
 
 table inet filter {
+    define ip-allowlist-ssh = {
+        127.0.0.1,
+        51.222.156.101, # 0.grapheneos.org
+        167.114.114.114, # 0.releases.grapheneos.org
+    }
+
+    define ip6-allowlist-ssh = {
+        ::1,
+        2607:5300:205:200::29c6, # 0.grapheneos.org
+        2607:5300:201:3100::6210, # 0.releases.grapheneos.org
+    }
+
+    set ip-connlimit-ssh {
+        type ipv4_addr
+        flags dynamic
+    }
+
+    set ip6-connlimit-ssh {
+        type ipv6_addr
+        flags dynamic
+    }
+
     chain prerouting-raw {
         type filter hook prerouting priority raw
 
@@ -14,7 +36,8 @@ table inet filter {
         # drop packets to address not configured on incoming interface (strong host model)
         fib daddr . iif type != { local, broadcast, multicast } counter drop
 
-        tcp dport { 22, 80, 443 } notrack accept
+        tcp dport { 80, 443 } notrack accept
+        tcp dport 22 tcp flags syn notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -22,7 +45,7 @@ table inet filter {
         type filter hook output priority raw
 
         oif lo notrack accept
-        tcp sport { 22, 80, 443 } notrack accept
+        tcp sport { 80, 443 } notrack accept
         meta l4proto { icmp, ipv6-icmp } notrack accept
     }
 
@@ -30,15 +53,17 @@ table inet filter {
         type filter hook input priority filter
         policy drop
 
+        iif lo tcp dport 22 tcp flags syn ip saddr != $ip-allowlist-ssh add @ip-connlimit-ssh { ip saddr ct count over 1 } reject with tcp reset
+        iif lo tcp dport 22 tcp flags syn ip6 saddr != $ip6-allowlist-ssh add @ip6-connlimit-ssh { ip6 saddr ct count over 1 } reject with tcp reset
+
         iif lo accept
-        tcp dport { 22, 80, 443 } accept
+        tcp dport { 80, 443 } accept
         meta l4proto { icmp, ipv6-icmp } accept
+        ct state vmap { established : accept, related : accept, new : goto graceful-reject }
 
-        ct state vmap { invalid : drop, established : accept, related : accept }
+        tcp dport 22 ip saddr @ip-connlimit-ssh reject with tcp reset
-
+        tcp dport 22 ip6 saddr @ip6-connlimit-ssh reject with tcp reset
-        meta l4proto udp reject
+        tcp dport 22 synproxy mss 1460 wscale 7 timestamp sack-perm
-        meta l4proto tcp reject with tcp reset
-        reject
     }
 
     chain forward {

ssh/sshd_config

@@ -105,7 +105,6 @@ ClientAliveInterval 60
 ClientAliveCountMax 10
 #UseDNS no
 #PidFile /run/sshd.pid
-PerSourceMaxStartups 1
 MaxStartups 4096
 #PermitTunnel no
 #ChrootDirectory none