shinobu/router: Implement QoS using HTB

This is an initial implementation and probably still needs tuning.

machines/fuuko/services/torrent.nix

@@ -1,3 +1,9 @@
+{ lib, ... }:
+let
+  endpoint = lib.splitString ":" (lib.elemAt (import ../secrets/wireguard-qbittorrent.nix).peers 0).endpoint;
+  endpointAddress = lib.elemAt endpoint 0;
+  endpointPort = lib.toInt (lib.elemAt endpoint 1);
+in
 {
   sbruder.qbittorrent = {
     enable = true;
@@ -9,4 +15,14 @@
     enableACME = false;
     forceSSL = false;
   };
+
+  networking.nftables.ruleset = ''
+    table inet qbittorrent {
+      chain output {
+        type filter hook output priority mangle
+        # AF13 = Class 1 (lowest), high drop probability
+        ip daddr ${endpointAddress} udp dport ${toString endpointPort} ip dscp set af13
+      }
+    }
+  '';
 }

machines/shinobu/services/router/common.nix

@@ -1,3 +1,65 @@
 {
   domain = "home.sbruder.de";
+  tc = {
+    interface = "enp1s0";
+    # 4160 kbit is slightly smaller than the average upload
+    rate = "4160kbit";
+    major = 1;
+    default = 2;
+    classes = [
+      # default
+      {
+        minor = 2;
+        rate = "1000kbit";
+        prio = 50;
+      }
+      # DNS, small packets (e.g., TCP ACK)
+      {
+        minor = 3;
+        rate = "250kbit";
+        prio = 0;
+        qdiscArgs = [ "pfifo_fast" ];
+      }
+      # interactive SSH
+      {
+        minor = 4;
+        rate = "128kbit";
+        prio = 2;
+      }
+      # torrent
+      {
+        minor = 5;
+        rate = "250kbit";
+        ceil = "3000kbit";
+        prio = 100;
+      }
+      # HTTP
+      {
+        minor = 6;
+        rate = "1500kbit";
+        prio = 25;
+      }
+      # wg-home
+      {
+        minor = 7;
+        rate = "250kbit";
+        prio = 10;
+      }
+      # VoIP
+      {
+        minor = 8;
+        rate = "256kbit";
+        ceil = "384kbit";
+        prio = 3;
+        qdiscArgs = [ "pfifo_fast" ];
+      }
+      # Backup
+      {
+        minor = 9;
+        rate = "350kbit";
+        ceil = "3000kbit";
+        prio = 90;
+      }
+    ];
+  };
 }

machines/shinobu/services/router/default.nix

@@ -33,6 +33,7 @@ in
   imports = [
     ./dnsmasq.nix
     ./nft.nix
+    ./tc.nix
     ./wlan.nix
   ];
 

machines/shinobu/services/router/rules.nft

@@ -46,6 +46,42 @@ table inet restrict-wan {
 	}
 }
 
+# Traffic control
+# Neets output and prerouting to match packets from localhost and lan
+table inet tc {
+	chain output {
+		type route hook output priority mangle
+
+		# hardcoded, but unlikely to change
+		ip daddr { "9.9.9.9", "149.112.112.112" } meta priority set 1:3 counter return comment "DNS (4)"
+		ip6 daddr { "2620:fe::9", "2620:fe::fe" } meta priority set 1:3 counter return comment "DNS (6)"
+
+		jump common
+	}
+
+	chain forward {
+		type filter hook forward priority mangle
+		jump common
+	}
+
+	chain common {
+		meta l4proto tcp meta length 1-64 meta priority set 1:3 counter return comment "small tcp packets"
+
+		tcp dport 22 ip  dscp af21 meta priority set 1:4 counter return comment "interactive SSH (4)"
+		tcp dport 22 ip6 dscp af21 meta priority set 1:4 counter return comment "interactive SSH (6)"
+
+		meta l4proto udp ip dscp af13 meta priority set 1:5 ip dscp set cs0 counter return comment "fuuko torrent"
+
+		meta l4proto { tcp, udp } th dport 443 meta priority set 1:6 counter return comment "HTTPS"
+
+		ip daddr 168.119.176.53 udp dport 51820 meta priority set 1:7 counter return comment "wg-home"
+
+		meta l4proto { tcp, udp } ip  dscp ef meta priority set 1:8 counter return comment "VoIP (4)"
+		meta l4proto { tcp, udp } ip6 dscp ef meta priority set 1:8 counter return comment "VoIP (6)"
+		meta l4proto { tcp, udp } th dport 64738 meta priority set 1:8 counter return comment "Mumble"
+	}
+}
+
 # Tracing infrastructure, can be used for debugging (nft monitor trace)
 table inet trace {
 	chain prerouting {

machines/shinobu/services/router/tc.nix

@@ -0,0 +1,49 @@
+# I measured the link capacity with iperf3:
+# The download is pretty exactly 7.5 MiB/s ≈ 62.9 Mbit/s
+# The upstream is more complicated.
+# It initially bursts around 50% higher than the sustained speed
+# and then falls down to 569 KiB/s ≈ 4.66 Mbit/s
+# However, abound every 2 to 3 seconds, it drops to 380 KiB/s ≈ 3.11 Mbit/s.
+# It does so pretty consistently and always at exactly that rate.
+# I averaged a longer iperf3 run to around 509 Kbit/s ≈ 4.17 Mbit/s (excluding the initial burst).
+{ lib, pkgs, utils, ... }:
+let
+  cfg = ((import ./common.nix).tc);
+
+  mkClass =
+    { minor
+    , rate
+    , ceil ? cfg.rate
+    , burst ? "15k"
+    , qdiscArgs ? [ "fq_codel" ]
+    , prio
+    }: ''
+      tc class add dev ${cfg.interface} parent ${toString cfg.major}:1 classid ${toString cfg.major}:${toString minor} htb rate ${rate} ceil ${ceil} burst ${burst} prio ${toString prio}
+      tc qdisc add dev ${cfg.interface} parent ${toString cfg.major}:${toString minor} handle ${toString minor}:1 ${lib.escapeShellArgs qdiscArgs}
+    '';
+in
+{
+  systemd.services.traffic-control = {
+    after = [ "sys-subsystem-net-devices-${utils.escapeSystemdPath cfg.interface}.device" ];
+    bindsTo = [ "sys-subsystem-net-devices-${utils.escapeSystemdPath cfg.interface}.device" ];
+    wantedBy = [ "network-online.target" ];
+
+    path = with pkgs; [ iproute2 ];
+
+    script = ''
+      set -euo pipefail
+
+      # deleting might fail
+      tc qdisc del root dev ${cfg.interface} || true
+
+      tc qdisc add dev ${cfg.interface} root handle ${toString cfg.major}:0 htb default 2
+      tc class add dev ${cfg.interface} parent ${toString cfg.major}:0 classid ${toString cfg.major}:1 htb rate ${toString cfg.rate} burst 15k
+
+      ${lib.concatMapStrings mkClass cfg.classes}
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+    };
+  };
+}