From fad8652bac6b3b0a447d86e7307586eac4f9dec9 Mon Sep 17 00:00:00 2001 From: nyanloutre Date: Thu, 28 Nov 2024 12:14:26 +0100 Subject: [PATCH 1/5] Try to use multiple tables with CONNMARK --- systems/LoutreOS/network.nix | 96 +++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 13 deletions(-) diff --git a/systems/LoutreOS/network.nix b/systems/LoutreOS/network.nix index 141298b..9dec1e7 100644 --- a/systems/LoutreOS/network.nix +++ b/systems/LoutreOS/network.nix @@ -114,8 +114,41 @@ ]; }; + ################# + # ROUTING RULES # + ################# + + # 0: from all lookup local + # 60: from all iif lo dport 25 lookup vpn + # 4000: from all fwmark 0x1 lookup fiber + # 5000: from all fwmark 0x2 lookup lte + # 6000: from all fwmark 0x3 lookup vpn + # 32766: from all lookup main + # 32767: from all lookup default + # 40000: from all lookup fiber + # 50000: from all lookup lte + + # TODO + + ################## + # iptables rules # + ################## + + # # Restore the packet's CONNMARK to the MARK. + # iptables -A PREROUTING -t mangle -j CONNMARK --restore-mark + # # If packet MARK is set, then it means that there is already a connection mark + # iptables -t mangle -A PREROUTING -m mark ! --mark 0 -j ACCEPT + # # Else, we need to mark the packet. If the packet is incoming on bouygues then set MARK to 1 + # iptables -A PREROUTING -t mangle -i bouygues -j MARK --set-mark 1 + # iptables -A PREROUTING -t mangle -i enp0s21u1 -j MARK --set-mark 2 + # iptables -A PREROUTING -t mangle -i wg0 -j MARK --set-mark 3 + # # Save MARK to CONNMARK. + # iptables -A PREROUTING -t mangle -j CONNMARK --save-mark + systemd.network = let routeTables = { + fiber = 1; + lte = 2; vpn = 3; }; in { @@ -164,14 +197,17 @@ # Set route metric to highest priority # Set DHCP client magic settings for Bouygues "40-bouygues" = { - dhcpV4Config.RouteMetric = 1; + dhcpV4Config.RouteTable = routeTables.fiber; dhcpV6Config = { DUIDRawData = "00:03:00:01:E8:AD:A6:21:73:68"; WithoutRA = "solicit"; }; - ipv6AcceptRAConfig.DHCPv6Client = true; + ipv6AcceptRAConfig = { + DHCPv6Client = true; + RouteTable = routeTables.fiber; + }; networkConfig = { KeepConfiguration = "dhcp-on-stop"; @@ -181,6 +217,25 @@ # Static attribution of first IPv6 subnet dhcpPrefixDelegationConfig.SubnetId = "0"; + + # Route everything to fiber link with a priority of 40000 + routingPolicyRules = [ + { + routingPolicyRuleConfig = { + FirewallMark = 1; + Table = routeTables.fiber; + Priority = 4000; + Family = "both"; + }; + } + { + routingPolicyRuleConfig = { + Table = routeTables.fiber; + Priority = 40000; + Family = "both"; + }; + } + ]; }; # Don't check VLAN physical interface as it is not directly used @@ -191,7 +246,28 @@ ####### # Set LTE route to lower priority - "40-enp0s21u1".dhcpV4Config.RouteMetric = 1024; + "40-enp0s21u1" = { + dhcpV4Config.RouteTable = routeTables.lte; + + # Route all to lte link with a priority of 50000 + routingPolicyRules = [ + { + routingPolicyRuleConfig = { + FirewallMark = 2; + Table = routeTables.lte; + Priority = 5000; + Family = "both"; + }; + } + { + routingPolicyRuleConfig = { + Table = routeTables.lte; + Priority = 50000; + Family = "both"; + }; + } + ]; + }; ####### # VPN # @@ -214,7 +290,7 @@ IncomingInterface = "lo"; DestinationPort = "25"; Table = routeTables.vpn; - Priority = 50; + Priority = 60; Family = "both"; }; } @@ -222,16 +298,10 @@ # Allow server to respond on the wg0 interface requests { routingPolicyRuleConfig = { - From = vpnIPv4; + FirewallMark = 3; Table = routeTables.vpn; - Priority = 49; - }; - } - { - routingPolicyRuleConfig = { - From = vpnIPv6; - Table = routeTables.vpn; - Priority = 49; + Priority = 6000; + Family = "both"; }; } ]; From e15cd2091e245dbbdb6df4c8fdcd4464f2b6141d Mon Sep 17 00:00:00 2001 From: nyanloutre Date: Thu, 28 Nov 2024 15:18:36 +0100 Subject: [PATCH 2/5] remove NixOS nat config, set networkd masquerade and add FW marks --- systems/LoutreOS/network.nix | 86 ++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/systems/LoutreOS/network.nix b/systems/LoutreOS/network.nix index 9dec1e7..e65303b 100644 --- a/systems/LoutreOS/network.nix +++ b/systems/LoutreOS/network.nix @@ -52,18 +52,6 @@ enp0s21u1.useDHCP = true; }; - # NAT bouygues <-> eno2 - nat = { - enable = true; - externalInterface = "bouygues"; - internalIPs = [ "10.30.0.0/16" ]; - internalInterfaces = [ "eno2" ]; - forwardPorts = [ - { destination = "10.30.0.1:22"; proto = "tcp"; sourcePort = 8443;} - { destination = "10.30.135.35:25565"; proto = "tcp"; sourcePort = 25565; loopbackIPs=[ "195.36.180.44" ];} - ]; - }; - firewall = { enable = true; allowedTCPPorts = [ 80 443 ]; @@ -84,21 +72,59 @@ }; extraCommands = '' + + ################ + # MANGLE rules # + ################ + + # Clean and recreate target + ip46tables -w -t mangle -D PREROUTING -j loutreos-mangle-pre 2>/dev/null || true + ip46tables -w -t mangle -F loutreos-mangle-pre 2>/dev/null || true + ip46tables -w -t mangle -X loutreos-mangle-pre 2>/dev/null || true + ip46tables -w -t mangle -N loutreos-mangle-pre + + # Restore the packet's CONNMARK to the MARK for existing connections + ip46tables -w -t mangle -A loutreos-mangle-pre -j CONNMARK --restore-mark + + # If packet MARK is set, then it means that there is already a connection mark + ip46tables -w -t mangle -A loutreos-mangle-pre -m mark ! --mark 0 -j ACCEPT + + # Else, we need to mark the packet. + # If the packet is incoming on bouygues then set MARK to 1, LTE MARK 2 and VPN MARK 3 + ip46tables -w -t mangle -A loutreos-mangle-pre -i bouygues -j MARK --set-mark 1 + ip46tables -w -t mangle -A loutreos-mangle-pre -i enp0s21u1 -j MARK --set-mark 2 + ip46tables -w -t mangle -A loutreos-mangle-pre -i wg0 -j MARK --set-mark 3 + + # Jump to newly created target + ip46tables -w -t mangle -A PREROUTING -j loutreos-mangle-pre + + # Save MARK to CONNMARK. + ip46tables -w -t mangle -D POSTROUTING -j CONNMARK --save-mark 2>/dev/null || true + ip46tables -w -t mangle -A POSTROUTING -j CONNMARK --save-mark + + ###################### + # IPv6 FORWARD rules # + ###################### + # Forward all IPv6 traffic from local network and reject incoming traffic ip6tables -w -D FORWARD -j loutreos-forward 2>/dev/null || true ip6tables -w -F loutreos-forward 2>/dev/null || true ip6tables -w -X loutreos-forward 2>/dev/null || true ip6tables -w -N loutreos-forward - ip6tables -A loutreos-forward -m state --state RELATED,ESTABLISHED -j ACCEPT - ip6tables -A loutreos-forward -j ACCEPT -i eno2 - ip6tables -A loutreos-forward -j nixos-fw-log-refuse + ip6tables -w -A loutreos-forward -m state --state RELATED,ESTABLISHED -j ACCEPT + ip6tables -w -A loutreos-forward -j ACCEPT -i eno2 + ip6tables -w -A loutreos-forward -j nixos-fw-log-refuse ip6tables -w -A FORWARD -j loutreos-forward + ############################################# + # Enable server access when fiber link down # + ############################################# + # Redirect local network request from server external IP to internal IP - # Make the server available even without internet access iptables -t nat -D PREROUTING -s 10.30.0.0/16 -d 176.180.172.105 -j DNAT --to 10.30.0.1 || true iptables -t nat -A PREROUTING -s 10.30.0.0/16 -d 176.180.172.105 -j DNAT --to 10.30.0.1 ''; + # remove refs to nixos-fw-log-refuse before restarting firewall # prevents "ressource busy" errors extraStopCommands = '' @@ -119,31 +145,14 @@ ################# # 0: from all lookup local - # 60: from all iif lo dport 25 lookup vpn - # 4000: from all fwmark 0x1 lookup fiber + # 60: from all iif lo dport 25 lookup vpn # mails are forced to vpn table + # 4000: from all fwmark 0x1 lookup fiber # fwmark indicate established connection that must go through same interface # 5000: from all fwmark 0x2 lookup lte # 6000: from all fwmark 0x3 lookup vpn - # 32766: from all lookup main + # 32766: from all lookup main # main table should contain no default routes, only local network routes # 32767: from all lookup default - # 40000: from all lookup fiber - # 50000: from all lookup lte - - # TODO - - ################## - # iptables rules # - ################## - - # # Restore the packet's CONNMARK to the MARK. - # iptables -A PREROUTING -t mangle -j CONNMARK --restore-mark - # # If packet MARK is set, then it means that there is already a connection mark - # iptables -t mangle -A PREROUTING -m mark ! --mark 0 -j ACCEPT - # # Else, we need to mark the packet. If the packet is incoming on bouygues then set MARK to 1 - # iptables -A PREROUTING -t mangle -i bouygues -j MARK --set-mark 1 - # iptables -A PREROUTING -t mangle -i enp0s21u1 -j MARK --set-mark 2 - # iptables -A PREROUTING -t mangle -i wg0 -j MARK --set-mark 3 - # # Save MARK to CONNMARK. - # iptables -A PREROUTING -t mangle -j CONNMARK --save-mark + # 40000: from all lookup fiber # first table encountered with a default route if fiber is up + # 50000: from all lookup lte # first table encountered with a default route if fiber is down systemd.network = let routeTables = { @@ -317,6 +326,7 @@ IPv6SendRA = true; DHCPPrefixDelegation = true; DHCPServer = true; + IPMasquerade = "ipv4"; }; dhcpServerConfig = { EmitRouter = true; From 268dbf181ab5aa27d49bb43eb79b4d4da9941b70 Mon Sep 17 00:00:00 2001 From: nyanloutre Date: Thu, 28 Nov 2024 15:30:55 +0100 Subject: [PATCH 3/5] simplify VPN network config --- systems/LoutreOS/network.nix | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/systems/LoutreOS/network.nix b/systems/LoutreOS/network.nix index e65303b..6c97c79 100644 --- a/systems/LoutreOS/network.nix +++ b/systems/LoutreOS/network.nix @@ -283,14 +283,11 @@ ####### # Wireguard ARN network configuation - "10-wg0" = let - vpnIPv4 = "89.234.141.196/32"; - vpnIPv6 = "2a00:5881:8119:400::1/128"; - in { + "10-wg0" = { matchConfig.Name = "wg0"; address = [ - vpnIPv4 - vpnIPv6 + "89.234.141.196/32" + "2a00:5881:8119:400::1/128" ]; routingPolicyRules = [ # Route outgoing emails to VPN table From 933d758e3b206865165b02ce2a4059e5b3bd8484 Mon Sep 17 00:00:00 2001 From: nyanloutre Date: Tue, 10 Dec 2024 00:33:31 +0100 Subject: [PATCH 4/5] force packet forwarding to true --- systems/LoutreOS/network.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/systems/LoutreOS/network.nix b/systems/LoutreOS/network.nix index 6c97c79..869597c 100644 --- a/systems/LoutreOS/network.nix +++ b/systems/LoutreOS/network.nix @@ -2,7 +2,12 @@ { boot = { - kernel.sysctl."net.ipv6.conf.all.forwarding" = true; + kernel.sysctl = { + "net.ipv6.conf.all.forwarding" = true; + "net.ipv6.conf.default.forwarding" = true; + "net.ipv4.conf.all.forwarding" = true; + "net.ipv4.conf.default.forwarding" = true; + }; }; # Enable LTE drivers From 59435f987b0e78706c2735230168d7454d4da6e5 Mon Sep 17 00:00:00 2001 From: nyanloutre Date: Tue, 10 Dec 2024 00:34:58 +0100 Subject: [PATCH 5/5] fix rule priority main table should be first, else masqueraded packet responses will never reach internal network --- systems/LoutreOS/network.nix | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/systems/LoutreOS/network.nix b/systems/LoutreOS/network.nix index 869597c..4c51361 100644 --- a/systems/LoutreOS/network.nix +++ b/systems/LoutreOS/network.nix @@ -151,13 +151,13 @@ # 0: from all lookup local # 60: from all iif lo dport 25 lookup vpn # mails are forced to vpn table - # 4000: from all fwmark 0x1 lookup fiber # fwmark indicate established connection that must go through same interface - # 5000: from all fwmark 0x2 lookup lte - # 6000: from all fwmark 0x3 lookup vpn # 32766: from all lookup main # main table should contain no default routes, only local network routes # 32767: from all lookup default - # 40000: from all lookup fiber # first table encountered with a default route if fiber is up - # 50000: from all lookup lte # first table encountered with a default route if fiber is down + # 41000: from all fwmark 0x1 lookup fiber # fwmark indicate established connection that must go through same interface + # 42000: from all fwmark 0x2 lookup lte + # 43000: from all fwmark 0x3 lookup vpn + # 51000: from all lookup fiber # first table encountered with a default route if fiber is up + # 52000: from all lookup lte # first table encountered with a default route if fiber is down systemd.network = let routeTables = { @@ -238,14 +238,14 @@ routingPolicyRuleConfig = { FirewallMark = 1; Table = routeTables.fiber; - Priority = 4000; + Priority = 41000; Family = "both"; }; } { routingPolicyRuleConfig = { Table = routeTables.fiber; - Priority = 40000; + Priority = 51000; Family = "both"; }; } @@ -269,14 +269,14 @@ routingPolicyRuleConfig = { FirewallMark = 2; Table = routeTables.lte; - Priority = 5000; + Priority = 42000; Family = "both"; }; } { routingPolicyRuleConfig = { Table = routeTables.lte; - Priority = 50000; + Priority = 52000; Family = "both"; }; } @@ -311,7 +311,7 @@ routingPolicyRuleConfig = { FirewallMark = 3; Table = routeTables.vpn; - Priority = 6000; + Priority = 43000; Family = "both"; }; }