TUN inbound: Support autoSystemRoutingTable and autoOutboundsInterface on macOS and Linux as well (#6366)

https://github.com/XTLS/Xray-core/pull/6366#issuecomment-4788510365

proxy/tun/config.go

@@ -3,8 +3,6 @@ package tun
 import (
 	"context"
 	"net"
-	"sort"
-	"strings"
 	"sync"
 
 	"github.com/xtls/xray-core/common/errors"
@@ -31,95 +29,23 @@ func (updater *InterfaceUpdater) Update() {
 	updater.Lock()
 	defer updater.Unlock()
 
-	if updater.iface != nil {
+	got, err := findOutboundInterface(updater.tunIndex, updater.fixedName)
-		iface, err := net.InterfaceByIndex(updater.iface.Index)
-		if err == nil && iface.Name == updater.iface.Name {
-			return
-		}
-	}
-
-	updater.iface = nil
-
-	interfaces, err := net.Interfaces()
 	if err != nil {
 		errors.LogInfoInner(context.Background(), err, "[tun] failed to update interface")
+		updater.iface = nil
 		return
 	}
 
-	var got *net.Interface
-	if updater.fixedName != "" {
-		for _, iface := range interfaces {
-			if iface.Index == updater.tunIndex {
-				continue
-			}
-			if iface.Name == updater.fixedName {
-				got = &iface
-				break
-			}
-		}
-	} else {
-		var ifs []struct {
-			index int
-			score int
-		}
-		for i, iface := range interfaces {
-			if iface.Index == updater.tunIndex {
-				continue
-			}
-			if strings.Contains(iface.Name, "vEthernet") {
-				continue
-			}
-			if iface.Flags&net.FlagUp == 0 {
-				continue
-			}
-			if iface.Flags&net.FlagLoopback != 0 {
-				continue
-			}
-			addrs, err := iface.Addrs()
-			if err != nil || len(addrs) == 0 {
-				continue
-			}
-			ifs = append(ifs, struct {
-				index int
-				score int
-			}{i, score(&iface, addrs)})
-		}
-		sort.Slice(ifs, func(i, j int) bool {
-			if ifs[i].score != ifs[j].score {
-				return ifs[i].score > ifs[j].score
-			}
-
-			return interfaces[ifs[i].index].Name < interfaces[ifs[j].index].Name
-		})
-		if len(ifs) > 0 {
-			iface := interfaces[ifs[0].index]
-			got = &iface
-		}
-	}
-
 	if got == nil {
 		errors.LogInfo(context.Background(), "[tun] failed to update interface > got == nil")
+		updater.iface = nil
+		return
+	}
+
+	if updater.iface != nil && updater.iface.Index == got.Index && updater.iface.Name == got.Name {
 		return
 	}
 
 	updater.iface = got
 	errors.LogInfo(context.Background(), "[tun] update interface ", got.Name, " ", got.Index)
 }
-
-func score(iface *net.Interface, addrs []net.Addr) int {
-	score := 0
-
-	name := strings.ToLower(iface.Name)
-	if strings.Contains(name, "wlan") || strings.Contains(name, "wi-fi") {
-		score += 2
-	}
-
-	for _, addr := range addrs {
-		if strings.HasPrefix(addr.String(), "192.168.") {
-			score += 1
-			break
-		}
-	}
-
-	return score
-}

proxy/tun/tun_android.go

@@ -81,3 +81,17 @@ func (t *AndroidTun) newEndpoint() (stack.LinkEndpoint, error) {
 func setinterface(network, address string, fd uintptr, iface *net.Interface) error {
 	return unix.BindToDevice(int(fd), iface.Name)
 }
+
+func findOutboundInterface(tunIndex int, fixedName string) (*net.Interface, error) {
+	if fixedName == "" {
+		return nil, errors.New("automatic outbound interface selection is not supported on this platform")
+	}
+	iface, err := net.InterfaceByName(fixedName)
+	if err != nil {
+		return nil, err
+	}
+	if iface.Index == tunIndex {
+		return nil, errors.New("outbound interface cannot be the TUN interface")
+	}
+	return iface, nil
+}

proxy/tun/tun_darwin.go

@@ -3,16 +3,20 @@
 package tun
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net"
 	"net/netip"
 	"os"
 	"strconv"
+	"sync"
 	"unsafe"
 
 	"github.com/xtls/xray-core/common/buf"
+	xerrors "github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/common/platform"
+	"golang.org/x/net/route"
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/buffer"
 	"gvisor.dev/gvisor/pkg/tcpip"
@@ -42,6 +46,10 @@ type DarwinTun struct {
 	options *Config
 	tunFd   int
 	ownsFd  bool // true for macOS (we created the fd), false for iOS (fd from system)
+
+	routeMonitor     *os.File
+	routeMonitorOnce sync.Once
+	systemRoutes     []netip.Prefix
 }
 
 var (
@@ -92,15 +100,53 @@ func NewTun(options *Config) (Tun, error) {
 }
 
 func (t *DarwinTun) Start() error {
+	if !t.ownsFd {
+		return nil
+	}
+
+	if err := t.setSystemRoutes(); err != nil {
+		return err
+	}
+
+	if updater != nil {
+		fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, 0)
+		if err != nil {
+			_ = t.unsetSystemRoutes()
+			return err
+		}
+		t.routeMonitor = os.NewFile(uintptr(fd), "xray-route-monitor")
+		go t.monitorRouteChanges()
+	}
 	return nil
 }
 
 func (t *DarwinTun) Close() error {
+	t.routeMonitorOnce.Do(func() {
+		if t.routeMonitor != nil {
+			_ = t.routeMonitor.Close()
+		}
+	})
+	routeErr := t.unsetSystemRoutes()
 	if t.ownsFd {
-		return t.tunFile.Close()
+		return xerrors.Combine(routeErr, t.tunFile.Close())
 	}
 	// iOS: don't close the fd, it's owned by NetworkExtension
-	return nil
+	return routeErr
+}
+
+func (t *DarwinTun) monitorRouteChanges() {
+	buffer := make([]byte, 64*1024)
+	for {
+		if _, err := t.routeMonitor.Read(buffer); err != nil {
+			if !errors.Is(err, os.ErrClosed) {
+				xerrors.LogInfoInner(context.Background(), err, "[tun] failed to monitor route changes")
+			}
+			return
+		}
+		if updater != nil {
+			updater.Update()
+		}
+	}
 }
 
 func (t *DarwinTun) Name() (string, error) {
@@ -388,3 +434,227 @@ func setinterface(network, address string, fd uintptr, iface *net.Interface) err
 
 	return errors.Join(err1, err2)
 }
+
+func findOutboundInterface(tunIndex int, fixedName string) (*net.Interface, error) {
+	if fixedName != "" {
+		iface, err := net.InterfaceByName(fixedName)
+		if err != nil {
+			return nil, err
+		}
+		if iface.Index == tunIndex {
+			return nil, errors.New("outbound interface cannot be the TUN interface")
+		}
+		return iface, nil
+	}
+
+	rib, err := route.FetchRIB(unix.AF_UNSPEC, route.RIBTypeRoute, 0)
+	if err != nil {
+		return nil, err
+	}
+	messages, err := route.ParseRIB(route.RIBTypeRoute, rib)
+	if err != nil {
+		return nil, err
+	}
+
+	var ipv6Index int
+	for _, message := range messages {
+		routeMessage, ok := message.(*route.RouteMessage)
+		if !ok || routeMessage.Index == tunIndex {
+			continue
+		}
+		if routeMessage.Flags&unix.RTF_UP == 0 || routeMessage.Flags&unix.RTF_GATEWAY == 0 {
+			continue
+		}
+
+		family, ok := defaultRouteFamily(routeMessage)
+		if !ok {
+			continue
+		}
+		if family == unix.AF_INET {
+			return usableDarwinInterface(routeMessage.Index)
+		}
+		if family == unix.AF_INET6 && ipv6Index == 0 {
+			ipv6Index = routeMessage.Index
+		}
+	}
+
+	if ipv6Index != 0 {
+		return usableDarwinInterface(ipv6Index)
+	}
+	return nil, errors.New("default route not found")
+}
+
+func defaultRouteFamily(message *route.RouteMessage) (int, bool) {
+	if len(message.Addrs) <= unix.RTAX_NETMASK {
+		return 0, false
+	}
+
+	switch destination := message.Addrs[unix.RTAX_DST].(type) {
+	case *route.Inet4Addr:
+		mask, ok := message.Addrs[unix.RTAX_NETMASK].(*route.Inet4Addr)
+		if !ok || destination.IP != netip.IPv4Unspecified().As4() {
+			return 0, false
+		}
+		ones, bits := net.IPMask(mask.IP[:]).Size()
+		return unix.AF_INET, ones == 0 && bits == 32
+	case *route.Inet6Addr:
+		mask, ok := message.Addrs[unix.RTAX_NETMASK].(*route.Inet6Addr)
+		if !ok || destination.IP != netip.IPv6Unspecified().As16() {
+			return 0, false
+		}
+		ones, bits := net.IPMask(mask.IP[:]).Size()
+		return unix.AF_INET6, ones == 0 && bits == 128
+	default:
+		return 0, false
+	}
+}
+
+func usableDarwinInterface(index int) (*net.Interface, error) {
+	iface, err := net.InterfaceByIndex(index)
+	if err != nil {
+		return nil, err
+	}
+	if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 {
+		return nil, errors.New("default route interface is not usable")
+	}
+	return iface, nil
+}
+
+func (t *DarwinTun) setSystemRoutes() error {
+	routes, err := buildDarwinSystemRoutes(t.options.AutoSystemRoutingTable)
+	if err != nil {
+		return err
+	}
+	if len(routes) == 0 {
+		return nil
+	}
+
+	tunIndex, err := t.Index()
+	if err != nil {
+		return err
+	}
+	for _, destination := range routes {
+		if err := execDarwinRoute(unix.RTM_ADD, tunIndex, destination); err != nil {
+			_ = t.unsetSystemRoutes()
+			return xerrors.New("failed to add system route ", destination).Base(err)
+		}
+		t.systemRoutes = append(t.systemRoutes, destination)
+	}
+	return nil
+}
+
+func (t *DarwinTun) unsetSystemRoutes() error {
+	var errs []error
+	tunIndex, indexErr := t.Index()
+	if indexErr != nil && len(t.systemRoutes) > 0 {
+		errs = append(errs, indexErr)
+	}
+	for i := len(t.systemRoutes) - 1; i >= 0; i-- {
+		destination := t.systemRoutes[i]
+		if err := execDarwinRoute(unix.RTM_DELETE, tunIndex, destination); err != nil && !errors.Is(err, unix.ESRCH) {
+			errs = append(errs, xerrors.New("failed to delete system route ", destination).Base(err))
+		}
+	}
+	t.systemRoutes = nil
+	return xerrors.Combine(errs...)
+}
+
+func buildDarwinSystemRoutes(configured []string) ([]netip.Prefix, error) {
+	routes := make([]netip.Prefix, 0, len(configured))
+	seen := make(map[netip.Prefix]struct{})
+
+	appendRoute := func(prefix netip.Prefix) {
+		prefix = prefix.Masked()
+		if _, found := seen[prefix]; found {
+			return
+		}
+		seen[prefix] = struct{}{}
+		routes = append(routes, prefix)
+	}
+
+	for _, value := range configured {
+		prefix, err := netip.ParsePrefix(value)
+		if err != nil {
+			return nil, xerrors.New("invalid system route ", value).Base(err)
+		}
+		prefix = prefix.Masked()
+		if prefix.Bits() == 0 {
+			for _, protected := range darwinProtectedDefaultRoutes(prefix.Addr().Is4()) {
+				appendRoute(protected)
+			}
+			continue
+		}
+		appendRoute(prefix)
+	}
+
+	return routes, nil
+}
+
+func darwinProtectedDefaultRoutes(ipv4 bool) []netip.Prefix {
+	routes := make([]netip.Prefix, 0, 8)
+	for i := 0; i < 8; i++ {
+		if ipv4 {
+			var address [4]byte
+			address[0] = 1 << i
+			routes = append(routes, netip.PrefixFrom(netip.AddrFrom4(address), 8-i))
+		} else {
+			var address [16]byte
+			address[0] = 1 << i
+			routes = append(routes, netip.PrefixFrom(netip.AddrFrom16(address), 8-i))
+		}
+	}
+	return routes
+}
+
+func execDarwinRoute(messageType int, interfaceIndex int, destination netip.Prefix) error {
+	message := route.RouteMessage{
+		Type:    messageType,
+		Version: unix.RTM_VERSION,
+		Flags:   unix.RTF_STATIC | unix.RTF_GATEWAY,
+		Seq:     1,
+	}
+	if messageType == unix.RTM_ADD {
+		message.Flags |= unix.RTF_UP
+	}
+
+	if destination.Addr().Is4() {
+		gatewayPrefix := netip.MustParsePrefix(gateway)
+		message.Addrs = []route.Addr{
+			unix.RTAX_DST:     &route.Inet4Addr{IP: destination.Addr().As4()},
+			unix.RTAX_NETMASK: &route.Inet4Addr{IP: prefixMask4(destination.Bits())},
+			unix.RTAX_GATEWAY: &route.Inet4Addr{IP: gatewayPrefix.Addr().As4()},
+		}
+	} else {
+		message.Flags &^= unix.RTF_GATEWAY
+		message.Index = interfaceIndex
+		message.Addrs = []route.Addr{
+			unix.RTAX_DST:     &route.Inet6Addr{IP: destination.Addr().As16()},
+			unix.RTAX_NETMASK: &route.Inet6Addr{IP: prefixMask6(destination.Bits())},
+			unix.RTAX_GATEWAY: &route.LinkAddr{Index: interfaceIndex},
+		}
+	}
+
+	request, err := message.Marshal()
+	if err != nil {
+		return err
+	}
+	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, 0)
+	if err != nil {
+		return err
+	}
+	defer unix.Close(fd)
+	_, err = unix.Write(fd, request)
+	return err
+}
+
+func prefixMask4(bits int) [4]byte {
+	var mask [4]byte
+	copy(mask[:], net.CIDRMask(bits, 32))
+	return mask
+}
+
+func prefixMask6(bits int) [16]byte {
+	var mask [16]byte
+	copy(mask[:], net.CIDRMask(bits, 128))
+	return mask
+}

proxy/tun/tun_default.go

@@ -42,3 +42,17 @@ func (t *DefaultTun) newEndpoint() (stack.LinkEndpoint, error) {
 func setinterface(string, string, uintptr, *net.Interface) error {
 	return errors.New("Tun is not supported on your platform")
 }
+
+func findOutboundInterface(tunIndex int, fixedName string) (*net.Interface, error) {
+	if fixedName == "" {
+		return nil, errors.New("automatic outbound interface selection is not supported on this platform")
+	}
+	iface, err := net.InterfaceByName(fixedName)
+	if err != nil {
+		return nil, err
+	}
+	if iface.Index == tunIndex {
+		return nil, errors.New("outbound interface cannot be the TUN interface")
+	}
+	return iface, nil
+}

proxy/tun/tun_freebsd.go

@@ -147,3 +147,17 @@ func (t *FreeBSDTun) newEndpoint() (stack.LinkEndpoint, error) {
 func setinterface(network, address string, fd uintptr, iface *net.Interface) error {
 	return nil
 }
+
+func findOutboundInterface(tunIndex int, fixedName string) (*net.Interface, error) {
+	if fixedName == "" {
+		return nil, errors.New("automatic outbound interface selection is not supported on this platform")
+	}
+	iface, err := net.InterfaceByName(fixedName)
+	if err != nil {
+		return nil, err
+	}
+	if iface.Index == tunIndex {
+		return nil, errors.New("outbound interface cannot be the TUN interface")
+	}
+	return iface, nil
+}

proxy/tun/tun_linux.go

@@ -3,8 +3,11 @@
 package tun
 
 import (
+	"context"
 	"net"
+	"net/netip"
 	"strconv"
+	"sync"
 
 	"github.com/vishvananda/netlink"
 	"github.com/xtls/xray-core/common/errors"
@@ -22,6 +25,10 @@ type LinuxTun struct {
 	tunLink netlink.Link
 	options *Config
 	ownsTun bool
+
+	systemRoutes     []netlink.Route
+	routeMonitorStop chan struct{}
+	routeMonitorOnce sync.Once
 }
 
 // LinuxTun implements Tun
@@ -161,16 +168,32 @@ func (t *LinuxTun) Start() error {
 		return nil
 	}
 
-	err := netlink.LinkSetUp(t.tunLink)
+	if err := netlink.LinkSetUp(t.tunLink); err != nil {
-	if err != nil {
 		return err
 	}
 
+	if err := t.setSystemRoutes(); err != nil {
+		return err
+	}
+
+	if updater != nil {
+		t.routeMonitorStop = make(chan struct{})
+		go t.monitorRouteChanges()
+	}
+
 	return nil
 }
 
 // Close is called to shut down the tun interface
 func (t *LinuxTun) Close() error {
+	t.routeMonitorOnce.Do(func() {
+		if t.routeMonitorStop != nil {
+			close(t.routeMonitorStop)
+		}
+	})
+
+	_ = t.unsetSystemRoutes()
+
 	if t.ownsTun {
 		_ = netlink.LinkSetDown(t.tunLink)
 	}
@@ -199,3 +222,131 @@ func (t *LinuxTun) newEndpoint() (stack.LinkEndpoint, error) {
 func setinterface(network, address string, fd uintptr, iface *net.Interface) error {
 	return unix.BindToDevice(int(fd), iface.Name)
 }
+
+func (t *LinuxTun) setSystemRoutes() error {
+	if len(t.options.AutoSystemRoutingTable) == 0 {
+		return nil
+	}
+	tunIndex := t.tunLink.Attrs().Index
+	for _, cidr := range t.options.AutoSystemRoutingTable {
+		prefix, err := netip.ParsePrefix(cidr)
+		if err != nil {
+			return errors.New("invalid system route ", cidr).Base(err)
+		}
+		prefix = prefix.Masked()
+		_, ipNet, _ := net.ParseCIDR(prefix.String())
+		route := netlink.Route{
+			LinkIndex: tunIndex,
+			Dst:       ipNet,
+			Priority:  1,
+		}
+		if err := netlink.RouteAdd(&route); err != nil {
+			_ = t.unsetSystemRoutes()
+			return errors.New("failed to add system route ", cidr).Base(err)
+		}
+		t.systemRoutes = append(t.systemRoutes, route)
+	}
+	return nil
+}
+
+func (t *LinuxTun) unsetSystemRoutes() error {
+	var errs []error
+	for i := len(t.systemRoutes) - 1; i >= 0; i-- {
+		route := t.systemRoutes[i]
+		if err := netlink.RouteDel(&route); err != nil {
+			errs = append(errs, errors.New("failed to delete system route").Base(err))
+		}
+	}
+	t.systemRoutes = nil
+	return errors.Combine(errs...)
+}
+
+func (t *LinuxTun) monitorRouteChanges() {
+	routeCh := make(chan netlink.RouteUpdate)
+	if err := netlink.RouteSubscribe(routeCh, t.routeMonitorStop); err != nil {
+		errors.LogInfoInner(context.Background(), err, "[tun] failed to subscribe route changes")
+		return
+	}
+
+	linkCh := make(chan netlink.LinkUpdate)
+	if err := netlink.LinkSubscribe(linkCh, t.routeMonitorStop); err != nil {
+		errors.LogInfoInner(context.Background(), err, "[tun] failed to subscribe link changes")
+		return
+	}
+
+	for {
+		select {
+		case _, ok := <-routeCh:
+			if !ok {
+				return
+			}
+			if updater != nil {
+				updater.Update()
+			}
+		case _, ok := <-linkCh:
+			if !ok {
+				return
+			}
+			if updater != nil {
+				updater.Update()
+			}
+		case <-t.routeMonitorStop:
+			return
+		}
+	}
+}
+
+func findOutboundInterface(tunIndex int, fixedName string) (*net.Interface, error) {
+	if fixedName != "" {
+		iface, err := net.InterfaceByName(fixedName)
+		if err != nil {
+			return nil, err
+		}
+		if iface.Index == tunIndex {
+			return nil, errors.New("outbound interface cannot be the TUN interface")
+		}
+		return iface, nil
+	}
+
+	probeIPs := []net.IP{
+		net.ParseIP("8.8.8.8"),
+		net.ParseIP("2001:4860:4860::8888"),
+	}
+
+	for _, ip := range probeIPs {
+		routes, err := netlink.RouteGet(ip)
+		if err != nil || len(routes) == 0 {
+			continue
+		}
+		route := routes[0]
+		if route.LinkIndex == tunIndex {
+			continue
+		}
+
+		link, err := netlink.LinkByIndex(route.LinkIndex)
+		if err != nil {
+			continue
+		}
+		attrs := link.Attrs()
+
+		if attrs.Flags&net.FlagUp == 0 {
+			continue
+		}
+		operState := attrs.OperState
+		if operState != netlink.OperUp && operState != netlink.OperUnknown {
+			continue
+		}
+
+		if route.Src == nil || route.Src.IsLoopback() || route.Src.IsLinkLocalUnicast() {
+			continue
+		}
+
+		iface, err := net.InterfaceByIndex(route.LinkIndex)
+		if err != nil {
+			continue
+		}
+		return iface, nil
+	}
+
+	return nil, errors.New("no usable outbound interface found")
+}

proxy/tun/tun_windows.go

@@ -8,6 +8,8 @@ import (
 	go_errors "errors"
 	"net"
 	"net/netip"
+	"sort"
+	"strings"
 	"sync"
 	"unsafe"
 
@@ -307,3 +309,77 @@ func setinterface(network, address string, fd uintptr, iface *net.Interface) err
 
 	return errors.Combine(err1, err2, err3, err4)
 }
+
+func findOutboundInterface(tunIndex int, fixedName string) (*net.Interface, error) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	if fixedName != "" {
+		for _, iface := range interfaces {
+			if iface.Index != tunIndex && iface.Name == fixedName {
+				return &iface, nil
+			}
+		}
+		return nil, nil
+	}
+
+	var candidates []struct {
+		index int
+		score int
+	}
+	for i, iface := range interfaces {
+		if iface.Index == tunIndex {
+			continue
+		}
+		if strings.Contains(iface.Name, "vEthernet") {
+			continue
+		}
+		if iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		if iface.Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		addrs, err := iface.Addrs()
+		if err != nil || len(addrs) == 0 {
+			continue
+		}
+		candidates = append(candidates, struct {
+			index int
+			score int
+		}{i, scoreWindowsInterface(&iface, addrs)})
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		if candidates[i].score != candidates[j].score {
+			return candidates[i].score > candidates[j].score
+		}
+		return interfaces[candidates[i].index].Name < interfaces[candidates[j].index].Name
+	})
+	if len(candidates) == 0 {
+		return nil, nil
+	}
+
+	iface := interfaces[candidates[0].index]
+	return &iface, nil
+}
+
+func scoreWindowsInterface(iface *net.Interface, addrs []net.Addr) int {
+	score := 0
+
+	name := strings.ToLower(iface.Name)
+	if strings.Contains(name, "wlan") || strings.Contains(name, "wi-fi") {
+		score += 2
+	}
+
+	for _, addr := range addrs {
+		if strings.HasPrefix(addr.String(), "192.168.") {
+			score++
+			break
+		}
+	}
+
+	return score
+}