Xray-core: Refactor geodata (#5814)

https://github.com/XTLS/Xray-core/issues/4422#issuecomment-3533007890

Breaking changes https://github.com/XTLS/Xray-core/pull/5569

Reverts https://github.com/XTLS/Xray-core/pull/5505

Closes https://github.com/XTLS/Xray-core/pull/643

infra/conf/dns.go

@@ -3,6 +3,7 @@ package conf
 import (
 	"bufio"
 	"encoding/json"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -10,8 +11,8 @@ import (
 	"strings"
 
 	"github.com/xtls/xray-core/app/dns"
-	"github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/common/errors"
+	"github.com/xtls/xray-core/common/geodata"
 	"github.com/xtls/xray-core/common/net"
 )
 
@@ -20,7 +21,7 @@ type NameServerConfig struct {
 	ClientIP        *Address   `json:"clientIp"`
 	Port            uint16     `json:"port"`
 	SkipFallback    bool       `json:"skipFallback"`
-	Domains         []string   `json:"domains"`
+	Domains         StringList `json:"domains"`
 	ExpectedIPs     StringList `json:"expectedIPs"`
 	ExpectIPs       StringList `json:"expectIPs"`
 	QueryStrategy   string     `json:"queryStrategy"`
@@ -46,7 +47,7 @@ func (c *NameServerConfig) UnmarshalJSON(data []byte) error {
 		ClientIP        *Address   `json:"clientIp"`
 		Port            uint16     `json:"port"`
 		SkipFallback    bool       `json:"skipFallback"`
-		Domains         []string   `json:"domains"`
+		Domains         StringList `json:"domains"`
 		ExpectedIPs     StringList `json:"expectedIPs"`
 		ExpectIPs       StringList `json:"expectIPs"`
 		QueryStrategy   string     `json:"queryStrategy"`
@@ -80,45 +81,14 @@ func (c *NameServerConfig) UnmarshalJSON(data []byte) error {
 	return errors.New("failed to parse name server: ", string(data))
 }
 
-func toDomainMatchingType(t router.Domain_Type) dns.DomainMatchingType {
-	switch t {
-	case router.Domain_Domain:
-		return dns.DomainMatchingType_Subdomain
-	case router.Domain_Full:
-		return dns.DomainMatchingType_Full
-	case router.Domain_Plain:
-		return dns.DomainMatchingType_Keyword
-	case router.Domain_Regex:
-		return dns.DomainMatchingType_Regex
-	default:
-		panic("unknown domain type")
-	}
-}
-
 func (c *NameServerConfig) Build() (*dns.NameServer, error) {
 	if c.Address == nil {
-		return nil, errors.New("NameServer address is not specified.")
+		return nil, errors.New("nameserver address is not specified")
 	}
 
-	var domains []*dns.NameServer_PriorityDomain
-	var originalRules []*dns.NameServer_OriginalRule
-
-	for _, rule := range c.Domains {
-		parsedDomain, err := parseDomainRule(rule)
-		if err != nil {
-			return nil, errors.New("invalid domain rule: ", rule).Base(err)
-		}
-
-		for _, pd := range parsedDomain {
-			domains = append(domains, &dns.NameServer_PriorityDomain{
-				Type:   toDomainMatchingType(pd.Type),
-				Domain: pd.Value,
-			})
-		}
-		originalRules = append(originalRules, &dns.NameServer_OriginalRule{
-			Rule: rule,
-			Size: uint32(len(parsedDomain)),
-		})
+	domainRules, err := geodata.ParseDomainRules(c.Domains, geodata.Domain_Substr)
+	if err != nil {
+		return nil, err
 	}
 
 	if len(c.ExpectedIPs) == 0 {
@@ -145,14 +115,14 @@ func (c *NameServerConfig) Build() (*dns.NameServer, error) {
 		}
 	}
 
-	expectedGeoipList, err := ToCidrList(newExpectedIPs)
+	expectedIPRules, err := geodata.ParseIPRules(newExpectedIPs)
 	if err != nil {
-		return nil, errors.New("invalid expected IP rule: ", c.ExpectedIPs).Base(err)
+		return nil, err
 	}
 
-	unexpectedGeoipList, err := ToCidrList(newUnexpectedIPs)
+	unexpectedIPRules, err := geodata.ParseIPRules(newUnexpectedIPs)
 	if err != nil {
-		return nil, errors.New("invalid unexpected IP rule: ", c.UnexpectedIPs).Base(err)
+		return nil, err
 	}
 
 	var myClientIP []byte
@@ -169,32 +139,24 @@ func (c *NameServerConfig) Build() (*dns.NameServer, error) {
 			Address: c.Address.Build(),
 			Port:    uint32(c.Port),
 		},
-		ClientIp:          myClientIP,
-		SkipFallback:      c.SkipFallback,
-		PrioritizedDomain: domains,
-		ExpectedGeoip:     expectedGeoipList,
-		OriginalRules:     originalRules,
-		QueryStrategy:     resolveQueryStrategy(c.QueryStrategy),
-		ActPrior:          actPrior,
-		Tag:               c.Tag,
-		TimeoutMs:         c.TimeoutMs,
-		DisableCache:      c.DisableCache,
-		ServeStale:        c.ServeStale,
-		ServeExpiredTTL:   c.ServeExpiredTTL,
-		FinalQuery:        c.FinalQuery,
-		UnexpectedGeoip:   unexpectedGeoipList,
-		ActUnprior:        actUnprior,
+		ClientIp:        myClientIP,
+		SkipFallback:    c.SkipFallback,
+		Domain:          domainRules,
+		ExpectedIp:      expectedIPRules,
+		QueryStrategy:   resolveQueryStrategy(c.QueryStrategy),
+		ActPrior:        actPrior,
+		Tag:             c.Tag,
+		TimeoutMs:       c.TimeoutMs,
+		DisableCache:    c.DisableCache,
+		ServeStale:      c.ServeStale,
+		ServeExpiredTTL: c.ServeExpiredTTL,
+		FinalQuery:      c.FinalQuery,
+		UnexpectedIp:    unexpectedIPRules,
+		ActUnprior:      actUnprior,
 	}, nil
 }
 
-var typeMap = map[router.Domain_Type]dns.DomainMatchingType{
-	router.Domain_Full:   dns.DomainMatchingType_Full,
-	router.Domain_Domain: dns.DomainMatchingType_Subdomain,
-	router.Domain_Plain:  dns.DomainMatchingType_Keyword,
-	router.Domain_Regex:  dns.DomainMatchingType_Regex,
-}
-
-// DNSConfig is a JSON serializable object for dns.Config.
+// DNSConfig is a JSON serializable object for dns.Config
 type DNSConfig struct {
 	Servers                []*NameServerConfig `json:"servers"`
 	Hosts                  *HostsWrapper       `json:"hosts"`
@@ -246,7 +208,7 @@ type HostsWrapper struct {
 	Hosts map[string]*HostAddress
 }
 
-func getHostMapping(ha *HostAddress) *dns.Config_HostMapping {
+func newHostMapping(ha *HostAddress) *dns.Config_HostMapping {
 	if ha.addr != nil {
 		if ha.addr.Family().IsDomain() {
 			return &dns.Config_HostMapping{
@@ -290,109 +252,15 @@ func (m *HostsWrapper) UnmarshalJSON(data []byte) error {
 
 // Build implements Buildable
 func (m *HostsWrapper) Build() ([]*dns.Config_HostMapping, error) {
-	mappings := make([]*dns.Config_HostMapping, 0, 20)
-
-	domains := make([]string, 0, len(m.Hosts))
-	for domain := range m.Hosts {
-		domains = append(domains, domain)
-	}
-	sort.Strings(domains)
-
-	for _, domain := range domains {
-		switch {
-		case strings.HasPrefix(domain, "domain:"):
-			domainName := domain[7:]
-			if len(domainName) == 0 {
-				return nil, errors.New("empty domain type of rule: ", domain)
-			}
-			mapping := getHostMapping(m.Hosts[domain])
-			mapping.Type = dns.DomainMatchingType_Subdomain
-			mapping.Domain = domainName
-			mappings = append(mappings, mapping)
-
-		case strings.HasPrefix(domain, "geosite:"):
-			listName := domain[8:]
-			if len(listName) == 0 {
-				return nil, errors.New("empty geosite rule: ", domain)
-			}
-			geositeList, err := loadGeositeWithAttr("geosite.dat", listName)
-			if err != nil {
-				return nil, errors.New("failed to load geosite: ", listName).Base(err)
-			}
-			for _, d := range geositeList {
-				mapping := getHostMapping(m.Hosts[domain])
-				mapping.Type = typeMap[d.Type]
-				mapping.Domain = d.Value
-				mappings = append(mappings, mapping)
-			}
-
-		case strings.HasPrefix(domain, "regexp:"):
-			regexpVal := domain[7:]
-			if len(regexpVal) == 0 {
-				return nil, errors.New("empty regexp type of rule: ", domain)
-			}
-			mapping := getHostMapping(m.Hosts[domain])
-			mapping.Type = dns.DomainMatchingType_Regex
-			mapping.Domain = regexpVal
-			mappings = append(mappings, mapping)
-
-		case strings.HasPrefix(domain, "keyword:"):
-			keywordVal := domain[8:]
-			if len(keywordVal) == 0 {
-				return nil, errors.New("empty keyword type of rule: ", domain)
-			}
-			mapping := getHostMapping(m.Hosts[domain])
-			mapping.Type = dns.DomainMatchingType_Keyword
-			mapping.Domain = keywordVal
-			mappings = append(mappings, mapping)
-
-		case strings.HasPrefix(domain, "full:"):
-			fullVal := domain[5:]
-			if len(fullVal) == 0 {
-				return nil, errors.New("empty full domain type of rule: ", domain)
-			}
-			mapping := getHostMapping(m.Hosts[domain])
-			mapping.Type = dns.DomainMatchingType_Full
-			mapping.Domain = fullVal
-			mappings = append(mappings, mapping)
-
-		case strings.HasPrefix(domain, "dotless:"):
-			mapping := getHostMapping(m.Hosts[domain])
-			mapping.Type = dns.DomainMatchingType_Regex
-			switch substr := domain[8:]; {
-			case substr == "":
-				mapping.Domain = "^[^.]*$"
-			case !strings.Contains(substr, "."):
-				mapping.Domain = "^[^.]*" + substr + "[^.]*$"
-			default:
-				return nil, errors.New("substr in dotless rule should not contain a dot: ", substr)
-			}
-			mappings = append(mappings, mapping)
-
-		case strings.HasPrefix(domain, "ext:"):
-			kv := strings.Split(domain[4:], ":")
-			if len(kv) != 2 {
-				return nil, errors.New("invalid external resource: ", domain)
-			}
-			filename := kv[0]
-			list := kv[1]
-			geositeList, err := loadGeositeWithAttr(filename, list)
-			if err != nil {
-				return nil, errors.New("failed to load domain list: ", list, " from ", filename).Base(err)
-			}
-			for _, d := range geositeList {
-				mapping := getHostMapping(m.Hosts[domain])
-				mapping.Type = typeMap[d.Type]
-				mapping.Domain = d.Value
-				mappings = append(mappings, mapping)
-			}
-
-		default:
-			mapping := getHostMapping(m.Hosts[domain])
-			mapping.Type = dns.DomainMatchingType_Full
-			mapping.Domain = domain
-			mappings = append(mappings, mapping)
+	mappings := make([]*dns.Config_HostMapping, 0, len(m.Hosts))
+	for rule, addrs := range m.Hosts {
+		mapping := newHostMapping(addrs)
+		rule, err := geodata.ParseDomainRule(rule, geodata.Domain_Full)
+		if err != nil {
+			return nil, err
 		}
+		mapping.Domain = rule
+		mappings = append(mappings, mapping)
 	}
 	return mappings, nil
 }
@@ -504,9 +372,7 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
 		if err != nil {
 			return nil, errors.New("failed to read system hosts").Base(err)
 		}
-		for domain, ips := range systemHosts {
-			config.StaticHosts = append(config.StaticHosts, &dns.Config_HostMapping{Ip: ips, Domain: domain, Type: dns.DomainMatchingType_Full})
-		}
+		config.StaticHosts = append(config.StaticHosts, systemHosts...)
 	}
 
 	return config, nil
@@ -527,7 +393,7 @@ func resolveQueryStrategy(queryStrategy string) dns.QueryStrategy {
 	}
 }
 
-func readSystemHosts() (map[string][][]byte, error) {
+func readSystemHosts() ([]*dns.Config_HostMapping, error) {
 	var hostsPath string
 	switch runtime.GOOS {
 	case "windows":
@@ -542,12 +408,16 @@ func readSystemHosts() (map[string][][]byte, error) {
 	}
 	defer file.Close()
 
-	hostsMap := make(map[string][][]byte)
-	scanner := bufio.NewScanner(file)
+	return readSystemHostsFrom(file)
+}
+
+func readSystemHostsFrom(r io.Reader) ([]*dns.Config_HostMapping, error) {
+	hosts := make(map[string][][]byte, 16)
+	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
 		if i := strings.IndexByte(line, '#'); i >= 0 {
-			// Discard comments.
+			// Strip inline comments before splitting the line into fields.
 			line = line[0:i]
 		}
 		f := strings.Fields(line)
@@ -558,19 +428,28 @@ func readSystemHosts() (map[string][][]byte, error) {
 		if addr.Family().IsDomain() {
 			continue
 		}
-		ip := addr.IP()
 		for i := 1; i < len(f); i++ {
 			domain := strings.TrimSuffix(f[i], ".")
 			domain = strings.ToLower(domain)
-			if v, ok := hostsMap[domain]; ok {
-				hostsMap[domain] = append(v, ip)
-			} else {
-				hostsMap[domain] = [][]byte{ip}
-			}
+			hosts[domain] = append(hosts[domain], addr.IP())
 		}
 	}
 	if err := scanner.Err(); err != nil {
 		return nil, err
 	}
+
+	hostsMap := make([]*dns.Config_HostMapping, 0, len(hosts))
+	for domain, ips := range hosts {
+		// ParseDomainRule accepts rule syntax too, not just plain domains.
+		rule, err := geodata.ParseDomainRule(domain, geodata.Domain_Full)
+		if err != nil {
+			return nil, err
+		}
+		hostsMap = append(hostsMap, &dns.Config_HostMapping{
+			Domain: rule,
+			Ip:     ips,
+		})
+	}
+
 	return hostsMap, nil
 }

infra/conf/dns_test.go

@@ -4,10 +4,13 @@ import (
 	"encoding/json"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/xtls/xray-core/app/dns"
+	"github.com/xtls/xray-core/common/geodata"
 	"github.com/xtls/xray-core/common/net"
 	. "github.com/xtls/xray-core/infra/conf"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
 )
 
 func TestDNSConfigParsing(t *testing.T) {
@@ -22,7 +25,7 @@ func TestDNSConfigParsing(t *testing.T) {
 	}
 	expectedServeStale := true
 	expectedServeExpiredTTL := uint32(172800)
-	runMultiTestCase(t, []TestCase{
+	testCases := []TestCase{
 		{
 			Input: `{
 				"servers": [{
@@ -61,16 +64,9 @@ func TestDNSConfigParsing(t *testing.T) {
 							Port:    5353,
 						},
 						SkipFallback: true,
-						PrioritizedDomain: []*dns.NameServer_PriorityDomain{
+						Domain: []*geodata.DomainRule{
 							{
-								Type:   dns.DomainMatchingType_Subdomain,
-								Domain: "example.com",
-							},
-						},
-						OriginalRules: []*dns.NameServer_OriginalRule{
-							{
-								Rule: "domain:example.com",
-								Size: 1,
+								Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Domain, Value: "example.com"}},
 							},
 						},
 						ServeStale:      &expectedServeStale,
@@ -80,28 +76,23 @@ func TestDNSConfigParsing(t *testing.T) {
 				},
 				StaticHosts: []*dns.Config_HostMapping{
 					{
-						Type:          dns.DomainMatchingType_Subdomain,
-						Domain:        "example.com",
+						Domain:        &geodata.DomainRule{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Domain, Value: "example.com"}}},
 						ProxiedDomain: "google.com",
 					},
 					{
-						Type:   dns.DomainMatchingType_Full,
-						Domain: "example.com",
+						Domain: &geodata.DomainRule{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Full, Value: "example.com"}}},
 						Ip:     [][]byte{{127, 0, 0, 1}},
 					},
 					{
-						Type:   dns.DomainMatchingType_Keyword,
-						Domain: "google",
+						Domain: &geodata.DomainRule{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Substr, Value: "google"}}},
 						Ip:     [][]byte{{8, 8, 8, 8}, {8, 8, 4, 4}},
 					},
 					{
-						Type:   dns.DomainMatchingType_Regex,
-						Domain: ".*\\.com",
+						Domain: &geodata.DomainRule{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Regex, Value: ".*\\.com"}}},
 						Ip:     [][]byte{{8, 8, 4, 4}},
 					},
 					{
-						Type:   dns.DomainMatchingType_Full,
-						Domain: "www.example.org",
+						Domain: &geodata.DomainRule{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Full, Value: "www.example.org"}}},
 						Ip:     [][]byte{{127, 0, 0, 1}, {127, 0, 0, 2}},
 					},
 				},
@@ -113,5 +104,21 @@ func TestDNSConfigParsing(t *testing.T) {
 				DisableFallback: true,
 			},
 		},
-	})
+	}
+
+	for _, testCase := range testCases {
+		actual, err := testCase.Parser(testCase.Input)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff(
+			testCase.Output,
+			actual,
+			protocmp.Transform(),
+			protocmp.SortRepeatedFields(&dns.Config{}, "static_hosts"),
+		); diff != "" {
+			t.Fatalf("Failed in test case:\n%s\nDiff (-want +got):\n%s", testCase.Input, diff)
+		}
+	}
 }

infra/conf/router.go

@@ -1,20 +1,14 @@
 package conf
 
 import (
-	"bufio"
-	"bytes"
 	"encoding/json"
-	"io"
-	"runtime"
-	"strconv"
 	"strings"
 
 	"github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/common/errors"
-	"github.com/xtls/xray-core/common/net"
-	"github.com/xtls/xray-core/common/platform"
-	"github.com/xtls/xray-core/common/platform/filesystem"
+	"github.com/xtls/xray-core/common/geodata"
 	"github.com/xtls/xray-core/common/serial"
+
 	"google.golang.org/protobuf/proto"
 )
 
@@ -104,15 +98,14 @@ func (c *RouterConfig) Build() (*router.Config, error) {
 	if c != nil {
 		rawRuleList = c.RuleList
 	}
-
 	for _, rawRule := range rawRuleList {
 		rule, err := parseRule(rawRule)
 		if err != nil {
 			return nil, err
 		}
-
 		config.Rule = append(config.Rule, rule)
 	}
+
 	for _, rawBalancer := range c.Balancers {
 		balancer, err := rawBalancer.Build()
 		if err != nil {
@@ -120,6 +113,7 @@ func (c *RouterConfig) Build() (*router.Config, error) {
 		}
 		config.BalancingRule = append(config.BalancingRule, balancer)
 	}
+
 	return config, nil
 }
 
@@ -129,399 +123,6 @@ type RouterRule struct {
 	BalancerTag string `json:"balancerTag"`
 }
 
-func parseIP(s string) (*router.CIDR, error) {
-	var addr, mask string
-	i := strings.Index(s, "/")
-	if i < 0 {
-		addr = s
-	} else {
-		addr = s[:i]
-		mask = s[i+1:]
-	}
-	ip := net.ParseAddress(addr)
-	switch ip.Family() {
-	case net.AddressFamilyIPv4:
-		bits := uint32(32)
-		if len(mask) > 0 {
-			bits64, err := strconv.ParseUint(mask, 10, 32)
-			if err != nil {
-				return nil, errors.New("invalid network mask for router: ", mask).Base(err)
-			}
-			bits = uint32(bits64)
-		}
-		if bits > 32 {
-			return nil, errors.New("invalid network mask for router: ", bits)
-		}
-		return &router.CIDR{
-			Ip:     []byte(ip.IP()),
-			Prefix: bits,
-		}, nil
-	case net.AddressFamilyIPv6:
-		bits := uint32(128)
-		if len(mask) > 0 {
-			bits64, err := strconv.ParseUint(mask, 10, 32)
-			if err != nil {
-				return nil, errors.New("invalid network mask for router: ", mask).Base(err)
-			}
-			bits = uint32(bits64)
-		}
-		if bits > 128 {
-			return nil, errors.New("invalid network mask for router: ", bits)
-		}
-		return &router.CIDR{
-			Ip:     []byte(ip.IP()),
-			Prefix: bits,
-		}, nil
-	default:
-		return nil, errors.New("unsupported address for router: ", s)
-	}
-}
-
-func loadFile(file, code string) ([]byte, error) {
-	runtime.GC()
-	r, err := filesystem.OpenAsset(file)
-	defer r.Close()
-	if err != nil {
-		return nil, errors.New("failed to open file: ", file).Base(err)
-	}
-	bs := find(r, []byte(code))
-	if bs == nil {
-		return nil, errors.New("code not found in ", file, ": ", code)
-	}
-	return bs, nil
-}
-
-func loadIP(file, code string) ([]*router.CIDR, error) {
-	bs, err := loadFile(file, code)
-	if err != nil {
-		return nil, err
-	}
-	var geoip router.GeoIP
-	if err := proto.Unmarshal(bs, &geoip); err != nil {
-		return nil, errors.New("error unmarshal IP in ", file, ": ", code).Base(err)
-	}
-	defer runtime.GC() // or debug.FreeOSMemory()
-	return geoip.Cidr, nil
-}
-
-func loadSite(file, code string) ([]*router.Domain, error) {
-
-	// Check if domain matcher cache is provided via environment
-	domainMatcherPath := platform.NewEnvFlag(platform.MphCachePath).GetValue(func() string { return "" })
-	if domainMatcherPath != "" {
-		return []*router.Domain{{}}, nil
-	}
-
-	bs, err := loadFile(file, code)
-	if err != nil {
-		return nil, err
-	}
-	var geosite router.GeoSite
-	if err := proto.Unmarshal(bs, &geosite); err != nil {
-		return nil, errors.New("error unmarshal Site in ", file, ": ", code).Base(err)
-	}
-	defer runtime.GC() // or debug.FreeOSMemory()
-	return geosite.Domain, nil
-}
-
-func decodeVarint(r *bufio.Reader) (uint64, error) {
-	var x uint64
-	for shift := uint(0); shift < 64; shift += 7 {
-		b, err := r.ReadByte()
-		if err != nil {
-			return 0, err
-		}
-		x |= (uint64(b) & 0x7F) << shift
-		if (b & 0x80) == 0 {
-			return x, nil
-		}
-	}
-	// The number is too large to represent in a 64-bit value.
-	return 0, errors.New("varint overflow")
-}
-
-func find(r io.Reader, code []byte) []byte {
-	codeL := len(code)
-	if codeL == 0 {
-		return nil
-	}
-
-	br := bufio.NewReaderSize(r, 64*1024)
-	need := 2 + codeL
-	prefixBuf := make([]byte, need)
-
-	for {
-		if _, err := br.ReadByte(); err != nil {
-			return nil
-		}
-
-		x, err := decodeVarint(br)
-		if err != nil {
-			return nil
-		}
-		bodyL := int(x)
-		if bodyL <= 0 {
-			return nil
-		}
-
-		prefixL := bodyL
-		if prefixL > need {
-			prefixL = need
-		}
-		prefix := prefixBuf[:prefixL]
-		if _, err := io.ReadFull(br, prefix); err != nil {
-			return nil
-		}
-
-		match := false
-		if bodyL >= need {
-			if int(prefix[1]) == codeL && bytes.Equal(prefix[2:need], code) {
-				match = true
-			}
-		}
-
-		remain := bodyL - prefixL
-		if match {
-			out := make([]byte, bodyL)
-			copy(out, prefix)
-			if remain > 0 {
-				if _, err := io.ReadFull(br, out[prefixL:]); err != nil {
-					return nil
-				}
-			}
-			return out
-		}
-
-		if remain > 0 {
-			if _, err := br.Discard(remain); err != nil {
-				return nil
-			}
-		}
-	}
-}
-
-type AttributeMatcher interface {
-	Match(*router.Domain) bool
-}
-
-type BooleanMatcher string
-
-func (m BooleanMatcher) Match(domain *router.Domain) bool {
-	for _, attr := range domain.Attribute {
-		if attr.Key == string(m) {
-			return true
-		}
-	}
-	return false
-}
-
-type AttributeList struct {
-	matcher []AttributeMatcher
-}
-
-func (al *AttributeList) Match(domain *router.Domain) bool {
-	for _, matcher := range al.matcher {
-		if !matcher.Match(domain) {
-			return false
-		}
-	}
-	return true
-}
-
-func (al *AttributeList) IsEmpty() bool {
-	return len(al.matcher) == 0
-}
-
-func parseAttrs(attrs []string) *AttributeList {
-	al := new(AttributeList)
-	for _, attr := range attrs {
-		lc := strings.ToLower(attr)
-		al.matcher = append(al.matcher, BooleanMatcher(lc))
-	}
-	return al
-}
-
-func loadGeositeWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) {
-	parts := strings.Split(siteWithAttr, "@")
-	if len(parts) == 0 {
-		return nil, errors.New("empty site")
-	}
-	country := strings.ToUpper(parts[0])
-	attrs := parseAttrs(parts[1:])
-	domains, err := loadSite(file, country)
-	if err != nil {
-		return nil, err
-	}
-
-	if attrs.IsEmpty() {
-		return domains, nil
-	}
-
-	filteredDomains := make([]*router.Domain, 0, len(domains))
-	for _, domain := range domains {
-		if attrs.Match(domain) {
-			filteredDomains = append(filteredDomains, domain)
-		}
-	}
-
-	return filteredDomains, nil
-}
-
-func parseDomainRule(domain string) ([]*router.Domain, error) {
-	if strings.HasPrefix(domain, "geosite:") {
-		country := strings.ToUpper(domain[8:])
-		domains, err := loadGeositeWithAttr("geosite.dat", country)
-		if err != nil {
-			return nil, errors.New("failed to load geosite: ", country).Base(err)
-		}
-		return domains, nil
-	}
-	isExtDatFile := 0
-	{
-		const prefix = "ext:"
-		if strings.HasPrefix(domain, prefix) {
-			isExtDatFile = len(prefix)
-		}
-		const prefixQualified = "ext-domain:"
-		if strings.HasPrefix(domain, prefixQualified) {
-			isExtDatFile = len(prefixQualified)
-		}
-	}
-	if isExtDatFile != 0 {
-		kv := strings.Split(domain[isExtDatFile:], ":")
-		if len(kv) != 2 {
-			return nil, errors.New("invalid external resource: ", domain)
-		}
-		filename := kv[0]
-		country := kv[1]
-		domains, err := loadGeositeWithAttr(filename, country)
-		if err != nil {
-			return nil, errors.New("failed to load external sites: ", country, " from ", filename).Base(err)
-		}
-		return domains, nil
-	}
-
-	domainRule := new(router.Domain)
-	switch {
-	case strings.HasPrefix(domain, "regexp:"):
-		domainRule.Type = router.Domain_Regex
-		domainRule.Value = domain[7:]
-
-	case strings.HasPrefix(domain, "domain:"):
-		domainRule.Type = router.Domain_Domain
-		domainRule.Value = domain[7:]
-
-	case strings.HasPrefix(domain, "full:"):
-		domainRule.Type = router.Domain_Full
-		domainRule.Value = domain[5:]
-
-	case strings.HasPrefix(domain, "keyword:"):
-		domainRule.Type = router.Domain_Plain
-		domainRule.Value = domain[8:]
-
-	case strings.HasPrefix(domain, "dotless:"):
-		domainRule.Type = router.Domain_Regex
-		switch substr := domain[8:]; {
-		case substr == "":
-			domainRule.Value = "^[^.]*$"
-		case !strings.Contains(substr, "."):
-			domainRule.Value = "^[^.]*" + substr + "[^.]*$"
-		default:
-			return nil, errors.New("substr in dotless rule should not contain a dot: ", substr)
-		}
-
-	default:
-		domainRule.Type = router.Domain_Plain
-		domainRule.Value = domain
-	}
-	return []*router.Domain{domainRule}, nil
-}
-
-func ToCidrList(ips StringList) ([]*router.GeoIP, error) {
-	var geoipList []*router.GeoIP
-	var customCidrs []*router.CIDR
-
-	for _, ip := range ips {
-		if strings.HasPrefix(ip, "geoip:") {
-			country := ip[6:]
-			isReverseMatch := false
-			if strings.HasPrefix(ip, "geoip:!") {
-				country = ip[7:]
-				isReverseMatch = true
-			}
-			if len(country) == 0 {
-				return nil, errors.New("empty country name in rule")
-			}
-			geoip, err := loadIP("geoip.dat", strings.ToUpper(country))
-			if err != nil {
-				return nil, errors.New("failed to load GeoIP: ", country).Base(err)
-			}
-
-			geoipList = append(geoipList, &router.GeoIP{
-				CountryCode:  strings.ToUpper(country),
-				Cidr:         geoip,
-				ReverseMatch: isReverseMatch,
-			})
-			continue
-		}
-		isExtDatFile := 0
-		{
-			const prefix = "ext:"
-			if strings.HasPrefix(ip, prefix) {
-				isExtDatFile = len(prefix)
-			}
-			const prefixQualified = "ext-ip:"
-			if strings.HasPrefix(ip, prefixQualified) {
-				isExtDatFile = len(prefixQualified)
-			}
-		}
-		if isExtDatFile != 0 {
-			kv := strings.Split(ip[isExtDatFile:], ":")
-			if len(kv) != 2 {
-				return nil, errors.New("invalid external resource: ", ip)
-			}
-
-			filename := kv[0]
-			country := kv[1]
-			if len(filename) == 0 || len(country) == 0 {
-				return nil, errors.New("empty filename or empty country in rule")
-			}
-
-			isReverseMatch := false
-			if strings.HasPrefix(country, "!") {
-				country = country[1:]
-				isReverseMatch = true
-			}
-			geoip, err := loadIP(filename, strings.ToUpper(country))
-			if err != nil {
-				return nil, errors.New("failed to load IPs: ", country, " from ", filename).Base(err)
-			}
-
-			geoipList = append(geoipList, &router.GeoIP{
-				CountryCode:  strings.ToUpper(filename + "_" + country),
-				Cidr:         geoip,
-				ReverseMatch: isReverseMatch,
-			})
-
-			continue
-		}
-
-		ipRule, err := parseIP(ip)
-		if err != nil {
-			return nil, errors.New("invalid IP: ", ip).Base(err)
-		}
-		customCidrs = append(customCidrs, ipRule)
-	}
-
-	if len(customCidrs) > 0 {
-		geoipList = append(geoipList, &router.GeoIP{
-			Cidr: customCidrs,
-		})
-	}
-
-	return geoipList, nil
-}
-
 type WebhookRuleConfig struct {
 	URL           string            `json:"url"`
 	Deduplication uint32            `json:"deduplication"`
@@ -571,31 +172,27 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
 	}
 
 	if rawFieldRule.Domain != nil {
-		for _, domain := range *rawFieldRule.Domain {
-			rules, err := parseDomainRule(domain)
-			if err != nil {
-				return nil, errors.New("failed to parse domain rule: ", domain).Base(err)
-			}
-			rule.Domain = append(rule.Domain, rules...)
-		}
-	}
-
-	if rawFieldRule.Domains != nil {
-		for _, domain := range *rawFieldRule.Domains {
-			rules, err := parseDomainRule(domain)
-			if err != nil {
-				return nil, errors.New("failed to parse domain rule: ", domain).Base(err)
-			}
-			rule.Domain = append(rule.Domain, rules...)
-		}
-	}
-
-	if rawFieldRule.IP != nil {
-		geoipList, err := ToCidrList(*rawFieldRule.IP)
+		rules, err := geodata.ParseDomainRules(*rawFieldRule.Domain, geodata.Domain_Substr)
 		if err != nil {
 			return nil, err
 		}
-		rule.Geoip = geoipList
+		rule.Domain = rules
+	}
+
+	if rawFieldRule.Domains != nil {
+		rules, err := geodata.ParseDomainRules(*rawFieldRule.Domains, geodata.Domain_Substr)
+		if err != nil {
+			return nil, err
+		}
+		rule.Domain = rules
+	}
+
+	if rawFieldRule.IP != nil {
+		rules, err := geodata.ParseIPRules(*rawFieldRule.IP)
+		if err != nil {
+			return nil, err
+		}
+		rule.Ip = rules
 	}
 
 	if rawFieldRule.Port != nil {
@@ -611,11 +208,11 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
 	}
 
 	if rawFieldRule.SourceIP != nil {
-		geoipList, err := ToCidrList(*rawFieldRule.SourceIP)
+		rules, err := geodata.ParseIPRules(*rawFieldRule.SourceIP)
 		if err != nil {
 			return nil, err
 		}
-		rule.SourceGeoip = geoipList
+		rule.SourceIp = rules
 	}
 
 	if rawFieldRule.SourcePort != nil {
@@ -623,11 +220,11 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
 	}
 
 	if rawFieldRule.LocalIP != nil {
-		geoipList, err := ToCidrList(*rawFieldRule.LocalIP)
+		rules, err := geodata.ParseIPRules(*rawFieldRule.LocalIP)
 		if err != nil {
 			return nil, err
 		}
-		rule.LocalGeoip = geoipList
+		rule.LocalIp = rules
 	}
 
 	if rawFieldRule.LocalPort != nil {

infra/conf/router_test.go

@@ -2,78 +2,19 @@ package conf_test
 
 import (
 	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 	_ "unsafe"
 
 	"github.com/xtls/xray-core/app/router"
-	"github.com/xtls/xray-core/common"
+	"github.com/xtls/xray-core/common/geodata"
 	"github.com/xtls/xray-core/common/net"
-	"github.com/xtls/xray-core/common/platform"
-	"github.com/xtls/xray-core/common/platform/filesystem"
 	"github.com/xtls/xray-core/common/serial"
 	. "github.com/xtls/xray-core/infra/conf"
+
 	"google.golang.org/protobuf/proto"
 )
 
-func getAssetPath(file string) (string, error) {
-	path := platform.GetAssetLocation(file)
-	_, err := os.Stat(path)
-	if os.IsNotExist(err) {
-		path := filepath.Join("..", "..", "resources", file)
-		_, err := os.Stat(path)
-		if os.IsNotExist(err) {
-			return "", fmt.Errorf("can't find %s in standard asset locations or {project_root}/resources", file)
-		}
-		if err != nil {
-			return "", fmt.Errorf("can't stat %s: %v", path, err)
-		}
-		return path, nil
-	}
-	if err != nil {
-		return "", fmt.Errorf("can't stat %s: %v", path, err)
-	}
-
-	return path, nil
-}
-
-func TestToCidrList(t *testing.T) {
-	tempDir, err := os.MkdirTemp("", "test-")
-	if err != nil {
-		t.Fatalf("can't create temp dir: %v", err)
-	}
-	defer os.RemoveAll(tempDir)
-
-	geoipPath, err := getAssetPath("geoip.dat")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	common.Must(filesystem.CopyFile(filepath.Join(tempDir, "geoip.dat"), geoipPath))
-	common.Must(filesystem.CopyFile(filepath.Join(tempDir, "geoiptestrouter.dat"), geoipPath))
-
-	os.Setenv("xray.location.asset", tempDir)
-	defer os.Unsetenv("xray.location.asset")
-
-	ips := StringList([]string{
-		"geoip:us",
-		"geoip:cn",
-		"geoip:!cn",
-		"ext:geoiptestrouter.dat:!cn",
-		"ext:geoiptestrouter.dat:ca",
-		"ext-ip:geoiptestrouter.dat:!cn",
-		"ext-ip:geoiptestrouter.dat:!ca",
-	})
-
-	_, err = ToCidrList(ips)
-	if err != nil {
-		t.Fatalf("Failed to parse geoip list, got %s", err)
-	}
-}
-
 func TestRouterConfig(t *testing.T) {
 	createParser := func() func(string) (proto.Message, error) {
 		return func(s string) (proto.Message, error) {
@@ -182,29 +123,27 @@ func TestRouterConfig(t *testing.T) {
 				},
 				Rule: []*router.RoutingRule{
 					{
-						Domain: []*router.Domain{
-							{
-								Type:  router.Domain_Plain,
-								Value: "baidu.com",
-							},
-							{
-								Type:  router.Domain_Plain,
-								Value: "qq.com",
-							},
+						Domain: []*geodata.DomainRule{
+							{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Substr, Value: "baidu.com"}}},
+							{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Substr, Value: "qq.com"}}},
 						},
 						TargetTag: &router.RoutingRule_Tag{
 							Tag: "direct",
 						},
 					},
 					{
-						Geoip: []*router.GeoIP{
+						Ip: []*geodata.IPRule{
 							{
-								Cidr: []*router.CIDR{
-									{
+								Value: &geodata.IPRule_Custom{
+									Custom: &geodata.CIDR{
 										Ip:     []byte{10, 0, 0, 0},
 										Prefix: 8,
 									},
-									{
+								},
+							},
+							{
+								Value: &geodata.IPRule_Custom{
+									Custom: &geodata.CIDR{
 										Ip:     []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
 										Prefix: 128,
 									},
@@ -265,29 +204,27 @@ func TestRouterConfig(t *testing.T) {
 				DomainStrategy: router.Config_IpIfNonMatch,
 				Rule: []*router.RoutingRule{
 					{
-						Domain: []*router.Domain{
-							{
-								Type:  router.Domain_Plain,
-								Value: "baidu.com",
-							},
-							{
-								Type:  router.Domain_Plain,
-								Value: "qq.com",
-							},
+						Domain: []*geodata.DomainRule{
+							{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Substr, Value: "baidu.com"}}},
+							{Value: &geodata.DomainRule_Custom{Custom: &geodata.Domain{Type: geodata.Domain_Substr, Value: "qq.com"}}},
 						},
 						TargetTag: &router.RoutingRule_Tag{
 							Tag: "direct",
 						},
 					},
 					{
-						Geoip: []*router.GeoIP{
+						Ip: []*geodata.IPRule{
 							{
-								Cidr: []*router.CIDR{
-									{
+								Value: &geodata.IPRule_Custom{
+									Custom: &geodata.CIDR{
 										Ip:     []byte{10, 0, 0, 0},
 										Prefix: 8,
 									},
-									{
+								},
+							},
+							{
+								Value: &geodata.IPRule_Custom{
+									Custom: &geodata.CIDR{
 										Ip:     []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
 										Prefix: 128,
 									},

infra/conf/xray.go

@@ -1,21 +1,16 @@
 package conf
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
-	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 
 	"github.com/xtls/xray-core/app/dispatcher"
 	"github.com/xtls/xray-core/app/proxyman"
-	"github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/app/stats"
 	"github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/common/net"
-	"github.com/xtls/xray-core/common/platform"
 	"github.com/xtls/xray-core/common/serial"
 	core "github.com/xtls/xray-core/core"
 	"github.com/xtls/xray-core/transport/internet"
@@ -617,187 +612,6 @@ func (c *Config) Build() (*core.Config, error) {
 	return config, nil
 }
 
-func (c *Config) BuildMPHCache(customMatcherFilePath *string) error {
-	var geosite []*router.GeoSite
-	deps := make(map[string][]string)
-	uniqueGeosites := make(map[string]bool)
-	uniqueTags := make(map[string]bool)
-	matcherFilePath := platform.GetAssetLocation("matcher.cache")
-
-	if customMatcherFilePath != nil {
-		matcherFilePath = *customMatcherFilePath
-	}
-
-	processGeosite := func(dStr string) bool {
-		prefix := ""
-		if strings.HasPrefix(dStr, "geosite:") {
-			prefix = "geosite:"
-		} else if strings.HasPrefix(dStr, "ext-domain:") {
-			prefix = "ext-domain:"
-		}
-		if prefix == "" {
-			return false
-		}
-		key := strings.ToLower(dStr)
-		country := strings.ToUpper(dStr[len(prefix):])
-		if !uniqueGeosites[country] {
-			ds, err := loadGeositeWithAttr("geosite.dat", country)
-			if err == nil {
-				uniqueGeosites[country] = true
-				geosite = append(geosite, &router.GeoSite{CountryCode: key, Domain: ds})
-			}
-		}
-		return true
-	}
-
-	processDomains := func(tag string, rawDomains []string) {
-		var manualDomains []*router.Domain
-		var dDeps []string
-		for _, dStr := range rawDomains {
-			if processGeosite(dStr) {
-				dDeps = append(dDeps, strings.ToLower(dStr))
-			} else {
-				ds, err := parseDomainRule(dStr)
-				if err == nil {
-					manualDomains = append(manualDomains, ds...)
-				}
-			}
-		}
-		if len(manualDomains) > 0 {
-			if !uniqueTags[tag] {
-				uniqueTags[tag] = true
-				geosite = append(geosite, &router.GeoSite{CountryCode: tag, Domain: manualDomains})
-			}
-		}
-		if len(dDeps) > 0 {
-			deps[tag] = append(deps[tag], dDeps...)
-		}
-	}
-
-	// proccess rules
-	if c.RouterConfig != nil {
-		for _, rawRule := range c.RouterConfig.RuleList {
-			type SimpleRule struct {
-				RuleTag string      `json:"ruleTag"`
-				Domain  *StringList `json:"domain"`
-				Domains *StringList `json:"domains"`
-			}
-			var sr SimpleRule
-			json.Unmarshal(rawRule, &sr)
-			if sr.RuleTag == "" {
-				continue
-			}
-			var allDomains []string
-			if sr.Domain != nil {
-				allDomains = append(allDomains, *sr.Domain...)
-			}
-			if sr.Domains != nil {
-				allDomains = append(allDomains, *sr.Domains...)
-			}
-			processDomains(sr.RuleTag, allDomains)
-		}
-	}
-
-	// proccess dns servers
-	if c.DNSConfig != nil {
-		for _, ns := range c.DNSConfig.Servers {
-			if ns.Tag == "" {
-				continue
-			}
-			processDomains(ns.Tag, ns.Domains)
-		}
-	}
-
-	var hostIPs map[string][]string
-	if c.DNSConfig != nil && c.DNSConfig.Hosts != nil {
-		hostIPs = make(map[string][]string)
-		var hostDeps []string
-		var hostPatterns []string
-
-		// use raw map to avoid expanding geosites
-		var domains []string
-		for domain := range c.DNSConfig.Hosts.Hosts {
-			domains = append(domains, domain)
-		}
-		sort.Strings(domains)
-
-		manualHostGroups := make(map[string][]*router.Domain)
-		manualHostIPs := make(map[string][]string)
-		manualHostNames := make(map[string]string)
-
-		for _, domain := range domains {
-			ha := c.DNSConfig.Hosts.Hosts[domain]
-			m := getHostMapping(ha)
-
-			var ips []string
-			if m.ProxiedDomain != "" {
-				ips = append(ips, m.ProxiedDomain)
-			} else {
-				for _, ip := range m.Ip {
-					ips = append(ips, net.IPAddress(ip).String())
-				}
-			}
-
-			if processGeosite(domain) {
-				tag := strings.ToLower(domain)
-				hostDeps = append(hostDeps, tag)
-				hostIPs[tag] = ips
-				hostPatterns = append(hostPatterns, domain)
-			} else {
-				// build manual domains by their destination IPs
-				sort.Strings(ips)
-				ipKey := strings.Join(ips, ",")
-				ds, err := parseDomainRule(domain)
-				if err == nil {
-					manualHostGroups[ipKey] = append(manualHostGroups[ipKey], ds...)
-					manualHostIPs[ipKey] = ips
-					if _, ok := manualHostNames[ipKey]; !ok {
-						manualHostNames[ipKey] = domain
-					}
-				}
-			}
-		}
-
-		// create manual host groups
-		var ipKeys []string
-		for k := range manualHostGroups {
-			ipKeys = append(ipKeys, k)
-		}
-		sort.Strings(ipKeys)
-
-		for _, k := range ipKeys {
-			tag := manualHostNames[k]
-			geosite = append(geosite, &router.GeoSite{CountryCode: tag, Domain: manualHostGroups[k]})
-			hostDeps = append(hostDeps, tag)
-			hostIPs[tag] = manualHostIPs[k]
-
-			// record tag _ORDER links the matcher to IP addresses
-			hostPatterns = append(hostPatterns, tag)
-		}
-
-		deps["HOSTS"] = hostDeps
-		hostIPs["_ORDER"] = hostPatterns
-	}
-
-	f, err := os.Create(matcherFilePath)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	var buf bytes.Buffer
-
-	if err := router.SerializeGeoSiteList(geosite, deps, hostIPs, &buf); err != nil {
-		return err
-	}
-
-	if _, err := f.Write(buf.Bytes()); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // Convert string to Address.
 func ParseSendThough(Addr *string) *Address {
 	var addr Address

infra/conf/xray_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/xtls/xray-core/app/proxyman"
 	"github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/common"
+	"github.com/xtls/xray-core/common/geodata"
 	clog "github.com/xtls/xray-core/common/log"
 	"github.com/xtls/xray-core/common/net"
 	"github.com/xtls/xray-core/common/protocol"
@@ -95,10 +96,10 @@ func TestXrayConfig(t *testing.T) {
 						DomainStrategy: router.Config_AsIs,
 						Rule: []*router.RoutingRule{
 							{
-								Geoip: []*router.GeoIP{
+								Ip: []*geodata.IPRule{
 									{
-										Cidr: []*router.CIDR{
-											{
+										Value: &geodata.IPRule_Custom{
+											Custom: &geodata.CIDR{
 												Ip:     []byte{10, 0, 0, 0},
 												Prefix: 8,
 											},