mirror of
https://github.com/XTLS/Xray-core.git
synced 2026-07-14 17:08:54 +00:00
Xray-core: Refactor geodata (#5814)
https://github.com/XTLS/Xray-core/issues/4422#issuecomment-3533007890 Breaking changes https://github.com/XTLS/Xray-core/pull/5569 Reverts https://github.com/XTLS/Xray-core/pull/5505 Closes https://github.com/XTLS/Xray-core/pull/643
This commit is contained in:
Meow
+26
-429
@@ -1,20 +1,14 @@
|
||||
package conf
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/xtls/xray-core/app/router"
|
||||
"github.com/xtls/xray-core/common/errors"
|
||||
"github.com/xtls/xray-core/common/net"
|
||||
"github.com/xtls/xray-core/common/platform"
|
||||
"github.com/xtls/xray-core/common/platform/filesystem"
|
||||
"github.com/xtls/xray-core/common/geodata"
|
||||
"github.com/xtls/xray-core/common/serial"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
@@ -104,15 +98,14 @@ func (c *RouterConfig) Build() (*router.Config, error) {
|
||||
if c != nil {
|
||||
rawRuleList = c.RuleList
|
||||
}
|
||||
|
||||
for _, rawRule := range rawRuleList {
|
||||
rule, err := parseRule(rawRule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.Rule = append(config.Rule, rule)
|
||||
}
|
||||
|
||||
for _, rawBalancer := range c.Balancers {
|
||||
balancer, err := rawBalancer.Build()
|
||||
if err != nil {
|
||||
@@ -120,6 +113,7 @@ func (c *RouterConfig) Build() (*router.Config, error) {
|
||||
}
|
||||
config.BalancingRule = append(config.BalancingRule, balancer)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
@@ -129,399 +123,6 @@ type RouterRule struct {
|
||||
BalancerTag string `json:"balancerTag"`
|
||||
}
|
||||
|
||||
func parseIP(s string) (*router.CIDR, error) {
|
||||
var addr, mask string
|
||||
i := strings.Index(s, "/")
|
||||
if i < 0 {
|
||||
addr = s
|
||||
} else {
|
||||
addr = s[:i]
|
||||
mask = s[i+1:]
|
||||
}
|
||||
ip := net.ParseAddress(addr)
|
||||
switch ip.Family() {
|
||||
case net.AddressFamilyIPv4:
|
||||
bits := uint32(32)
|
||||
if len(mask) > 0 {
|
||||
bits64, err := strconv.ParseUint(mask, 10, 32)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid network mask for router: ", mask).Base(err)
|
||||
}
|
||||
bits = uint32(bits64)
|
||||
}
|
||||
if bits > 32 {
|
||||
return nil, errors.New("invalid network mask for router: ", bits)
|
||||
}
|
||||
return &router.CIDR{
|
||||
Ip: []byte(ip.IP()),
|
||||
Prefix: bits,
|
||||
}, nil
|
||||
case net.AddressFamilyIPv6:
|
||||
bits := uint32(128)
|
||||
if len(mask) > 0 {
|
||||
bits64, err := strconv.ParseUint(mask, 10, 32)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid network mask for router: ", mask).Base(err)
|
||||
}
|
||||
bits = uint32(bits64)
|
||||
}
|
||||
if bits > 128 {
|
||||
return nil, errors.New("invalid network mask for router: ", bits)
|
||||
}
|
||||
return &router.CIDR{
|
||||
Ip: []byte(ip.IP()),
|
||||
Prefix: bits,
|
||||
}, nil
|
||||
default:
|
||||
return nil, errors.New("unsupported address for router: ", s)
|
||||
}
|
||||
}
|
||||
|
||||
func loadFile(file, code string) ([]byte, error) {
|
||||
runtime.GC()
|
||||
r, err := filesystem.OpenAsset(file)
|
||||
defer r.Close()
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to open file: ", file).Base(err)
|
||||
}
|
||||
bs := find(r, []byte(code))
|
||||
if bs == nil {
|
||||
return nil, errors.New("code not found in ", file, ": ", code)
|
||||
}
|
||||
return bs, nil
|
||||
}
|
||||
|
||||
func loadIP(file, code string) ([]*router.CIDR, error) {
|
||||
bs, err := loadFile(file, code)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var geoip router.GeoIP
|
||||
if err := proto.Unmarshal(bs, &geoip); err != nil {
|
||||
return nil, errors.New("error unmarshal IP in ", file, ": ", code).Base(err)
|
||||
}
|
||||
defer runtime.GC() // or debug.FreeOSMemory()
|
||||
return geoip.Cidr, nil
|
||||
}
|
||||
|
||||
func loadSite(file, code string) ([]*router.Domain, error) {
|
||||
|
||||
// Check if domain matcher cache is provided via environment
|
||||
domainMatcherPath := platform.NewEnvFlag(platform.MphCachePath).GetValue(func() string { return "" })
|
||||
if domainMatcherPath != "" {
|
||||
return []*router.Domain{{}}, nil
|
||||
}
|
||||
|
||||
bs, err := loadFile(file, code)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var geosite router.GeoSite
|
||||
if err := proto.Unmarshal(bs, &geosite); err != nil {
|
||||
return nil, errors.New("error unmarshal Site in ", file, ": ", code).Base(err)
|
||||
}
|
||||
defer runtime.GC() // or debug.FreeOSMemory()
|
||||
return geosite.Domain, nil
|
||||
}
|
||||
|
||||
func decodeVarint(r *bufio.Reader) (uint64, error) {
|
||||
var x uint64
|
||||
for shift := uint(0); shift < 64; shift += 7 {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
x |= (uint64(b) & 0x7F) << shift
|
||||
if (b & 0x80) == 0 {
|
||||
return x, nil
|
||||
}
|
||||
}
|
||||
// The number is too large to represent in a 64-bit value.
|
||||
return 0, errors.New("varint overflow")
|
||||
}
|
||||
|
||||
func find(r io.Reader, code []byte) []byte {
|
||||
codeL := len(code)
|
||||
if codeL == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
br := bufio.NewReaderSize(r, 64*1024)
|
||||
need := 2 + codeL
|
||||
prefixBuf := make([]byte, need)
|
||||
|
||||
for {
|
||||
if _, err := br.ReadByte(); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
x, err := decodeVarint(br)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
bodyL := int(x)
|
||||
if bodyL <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
prefixL := bodyL
|
||||
if prefixL > need {
|
||||
prefixL = need
|
||||
}
|
||||
prefix := prefixBuf[:prefixL]
|
||||
if _, err := io.ReadFull(br, prefix); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
match := false
|
||||
if bodyL >= need {
|
||||
if int(prefix[1]) == codeL && bytes.Equal(prefix[2:need], code) {
|
||||
match = true
|
||||
}
|
||||
}
|
||||
|
||||
remain := bodyL - prefixL
|
||||
if match {
|
||||
out := make([]byte, bodyL)
|
||||
copy(out, prefix)
|
||||
if remain > 0 {
|
||||
if _, err := io.ReadFull(br, out[prefixL:]); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
if remain > 0 {
|
||||
if _, err := br.Discard(remain); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type AttributeMatcher interface {
|
||||
Match(*router.Domain) bool
|
||||
}
|
||||
|
||||
type BooleanMatcher string
|
||||
|
||||
func (m BooleanMatcher) Match(domain *router.Domain) bool {
|
||||
for _, attr := range domain.Attribute {
|
||||
if attr.Key == string(m) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type AttributeList struct {
|
||||
matcher []AttributeMatcher
|
||||
}
|
||||
|
||||
func (al *AttributeList) Match(domain *router.Domain) bool {
|
||||
for _, matcher := range al.matcher {
|
||||
if !matcher.Match(domain) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (al *AttributeList) IsEmpty() bool {
|
||||
return len(al.matcher) == 0
|
||||
}
|
||||
|
||||
func parseAttrs(attrs []string) *AttributeList {
|
||||
al := new(AttributeList)
|
||||
for _, attr := range attrs {
|
||||
lc := strings.ToLower(attr)
|
||||
al.matcher = append(al.matcher, BooleanMatcher(lc))
|
||||
}
|
||||
return al
|
||||
}
|
||||
|
||||
func loadGeositeWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) {
|
||||
parts := strings.Split(siteWithAttr, "@")
|
||||
if len(parts) == 0 {
|
||||
return nil, errors.New("empty site")
|
||||
}
|
||||
country := strings.ToUpper(parts[0])
|
||||
attrs := parseAttrs(parts[1:])
|
||||
domains, err := loadSite(file, country)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if attrs.IsEmpty() {
|
||||
return domains, nil
|
||||
}
|
||||
|
||||
filteredDomains := make([]*router.Domain, 0, len(domains))
|
||||
for _, domain := range domains {
|
||||
if attrs.Match(domain) {
|
||||
filteredDomains = append(filteredDomains, domain)
|
||||
}
|
||||
}
|
||||
|
||||
return filteredDomains, nil
|
||||
}
|
||||
|
||||
func parseDomainRule(domain string) ([]*router.Domain, error) {
|
||||
if strings.HasPrefix(domain, "geosite:") {
|
||||
country := strings.ToUpper(domain[8:])
|
||||
domains, err := loadGeositeWithAttr("geosite.dat", country)
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to load geosite: ", country).Base(err)
|
||||
}
|
||||
return domains, nil
|
||||
}
|
||||
isExtDatFile := 0
|
||||
{
|
||||
const prefix = "ext:"
|
||||
if strings.HasPrefix(domain, prefix) {
|
||||
isExtDatFile = len(prefix)
|
||||
}
|
||||
const prefixQualified = "ext-domain:"
|
||||
if strings.HasPrefix(domain, prefixQualified) {
|
||||
isExtDatFile = len(prefixQualified)
|
||||
}
|
||||
}
|
||||
if isExtDatFile != 0 {
|
||||
kv := strings.Split(domain[isExtDatFile:], ":")
|
||||
if len(kv) != 2 {
|
||||
return nil, errors.New("invalid external resource: ", domain)
|
||||
}
|
||||
filename := kv[0]
|
||||
country := kv[1]
|
||||
domains, err := loadGeositeWithAttr(filename, country)
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to load external sites: ", country, " from ", filename).Base(err)
|
||||
}
|
||||
return domains, nil
|
||||
}
|
||||
|
||||
domainRule := new(router.Domain)
|
||||
switch {
|
||||
case strings.HasPrefix(domain, "regexp:"):
|
||||
domainRule.Type = router.Domain_Regex
|
||||
domainRule.Value = domain[7:]
|
||||
|
||||
case strings.HasPrefix(domain, "domain:"):
|
||||
domainRule.Type = router.Domain_Domain
|
||||
domainRule.Value = domain[7:]
|
||||
|
||||
case strings.HasPrefix(domain, "full:"):
|
||||
domainRule.Type = router.Domain_Full
|
||||
domainRule.Value = domain[5:]
|
||||
|
||||
case strings.HasPrefix(domain, "keyword:"):
|
||||
domainRule.Type = router.Domain_Plain
|
||||
domainRule.Value = domain[8:]
|
||||
|
||||
case strings.HasPrefix(domain, "dotless:"):
|
||||
domainRule.Type = router.Domain_Regex
|
||||
switch substr := domain[8:]; {
|
||||
case substr == "":
|
||||
domainRule.Value = "^[^.]*$"
|
||||
case !strings.Contains(substr, "."):
|
||||
domainRule.Value = "^[^.]*" + substr + "[^.]*$"
|
||||
default:
|
||||
return nil, errors.New("substr in dotless rule should not contain a dot: ", substr)
|
||||
}
|
||||
|
||||
default:
|
||||
domainRule.Type = router.Domain_Plain
|
||||
domainRule.Value = domain
|
||||
}
|
||||
return []*router.Domain{domainRule}, nil
|
||||
}
|
||||
|
||||
func ToCidrList(ips StringList) ([]*router.GeoIP, error) {
|
||||
var geoipList []*router.GeoIP
|
||||
var customCidrs []*router.CIDR
|
||||
|
||||
for _, ip := range ips {
|
||||
if strings.HasPrefix(ip, "geoip:") {
|
||||
country := ip[6:]
|
||||
isReverseMatch := false
|
||||
if strings.HasPrefix(ip, "geoip:!") {
|
||||
country = ip[7:]
|
||||
isReverseMatch = true
|
||||
}
|
||||
if len(country) == 0 {
|
||||
return nil, errors.New("empty country name in rule")
|
||||
}
|
||||
geoip, err := loadIP("geoip.dat", strings.ToUpper(country))
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to load GeoIP: ", country).Base(err)
|
||||
}
|
||||
|
||||
geoipList = append(geoipList, &router.GeoIP{
|
||||
CountryCode: strings.ToUpper(country),
|
||||
Cidr: geoip,
|
||||
ReverseMatch: isReverseMatch,
|
||||
})
|
||||
continue
|
||||
}
|
||||
isExtDatFile := 0
|
||||
{
|
||||
const prefix = "ext:"
|
||||
if strings.HasPrefix(ip, prefix) {
|
||||
isExtDatFile = len(prefix)
|
||||
}
|
||||
const prefixQualified = "ext-ip:"
|
||||
if strings.HasPrefix(ip, prefixQualified) {
|
||||
isExtDatFile = len(prefixQualified)
|
||||
}
|
||||
}
|
||||
if isExtDatFile != 0 {
|
||||
kv := strings.Split(ip[isExtDatFile:], ":")
|
||||
if len(kv) != 2 {
|
||||
return nil, errors.New("invalid external resource: ", ip)
|
||||
}
|
||||
|
||||
filename := kv[0]
|
||||
country := kv[1]
|
||||
if len(filename) == 0 || len(country) == 0 {
|
||||
return nil, errors.New("empty filename or empty country in rule")
|
||||
}
|
||||
|
||||
isReverseMatch := false
|
||||
if strings.HasPrefix(country, "!") {
|
||||
country = country[1:]
|
||||
isReverseMatch = true
|
||||
}
|
||||
geoip, err := loadIP(filename, strings.ToUpper(country))
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to load IPs: ", country, " from ", filename).Base(err)
|
||||
}
|
||||
|
||||
geoipList = append(geoipList, &router.GeoIP{
|
||||
CountryCode: strings.ToUpper(filename + "_" + country),
|
||||
Cidr: geoip,
|
||||
ReverseMatch: isReverseMatch,
|
||||
})
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
ipRule, err := parseIP(ip)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid IP: ", ip).Base(err)
|
||||
}
|
||||
customCidrs = append(customCidrs, ipRule)
|
||||
}
|
||||
|
||||
if len(customCidrs) > 0 {
|
||||
geoipList = append(geoipList, &router.GeoIP{
|
||||
Cidr: customCidrs,
|
||||
})
|
||||
}
|
||||
|
||||
return geoipList, nil
|
||||
}
|
||||
|
||||
type WebhookRuleConfig struct {
|
||||
URL string `json:"url"`
|
||||
Deduplication uint32 `json:"deduplication"`
|
||||
@@ -571,31 +172,27 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
|
||||
}
|
||||
|
||||
if rawFieldRule.Domain != nil {
|
||||
for _, domain := range *rawFieldRule.Domain {
|
||||
rules, err := parseDomainRule(domain)
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to parse domain rule: ", domain).Base(err)
|
||||
}
|
||||
rule.Domain = append(rule.Domain, rules...)
|
||||
}
|
||||
}
|
||||
|
||||
if rawFieldRule.Domains != nil {
|
||||
for _, domain := range *rawFieldRule.Domains {
|
||||
rules, err := parseDomainRule(domain)
|
||||
if err != nil {
|
||||
return nil, errors.New("failed to parse domain rule: ", domain).Base(err)
|
||||
}
|
||||
rule.Domain = append(rule.Domain, rules...)
|
||||
}
|
||||
}
|
||||
|
||||
if rawFieldRule.IP != nil {
|
||||
geoipList, err := ToCidrList(*rawFieldRule.IP)
|
||||
rules, err := geodata.ParseDomainRules(*rawFieldRule.Domain, geodata.Domain_Substr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rule.Geoip = geoipList
|
||||
rule.Domain = rules
|
||||
}
|
||||
|
||||
if rawFieldRule.Domains != nil {
|
||||
rules, err := geodata.ParseDomainRules(*rawFieldRule.Domains, geodata.Domain_Substr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rule.Domain = rules
|
||||
}
|
||||
|
||||
if rawFieldRule.IP != nil {
|
||||
rules, err := geodata.ParseIPRules(*rawFieldRule.IP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rule.Ip = rules
|
||||
}
|
||||
|
||||
if rawFieldRule.Port != nil {
|
||||
@@ -611,11 +208,11 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
|
||||
}
|
||||
|
||||
if rawFieldRule.SourceIP != nil {
|
||||
geoipList, err := ToCidrList(*rawFieldRule.SourceIP)
|
||||
rules, err := geodata.ParseIPRules(*rawFieldRule.SourceIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rule.SourceGeoip = geoipList
|
||||
rule.SourceIp = rules
|
||||
}
|
||||
|
||||
if rawFieldRule.SourcePort != nil {
|
||||
@@ -623,11 +220,11 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
|
||||
}
|
||||
|
||||
if rawFieldRule.LocalIP != nil {
|
||||
geoipList, err := ToCidrList(*rawFieldRule.LocalIP)
|
||||
rules, err := geodata.ParseIPRules(*rawFieldRule.LocalIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rule.LocalGeoip = geoipList
|
||||
rule.LocalIp = rules
|
||||
}
|
||||
|
||||
if rawFieldRule.LocalPort != nil {
|
||||
|
||||
Reference in New Issue
Block a user