Geodata: Reduce memory usage again (#5975)

https://github.com/XTLS/Xray-core/pull/5975#issuecomment-4274779560

common/geodata/strmatcher/anymatcher_linear.go

@@ -0,0 +1,53 @@
+package strmatcher
+
+// LinearAnyMatcher is an implementation of AnyMatcher.
+type LinearAnyMatcher struct {
+	full   *FullMatcherSet
+	domain *DomainMatcherSet
+	substr *SubstrMatcherSet
+	regex  *SimpleMatcherSet
+}
+
+func NewLinearAnyMatcher() *LinearAnyMatcher {
+	return new(LinearAnyMatcher)
+}
+
+// Add implements AnyMatcher.Add.
+func (s *LinearAnyMatcher) Add(matcher Matcher) {
+	switch matcher := matcher.(type) {
+	case FullMatcher:
+		if s.full == nil {
+			s.full = NewFullMatcherSet()
+		}
+		s.full.AddFullMatcher(matcher)
+	case DomainMatcher:
+		if s.domain == nil {
+			s.domain = NewDomainMatcherSet()
+		}
+		s.domain.AddDomainMatcher(matcher)
+	case SubstrMatcher:
+		if s.substr == nil {
+			s.substr = new(SubstrMatcherSet)
+		}
+		s.substr.AddSubstrMatcher(matcher)
+	default:
+		if s.regex == nil {
+			s.regex = new(SimpleMatcherSet)
+		}
+		s.regex.AddMatcher(matcher)
+	}
+}
+
+// MatchAny implements AnyMatcher.MatchAny.
+func (s *LinearAnyMatcher) MatchAny(input string) bool {
+	if s.full != nil && s.full.MatchAny(input) {
+		return true
+	}
+	if s.domain != nil && s.domain.MatchAny(input) {
+		return true
+	}
+	if s.substr != nil && s.substr.MatchAny(input) {
+		return true
+	}
+	return s.regex != nil && s.regex.MatchAny(input)
+}

common/geodata/strmatcher/matchers.go

@@ -288,3 +288,65 @@ func CompositeMatchesReverse(matches [][]uint32) []uint32 {
 		return result
 	}
 }
+
+// MatcherSetForAll is an interface indicating a MatcherSet could accept all types of matchers.
+type MatcherSetForAll interface {
+	AddMatcher(matcher Matcher)
+}
+
+// MatcherSetForFull is an interface indicating a MatcherSet could accept FullMatchers.
+type MatcherSetForFull interface {
+	AddFullMatcher(matcher FullMatcher)
+}
+
+// MatcherSetForDomain is an interface indicating a MatcherSet could accept DomainMatchers.
+type MatcherSetForDomain interface {
+	AddDomainMatcher(matcher DomainMatcher)
+}
+
+// MatcherSetForSubstr is an interface indicating a MatcherSet could accept SubstrMatchers.
+type MatcherSetForSubstr interface {
+	AddSubstrMatcher(matcher SubstrMatcher)
+}
+
+// MatcherSetForRegex is an interface indicating a MatcherSet could accept RegexMatchers.
+type MatcherSetForRegex interface {
+	AddRegexMatcher(matcher *RegexMatcher)
+}
+
+// AddMatcherToSet is a helper function to try to add a Matcher to any kind of MatcherSet.
+// It returns error if the MatcherSet does not accept the provided Matcher's type.
+// This function is provided to help writing code to test a MatcherSet.
+func AddMatcherToSet(s MatcherSet, matcher Matcher) error {
+	if s, ok := s.(IndexMatcher); ok {
+		s.Add(matcher)
+		return nil
+	}
+	if s, ok := s.(MatcherSetForAll); ok {
+		s.AddMatcher(matcher)
+		return nil
+	}
+	switch matcher := matcher.(type) {
+	case FullMatcher:
+		if s, ok := s.(MatcherSetForFull); ok {
+			s.AddFullMatcher(matcher)
+			return nil
+		}
+	case DomainMatcher:
+		if s, ok := s.(MatcherSetForDomain); ok {
+			s.AddDomainMatcher(matcher)
+			return nil
+		}
+	case SubstrMatcher:
+		if s, ok := s.(MatcherSetForSubstr); ok {
+			s.AddSubstrMatcher(matcher)
+			return nil
+		}
+	case *RegexMatcher:
+		if s, ok := s.(MatcherSetForRegex); ok {
+			s.AddRegexMatcher(matcher)
+			return nil
+		}
+	}
+	return errors.New("cannot add matcher to matcher set")
+}

common/geodata/strmatcher/matcherset_domain.go

@@ -0,0 +1,79 @@
+package strmatcher
+
+type trieNode2 struct {
+	matched  bool
+	children map[string]*trieNode2
+}
+
+// DomainMatcherSet is an implementation of MatcherSet.
+// It uses trie to optimize both memory consumption and lookup speed. Trie node is domain label based.
+type DomainMatcherSet struct {
+	root *trieNode2
+}
+
+func NewDomainMatcherSet() *DomainMatcherSet {
+	return &DomainMatcherSet{
+		root: new(trieNode2),
+	}
+}
+
+// AddDomainMatcher implements MatcherSetForDomain.AddDomainMatcher.
+func (s *DomainMatcherSet) AddDomainMatcher(matcher DomainMatcher) {
+	node := s.root
+	pattern := matcher.Pattern()
+	for i := len(pattern); i > 0; {
+		var part string
+		for j := i - 1; ; j-- {
+			if pattern[j] == '.' {
+				part = pattern[j+1 : i]
+				i = j
+				break
+			}
+			if j == 0 {
+				part = pattern[j:i]
+				i = j
+				break
+			}
+		}
+		if node.children == nil {
+			node.children = make(map[string]*trieNode2)
+		}
+		next := node.children[part]
+		if next == nil {
+			next = new(trieNode2)
+			node.children[part] = next
+		}
+		node = next
+	}
+
+	node.matched = true
+}
+
+// MatchAny implements MatcherSet.MatchAny.
+func (s *DomainMatcherSet) MatchAny(input string) bool {
+	node := s.root
+	for i := len(input); i > 0; {
+		for j := i - 1; ; j-- {
+			if input[j] == '.' {
+				node = node.children[input[j+1:i]]
+				i = j
+				break
+			}
+			if j == 0 {
+				node = node.children[input[j:i]]
+				i = j
+				break
+			}
+		}
+		if node == nil {
+			return false
+		}
+		if node.matched {
+			return true
+		}
+		if node.children == nil {
+			return false
+		}
+	}
+	return false
+}

common/geodata/strmatcher/matcherset_domain_test.go

@@ -0,0 +1,95 @@
+package strmatcher_test
+
+import (
+	"reflect"
+	"testing"
+
+	. "github.com/xtls/xray-core/common/geodata/strmatcher"
+)
+
+func TestDomainMatcherSet(t *testing.T) {
+	patterns := []struct {
+		Pattern string
+	}{
+		{
+			Pattern: "example.com",
+		},
+		{
+			Pattern: "google.com",
+		},
+		{
+			Pattern: "x.a.com",
+		},
+		{
+			Pattern: "a.b.com",
+		},
+		{
+			Pattern: "c.a.b.com",
+		},
+		{
+			Pattern: "x.y.com",
+		},
+		{
+			Pattern: "x.y.com",
+		},
+	}
+	testCases := []struct {
+		Domain string
+		Result bool
+	}{
+		{
+			Domain: "x.example.com",
+			Result: true,
+		},
+		{
+			Domain: "y.com",
+			Result: false,
+		},
+		{
+			Domain: "a.b.com",
+			Result: true,
+		},
+		{
+			Domain: "c.a.b.com",
+			Result: true,
+		},
+		{
+			Domain: "c.a..b.com",
+			Result: false,
+		},
+		{
+			Domain: ".com",
+			Result: false,
+		},
+		{
+			Domain: "com",
+			Result: false,
+		},
+		{
+			Domain: "",
+			Result: false,
+		},
+		{
+			Domain: "x.y.com",
+			Result: true,
+		},
+	}
+	s := NewDomainMatcherSet()
+	for _, pattern := range patterns {
+		AddMatcherToSet(s, DomainMatcher(pattern.Pattern))
+	}
+	for _, testCase := range testCases {
+		r := s.MatchAny(testCase.Domain)
+		if !reflect.DeepEqual(r, testCase.Result) {
+			t.Error("Failed to match domain: ", testCase.Domain, ", expect ", testCase.Result, ", but got ", r)
+		}
+	}
+}
+
+func TestEmptyDomainMatcherSet(t *testing.T) {
+	s := NewDomainMatcherSet()
+	r := s.MatchAny("example.com")
+	if r {
+		t.Error("Expect false, but ", r)
+	}
+}

common/geodata/strmatcher/matcherset_full.go

@@ -0,0 +1,24 @@
+package strmatcher
+
+// FullMatcherSet is an implementation of MatcherSet.
+// It uses a hash table to facilitate exact match lookup.
+type FullMatcherSet struct {
+	matchers map[string]struct{}
+}
+
+func NewFullMatcherSet() *FullMatcherSet {
+	return &FullMatcherSet{
+		matchers: make(map[string]struct{}),
+	}
+}
+
+// AddFullMatcher implements MatcherSetForFull.AddFullMatcher.
+func (s *FullMatcherSet) AddFullMatcher(matcher FullMatcher) {
+	s.matchers[matcher.Pattern()] = struct{}{}
+}
+
+// MatchAny implements MatcherSet.Any.
+func (s *FullMatcherSet) MatchAny(input string) bool {
+	_, found := s.matchers[input]
+	return found
+}

common/geodata/strmatcher/matcherset_full_test.go

@@ -0,0 +1,65 @@
+package strmatcher_test
+
+import (
+	"reflect"
+	"testing"
+
+	. "github.com/xtls/xray-core/common/geodata/strmatcher"
+)
+
+func TestFullMatcherSet(t *testing.T) {
+	patterns := []struct {
+		Pattern string
+	}{
+		{
+			Pattern: "example.com",
+		},
+		{
+			Pattern: "google.com",
+		},
+		{
+			Pattern: "x.a.com",
+		},
+		{
+			Pattern: "x.y.com",
+		},
+		{
+			Pattern: "x.y.com",
+		},
+	}
+	testCases := []struct {
+		Domain string
+		Result bool
+	}{
+		{
+			Domain: "example.com",
+			Result: true,
+		},
+		{
+			Domain: "y.com",
+			Result: false,
+		},
+		{
+			Domain: "x.y.com",
+			Result: true,
+		},
+	}
+	s := NewFullMatcherSet()
+	for _, pattern := range patterns {
+		AddMatcherToSet(s, FullMatcher(pattern.Pattern))
+	}
+	for _, testCase := range testCases {
+		r := s.MatchAny(testCase.Domain)
+		if !reflect.DeepEqual(r, testCase.Result) {
+			t.Error("Failed to match domain: ", testCase.Domain, ", expect ", testCase.Result, ", but got ", r)
+		}
+	}
+}
+
+func TestEmptyFullMatcherSet(t *testing.T) {
+	s := NewFullMatcherSet()
+	r := s.MatchAny("example.com")
+	if r {
+		t.Error("Expect false, but ", r)
+	}
+}

common/geodata/strmatcher/matcherset_simple.go

@@ -0,0 +1,22 @@
+package strmatcher
+
+// SimpleMatcherSet is an implementation of MatcherSet.
+// It simply stores all matchers in an array and sequentially matches them.
+type SimpleMatcherSet struct {
+	matchers []Matcher
+}
+
+// AddMatcher implements MatcherSetForAll.AddMatcher.
+func (s *SimpleMatcherSet) AddMatcher(matcher Matcher) {
+	s.matchers = append(s.matchers, matcher)
+}
+
+// MatchAny implements MatcherSet.MatchAny.
+func (s *SimpleMatcherSet) MatchAny(input string) bool {
+	for _, m := range s.matchers {
+		if m.Match(input) {
+			return true
+		}
+	}
+	return false
+}

common/geodata/strmatcher/matcherset_simple_test.go

@@ -0,0 +1,69 @@
+package strmatcher_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/xtls/xray-core/common"
+	. "github.com/xtls/xray-core/common/geodata/strmatcher"
+)
+
+func TestSimpleMatcherSet(t *testing.T) {
+	patterns := []struct {
+		pattern string
+		mType   Type
+	}{
+		{
+			pattern: "example.com",
+			mType:   Domain,
+		},
+		{
+			pattern: "example.com",
+			mType:   Full,
+		},
+		{
+			pattern: "example.com",
+			mType:   Regex,
+		},
+	}
+	cases := []struct {
+		input  string
+		output bool
+	}{
+		{
+			input:  "www.example.com",
+			output: true,
+		},
+		{
+			input:  "example.com",
+			output: true,
+		},
+		{
+			input:  "www.e3ample.com",
+			output: false,
+		},
+		{
+			input:  "xample.com",
+			output: false,
+		},
+		{
+			input:  "xexample.com",
+			output: true,
+		},
+		{
+			input:  "examplexcom",
+			output: true,
+		},
+	}
+	matcherSet := &SimpleMatcherSet{}
+	for _, entry := range patterns {
+		matcher, err := entry.mType.New(entry.pattern)
+		common.Must(err)
+		common.Must(AddMatcherToSet(matcherSet, matcher))
+	}
+	for _, test := range cases {
+		if r := matcherSet.MatchAny(test.input); !reflect.DeepEqual(r, test.output) {
+			t.Error("unexpected output: ", r, " for test case ", test)
+		}
+	}
+}

common/geodata/strmatcher/matcherset_substr.go

@@ -0,0 +1,24 @@
+package strmatcher
+
+import "strings"
+
+// SubstrMatcherSet is implementation of MatcherSet,
+// It is simply implmeneted to comply with the priority specification of Substr matchers.
+type SubstrMatcherSet struct {
+	patterns []string
+}
+
+// AddSubstrMatcher implements MatcherSetForSubstr.AddSubstrMatcher.
+func (s *SubstrMatcherSet) AddSubstrMatcher(matcher SubstrMatcher) {
+	s.patterns = append(s.patterns, matcher.Pattern())
+}
+
+// MatchAny implements MatcherSet.MatchAny.
+func (s *SubstrMatcherSet) MatchAny(input string) bool {
+	for _, pattern := range s.patterns {
+		if strings.Contains(input, pattern) {
+			return true
+		}
+	}
+	return false
+}

common/geodata/strmatcher/matcherset_substr_test.go

@@ -0,0 +1,77 @@
+package strmatcher_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/xtls/xray-core/common"
+	. "github.com/xtls/xray-core/common/geodata/strmatcher"
+)
+
+func TestSubstrMatcherSet(t *testing.T) {
+	patterns := []struct {
+		pattern string
+		mType   Type
+	}{
+		{
+			pattern: "apis",
+			mType:   Substr,
+		},
+		{
+			pattern: "google",
+			mType:   Substr,
+		},
+		{
+			pattern: "apis",
+			mType:   Substr,
+		},
+	}
+	cases := []struct {
+		input  string
+		output bool
+	}{
+		{
+			input:  "google.com",
+			output: true,
+		},
+		{
+			input:  "apis.com",
+			output: true,
+		},
+		{
+			input:  "googleapis.com",
+			output: true,
+		},
+		{
+			input:  "fonts.googleapis.com",
+			output: true,
+		},
+		{
+			input:  "apis.googleapis.com",
+			output: true,
+		},
+		{
+			input:  "baidu.com",
+			output: false,
+		},
+		{
+			input:  "goog",
+			output: false,
+		},
+		{
+			input:  "api",
+			output: false,
+		},
+	}
+	matcherSet := &SubstrMatcherSet{}
+	for _, entry := range patterns {
+		matcher, err := entry.mType.New(entry.pattern)
+		common.Must(err)
+		common.Must(AddMatcherToSet(matcherSet, matcher))
+	}
+	for _, test := range cases {
+		if r := matcherSet.MatchAny(test.input); !reflect.DeepEqual(r, test.output) {
+			t.Error("unexpected output: ", r, " for test case ", test)
+		}
+	}
+}

common/geodata/strmatcher/strmatcher.go

@@ -15,7 +15,7 @@ const (
 )
 
 // Matcher is the interface to determine a string matches a pattern.
-//   - This is a basic matcher to represent a certain kind of match semantic(full, substr, domain or regex).
+//   - This is a basic matcher to represent a certain kind of match semantic (full, substr, domain or regex).
 type Matcher interface {
 	// Type returns the matcher's type.
 	Type() Type
@@ -101,3 +101,21 @@ type ValueMatcher interface {
 	// MatchAny returns true as soon as one matching matcher is found.
 	MatchAny(input string) bool
 }
+
+// MatcherSet is an advanced type of matcher to accept a bunch of basic Matchers (of certain type, not all matcher types).
+// For example:
+//   - FullMatcherSet accepts FullMatcher and uses a hash table to facilitate lookup.
+//   - DomainMatcherSet accepts DomainMatcher and uses a trie to optimize both memory consumption and lookup speed.
+type MatcherSet interface {
+	// MatchAny returns true as soon as one matching matcher is found.
+	MatchAny(input string) bool
+}
+
+// AnyMatcher is a lightweight matcher for callers that only need existence checks.
+type AnyMatcher interface {
+	// Add adds a new Matcher to AnyMatcher.
+	Add(matcher Matcher)
+
+	// MatchAny returns true as soon as one matching matcher is found.
+	MatchAny(input string) bool
+}