Geodata: Prefilter regexp rules by their required literals (#6818)

https://github.com/XTLS/Xray-core/pull/6818#issuecomment-5833512067

common/geodata/strmatcher/benchmark_matchers_test.go

@@ -1,6 +1,7 @@
 package strmatcher_test
 
 import (
+	"regexp"
 	"strconv"
 	"testing"
 
@@ -72,6 +73,64 @@ func BenchmarkSubstrMatcher(b *testing.B) {
 	})
 }
 
+func BenchmarkRegexMatcher(b *testing.B) {
+	patterns := []string{ // taken from geosite
+		`(^|\.)91porn\.(best|com|cool|fun|group|party|plus|site|tw|work)$`,
+		`(^|\.)91porn[0-9]{3}\.me$`,
+		`(^|\.)apiproxy-device-prod-nlb-.+\.amazonaws\.com$`,
+		`(^|\.)dualstack\.apiproxy-.+\.amazonaws\.com$`,
+		`(^|\.)aqdk[0-9]{3}\.com$`,
+		`(^|\.)bilibili3(0[1-9]|1[0-2])\.xyz$`,
+		`(^|\.)byyum([3589]|2[235689]|3[34]|4[1-9]|5[1-79]|6[0134679])?\.com$`,
+		`(^|\.)fiftymvapi\..+$`,
+		`(^|\.)gossipfuli[0-9]{3,4}\.xyz$`,
+		`(^|\.)kpkuang\.(bond|fun|info|one|us)$`,
+		`(^|\.)rule34\.(asia|us|world|xxx|xyz)$`,
+		`(^|\.)[a-z][1-9][0-9][a-z]\.com$`,
+		`.+\.awsdns-[0-9][0-9]\.(co\.uk|com|net|org)$`,
+		`.+\.dkr\.ecr\.[^\.]+\.amazonaws\.com$`,
+		`^(.+\.)*zh\.okaapps\.com$`,
+		`^cdn\d-epicgames-\d+\.file\.myqcloud\.com$`,
+		`^chatgpt-async-webps-prod-\S+-\d+\.webpubsub\.azure\.com$`,
+		`^r+[0-9]+(---|\.)sn-(2x3|ni5|j5o)\w{5}\.googlevideo\.com$`,
+		`^speed\.(coe|open)\.ad\.[a-z]{2,6}\.prod\.hosts\.ooklaserver\.net$`,
+		`javdb\d+\.com$`,
+	}
+	domains := []string{
+		"www.google.com", "rr3---sn-4g5edndy.googlevideo.com", "r1---sn-2x3abcde.googlevideo.com", "i.ytimg.com",
+		"graph.facebook.com", "api.twitter.com", "www.baidu.com", "github.com", "objects.githubusercontent.com",
+		"login.microsoftonline.com", "e1234.dscb.akamaiedge.net", "d1a2b3c4d5e6f7.cloudfront.net",
+		"s3.us-east-1.amazonaws.com", "123456789012.dkr.ecr.us-east-1.amazonaws.com", "www.wikipedia.org",
+		"discord.com", "telegram.org", "store.steampowered.com", "www.91porn.com", "ns-1234.awsdns-12.org",
+	}
+	bench := func(b *testing.B, ctor func(pattern string) func(string) bool) {
+		var matchers []func(string) bool
+		for _, p := range patterns {
+			matchers = append(matchers, ctor(p))
+		}
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			for _, d := range domains {
+				for _, match := range matchers {
+					_ = match(d)
+				}
+			}
+		}
+	}
+	b.Run("regexp", func(b *testing.B) {
+		bench(b, func(pattern string) func(string) bool {
+			return regexp.MustCompile(pattern).MatchString
+		})
+	})
+	b.Run("prefilter", func(b *testing.B) {
+		bench(b, func(pattern string) func(string) bool {
+			m, err := Regex.New(pattern)
+			common.Must(err)
+			return m.Match
+		})
+	})
+}
+
 // Utility functions for benchmark
 
 func benchmarkMatcherType(b *testing.B, t Type, ctor func() MatcherGroup) {

common/geodata/strmatcher/matchers.go

@@ -3,6 +3,7 @@ package strmatcher
 import (
 	"errors"
 	"regexp"
+	"regexp/syntax"
 	"slices"
 	"strings"
 	"unicode/utf8"
@@ -73,7 +74,43 @@ func (m SubstrMatcher) Match(s string) bool {
 
 // RegexMatcher is an implementation of Matcher.
 type RegexMatcher struct {
-	pattern *regexp.Regexp
+	pattern  *regexp.Regexp
+	literals []string // every match contains all of them, longest first
+}
+
+func newRegexMatcher(pattern string) (Matcher, error) {
+	regex, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, err
+	}
+	m := &RegexMatcher{pattern: regex}
+	if re, err := syntax.Parse(pattern, syntax.Perl); err == nil { // same flags as regexp.Compile
+		m.literals = requiredLiterals(re, nil)
+		slices.SortStableFunc(m.literals, func(a, b string) int { return len(b) - len(a) })
+	}
+	return m, nil
+}
+
+// requiredLiterals appends to dst the case-sensitive strings that every match of re contains.
+func requiredLiterals(re *syntax.Regexp, dst []string) []string {
+	switch re.Op {
+	case syntax.OpLiteral:
+		// regexp matches U+FFFD against invalid UTF-8 bytes, strings.Contains does not
+		if re.Flags&syntax.FoldCase == 0 && !slices.Contains(re.Rune, utf8.RuneError) {
+			dst = append(dst, string(re.Rune))
+		}
+	case syntax.OpCapture, syntax.OpPlus:
+		dst = requiredLiterals(re.Sub[0], dst)
+	case syntax.OpRepeat:
+		if re.Min > 0 {
+			dst = requiredLiterals(re.Sub[0], dst)
+		}
+	case syntax.OpConcat:
+		for _, sub := range re.Sub {
+			dst = requiredLiterals(sub, dst)
+		}
+	}
+	return dst
 }
 
 func (*RegexMatcher) Type() Type {
@@ -89,6 +126,11 @@ func (m *RegexMatcher) String() string {
 }
 
 func (m *RegexMatcher) Match(s string) bool {
+	for _, l := range m.literals {
+		if !strings.Contains(s, l) {
+			return false
+		}
+	}
 	return m.pattern.MatchString(s)
 }
 
@@ -102,11 +144,7 @@ func (t Type) New(pattern string) (Matcher, error) {
 	case Domain:
 		return DomainMatcher(pattern), nil
 	case Regex: // 1. regex matching is case-sensitive
-		regex, err := regexp.Compile(pattern)
-		if err != nil {
-			return nil, err
-		}
-		return &RegexMatcher{pattern: regex}, nil
+		return newRegexMatcher(pattern)
 	default:
 		return nil, errors.New("unknown matcher type")
 	}
@@ -135,11 +173,7 @@ func (t Type) NewDomainPattern(pattern string) (Matcher, error) {
 		}
 		return DomainMatcher(pattern), nil
 	case Regex: // Regex's charset not in LDH subset
-		regex, err := regexp.Compile(pattern)
-		if err != nil {
-			return nil, err
-		}
-		return &RegexMatcher{pattern: regex}, nil
+		return newRegexMatcher(pattern)
 	default:
 		return nil, errors.New("unknown matcher type")
 	}

common/geodata/strmatcher/matchers_regex_test.go

@@ -0,0 +1,60 @@
+package strmatcher
+
+import (
+	"regexp"
+	"slices"
+	"testing"
+)
+
+var regexLiteralCases = []struct {
+	pattern  string
+	literals []string
+}{
+	{`(^|\.)91porn\.(best|com)$`, []string{"91porn."}},
+	{`.+\.awsdns-cn-[0-9][0-9]\.(biz|com|net|top)$`, []string{".awsdns-cn-", "."}},
+	{`^r+[0-9]+(---|\.)sn-(2x3|ni5|j5o)\w{5}\.googlevideo\.com$`, []string{".googlevideo.com", "sn-", "r"}},
+	{`(?i)abc`, nil},
+	{`ab(?i:CD)ef`, []string{"ab", "ef"}},
+	{`(abc)?x`, []string{"x"}},
+	{`(abc)*x`, []string{"x"}},
+	{`x{0,3}yy`, []string{"yy"}},
+	{`(ab)+c{2}`, []string{"ab", "c"}},
+	{`abc|abd`, []string{"ab"}},
+	{`\Qa.b\E`, []string{"a.b"}},
+	{`a\x{FFFD}b`, nil},
+	{`^[^.]+$`, nil},
+}
+
+func TestRegexRequiredLiterals(t *testing.T) {
+	for _, test := range regexLiteralCases {
+		m, err := newRegexMatcher(test.pattern)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got := m.(*RegexMatcher).literals; !slices.Equal(got, test.literals) {
+			t.Errorf("%s: got %q, want %q", test.pattern, got, test.literals)
+		}
+	}
+}
+
+func FuzzRegexMatcher(f *testing.F) {
+	inputs := []string{
+		"", "x", "yy", "abd", "ccc", "ABC", "abCDef", "abcdef", "abababcc", "a.b", "a\xffb", "a\uFFFDb",
+		"www.91porn.com", "ns1.awsdns-cn-01.top", "r1---sn-2x3abcde.googlevideo.com",
+	}
+	for _, test := range regexLiteralCases {
+		for _, s := range inputs {
+			f.Add(test.pattern, s)
+		}
+	}
+	f.Fuzz(func(t *testing.T, pattern, s string) {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return
+		}
+		m, _ := newRegexMatcher(pattern)
+		if got, want := m.Match(s), re.MatchString(s); got != want {
+			t.Errorf("pattern %q, input %q: got %v, want %v", pattern, s, got, want)
+		}
+	})
+}