From cf628769d645f5f97f3006e085a86cd57b8a1ab9 Mon Sep 17 00:00:00 2001 From: Guimu <30684111+daguimu@users.noreply.github.com> Date: Fri, 29 May 2026 10:39:04 +0800 Subject: [PATCH] fix: preserve commas inside regex grouping when splitting filter patterns (#5588) AviaterRegexFilter#init splits the configured pattern on every comma to separate multiple regular expressions. That blindly breaks regex quantifiers such as \d{1,2} or non-capturing alternations like (a,b|c) because the comma inside the grouping construct is treated as a top-level separator. Replace the StringUtils.split call with a parse-aware splitter that tracks the depth of {}, [], () and respects backslash escapes, so only top-level commas separate patterns. Apply the change to both copies of AviaterRegexFilter (filter module and connector/core module). Add AviaterFilterTest#test_regex_with_quantifier_comma covering: a single-pattern \d{1,2} quantifier, a multi-pattern list combined with \d{1,2}, and an open-ended {n,} quantifier. Fixes #5442 --- .../core/filter/AviaterRegexFilter.java | 61 +++++++++++++++++-- .../filter/aviater/AviaterRegexFilter.java | 61 +++++++++++++++++-- .../otter/canal/filter/AviaterFilterTest.java | 22 +++++++ 3 files changed, 136 insertions(+), 8 deletions(-) diff --git a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java index 6d7bb283..384d417e 100644 --- a/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java +++ b/connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java @@ -1,7 +1,6 @@ package com.alibaba.otter.canal.connector.core.filter; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -19,7 +18,6 @@ import com.googlecode.aviator.Expression; */ public class AviaterRegexFilter { - private static final String SPLIT = ","; private static final String PATTERN_SPLIT = "|"; private static final String FILTER_EXPRESSION = "regex(pattern,target)"; private static final RegexFunction regexFunction = new RegexFunction(); @@ -43,8 +41,9 @@ public class AviaterRegexFilter { if (StringUtils.isEmpty(pattern)) { list = new ArrayList<>(); } else { - String[] ss = StringUtils.split(pattern, SPLIT); - list = Arrays.asList(ss); + // fix issue #5442: only split on commas outside regex grouping constructs, + // so quantifiers like \d{1,2} are kept intact. + list = splitPattern(pattern); } // 对pattern按照从长到短的排序 @@ -56,6 +55,60 @@ public class AviaterRegexFilter { this.pattern = StringUtils.join(list, PATTERN_SPLIT); } + /** + * 按顶层逗号拆分多个正则表达式，跳过位于 {}, [], () 内的逗号，避免破坏类似 \d{1,2} 的量词写法。 + */ + private static List splitPattern(String pattern) { + List result = new ArrayList<>(); + int braceDepth = 0; + int bracketDepth = 0; + int parenDepth = 0; + int start = 0; + int len = pattern.length(); + for (int i = 0; i < len; i++) { + char c = pattern.charAt(i); + // 反斜杠转义：跳过下一个字符，避免把 \{ \[ \( 等误判为分组开始 + if (c == '\\' && i + 1 < len) { + i++; + continue; + } + switch (c) { + case '{': + braceDepth++; + break; + case '}': + if (braceDepth > 0) braceDepth--; + break; + case '[': + bracketDepth++; + break; + case ']': + if (bracketDepth > 0) bracketDepth--; + break; + case '(': + parenDepth++; + break; + case ')': + if (parenDepth > 0) parenDepth--; + break; + case ',': + if (braceDepth == 0 && bracketDepth == 0 && parenDepth == 0) { + if (i > start) { + result.add(pattern.substring(start, i)); + } + start = i + 1; + } + break; + default: + break; + } + } + if (start < len) { + result.add(pattern.substring(start)); + } + return result; + } + public boolean filter(String filtered) { if (StringUtils.isEmpty(pattern)) { return defaultEmptyValue; diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java index cb20762a..48e51e63 100644 --- a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java +++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java @@ -1,7 +1,6 @@ package com.alibaba.otter.canal.filter.aviater; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -21,7 +20,6 @@ import com.googlecode.aviator.Expression; */ public class AviaterRegexFilter implements CanalEventFilter { - private static final String SPLIT = ","; private static final String PATTERN_SPLIT = "|"; private static final String FILTER_EXPRESSION = "regex(pattern,target)"; private static final RegexFunction regexFunction = new RegexFunction(); @@ -45,8 +43,9 @@ public class AviaterRegexFilter implements CanalEventFilter { if (StringUtils.isEmpty(pattern)) { list = new ArrayList<>(); } else { - String[] ss = StringUtils.split(pattern, SPLIT); - list = Arrays.asList(ss); + // fix issue #5442: only split on commas outside regex grouping constructs, + // so quantifiers like \d{1,2} are kept intact. + list = splitPattern(pattern); } // 对pattern按照从长到短的排序 @@ -58,6 +57,60 @@ public class AviaterRegexFilter implements CanalEventFilter { this.pattern = StringUtils.join(list, PATTERN_SPLIT); } + /** + * 按顶层逗号拆分多个正则表达式，跳过位于 {}, [], () 内的逗号，避免破坏类似 \d{1,2} 的量词写法。 + */ + private static List splitPattern(String pattern) { + List result = new ArrayList<>(); + int braceDepth = 0; + int bracketDepth = 0; + int parenDepth = 0; + int start = 0; + int len = pattern.length(); + for (int i = 0; i < len; i++) { + char c = pattern.charAt(i); + // 反斜杠转义：跳过下一个字符，避免把 \{ \[ \( 等误判为分组开始 + if (c == '\\' && i + 1 < len) { + i++; + continue; + } + switch (c) { + case '{': + braceDepth++; + break; + case '}': + if (braceDepth > 0) braceDepth--; + break; + case '[': + bracketDepth++; + break; + case ']': + if (bracketDepth > 0) bracketDepth--; + break; + case '(': + parenDepth++; + break; + case ')': + if (parenDepth > 0) parenDepth--; + break; + case ',': + if (braceDepth == 0 && bracketDepth == 0 && parenDepth == 0) { + if (i > start) { + result.add(pattern.substring(start, i)); + } + start = i + 1; + } + break; + default: + break; + } + } + if (start < len) { + result.add(pattern.substring(start)); + } + return result; + } + public boolean filter(String filtered) throws CanalFilterException { if (StringUtils.isEmpty(pattern)) { return defaultEmptyValue; diff --git a/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java b/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java index 39bcd013..b82d3ace 100644 --- a/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java +++ b/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java @@ -77,6 +77,28 @@ public class AviaterFilterTest { Assert.assertEquals(true, result); } + @Test + public void test_regex_with_quantifier_comma() { + // fix issue #5442: comma inside {n,m} quantifier should not split the pattern + AviaterRegexFilter filter = new AviaterRegexFilter("test\\.order_\\d{1,2}"); + Assert.assertTrue(filter.filter("test.order_5")); + Assert.assertTrue(filter.filter("test.order_42")); + Assert.assertFalse(filter.filter("test.order_123")); + Assert.assertFalse(filter.filter("test.order_")); + + // multiple patterns combined with quantifier should still split correctly + AviaterRegexFilter filter2 = new AviaterRegexFilter("test\\.order_\\d{1,2},test\\.user_\\d+"); + Assert.assertTrue(filter2.filter("test.order_7")); + Assert.assertTrue(filter2.filter("test.user_99")); + Assert.assertFalse(filter2.filter("test.order_123")); + + // open-ended quantifier {n,} + AviaterRegexFilter filter3 = new AviaterRegexFilter("test\\.t_\\d{2,}"); + Assert.assertTrue(filter3.filter("test.t_12")); + Assert.assertTrue(filter3.filter("test.t_123")); + Assert.assertFalse(filter3.filter("test.t_1")); + } + @Test public void testDisordered() { AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");