mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fix: preserve commas inside regex grouping when splitting filter patterns (#5588)
AviaterRegexFilter#init splits the configured pattern on every comma to
separate multiple regular expressions. That blindly breaks regex
quantifiers such as \d{1,2} or non-capturing alternations like
(a,b|c) because the comma inside the grouping construct is treated as
a top-level separator.
Replace the StringUtils.split call with a parse-aware splitter that
tracks the depth of {}, [], () and respects backslash escapes, so only
top-level commas separate patterns. Apply the change to both copies of
AviaterRegexFilter (filter module and connector/core module).
Add AviaterFilterTest#test_regex_with_quantifier_comma covering: a
single-pattern \d{1,2} quantifier, a multi-pattern list combined with
\d{1,2}, and an open-ended {n,} quantifier.
Fixes #5442
This commit is contained in:
@@ -1,7 +1,6 @@
|
|||||||
package com.alibaba.otter.canal.connector.core.filter;
|
package com.alibaba.otter.canal.connector.core.filter;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -19,7 +18,6 @@ import com.googlecode.aviator.Expression;
|
|||||||
*/
|
*/
|
||||||
public class AviaterRegexFilter {
|
public class AviaterRegexFilter {
|
||||||
|
|
||||||
private static final String SPLIT = ",";
|
|
||||||
private static final String PATTERN_SPLIT = "|";
|
private static final String PATTERN_SPLIT = "|";
|
||||||
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
|
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
|
||||||
private static final RegexFunction regexFunction = new RegexFunction();
|
private static final RegexFunction regexFunction = new RegexFunction();
|
||||||
@@ -43,8 +41,9 @@ public class AviaterRegexFilter {
|
|||||||
if (StringUtils.isEmpty(pattern)) {
|
if (StringUtils.isEmpty(pattern)) {
|
||||||
list = new ArrayList<>();
|
list = new ArrayList<>();
|
||||||
} else {
|
} else {
|
||||||
String[] ss = StringUtils.split(pattern, SPLIT);
|
// fix issue #5442: only split on commas outside regex grouping constructs,
|
||||||
list = Arrays.asList(ss);
|
// so quantifiers like \d{1,2} are kept intact.
|
||||||
|
list = splitPattern(pattern);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 对pattern按照从长到短的排序
|
// 对pattern按照从长到短的排序
|
||||||
@@ -56,6 +55,60 @@ public class AviaterRegexFilter {
|
|||||||
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
|
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按顶层逗号拆分多个正则表达式,跳过位于 {}, [], () 内的逗号,避免破坏类似 \d{1,2} 的量词写法。
|
||||||
|
*/
|
||||||
|
private static List<String> splitPattern(String pattern) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
int braceDepth = 0;
|
||||||
|
int bracketDepth = 0;
|
||||||
|
int parenDepth = 0;
|
||||||
|
int start = 0;
|
||||||
|
int len = pattern.length();
|
||||||
|
for (int i = 0; i < len; i++) {
|
||||||
|
char c = pattern.charAt(i);
|
||||||
|
// 反斜杠转义:跳过下一个字符,避免把 \{ \[ \( 等误判为分组开始
|
||||||
|
if (c == '\\' && i + 1 < len) {
|
||||||
|
i++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
switch (c) {
|
||||||
|
case '{':
|
||||||
|
braceDepth++;
|
||||||
|
break;
|
||||||
|
case '}':
|
||||||
|
if (braceDepth > 0) braceDepth--;
|
||||||
|
break;
|
||||||
|
case '[':
|
||||||
|
bracketDepth++;
|
||||||
|
break;
|
||||||
|
case ']':
|
||||||
|
if (bracketDepth > 0) bracketDepth--;
|
||||||
|
break;
|
||||||
|
case '(':
|
||||||
|
parenDepth++;
|
||||||
|
break;
|
||||||
|
case ')':
|
||||||
|
if (parenDepth > 0) parenDepth--;
|
||||||
|
break;
|
||||||
|
case ',':
|
||||||
|
if (braceDepth == 0 && bracketDepth == 0 && parenDepth == 0) {
|
||||||
|
if (i > start) {
|
||||||
|
result.add(pattern.substring(start, i));
|
||||||
|
}
|
||||||
|
start = i + 1;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (start < len) {
|
||||||
|
result.add(pattern.substring(start));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public boolean filter(String filtered) {
|
public boolean filter(String filtered) {
|
||||||
if (StringUtils.isEmpty(pattern)) {
|
if (StringUtils.isEmpty(pattern)) {
|
||||||
return defaultEmptyValue;
|
return defaultEmptyValue;
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package com.alibaba.otter.canal.filter.aviater;
|
package com.alibaba.otter.canal.filter.aviater;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -21,7 +20,6 @@ import com.googlecode.aviator.Expression;
|
|||||||
*/
|
*/
|
||||||
public class AviaterRegexFilter implements CanalEventFilter<String> {
|
public class AviaterRegexFilter implements CanalEventFilter<String> {
|
||||||
|
|
||||||
private static final String SPLIT = ",";
|
|
||||||
private static final String PATTERN_SPLIT = "|";
|
private static final String PATTERN_SPLIT = "|";
|
||||||
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
|
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
|
||||||
private static final RegexFunction regexFunction = new RegexFunction();
|
private static final RegexFunction regexFunction = new RegexFunction();
|
||||||
@@ -45,8 +43,9 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
|
|||||||
if (StringUtils.isEmpty(pattern)) {
|
if (StringUtils.isEmpty(pattern)) {
|
||||||
list = new ArrayList<>();
|
list = new ArrayList<>();
|
||||||
} else {
|
} else {
|
||||||
String[] ss = StringUtils.split(pattern, SPLIT);
|
// fix issue #5442: only split on commas outside regex grouping constructs,
|
||||||
list = Arrays.asList(ss);
|
// so quantifiers like \d{1,2} are kept intact.
|
||||||
|
list = splitPattern(pattern);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 对pattern按照从长到短的排序
|
// 对pattern按照从长到短的排序
|
||||||
@@ -58,6 +57,60 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
|
|||||||
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
|
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按顶层逗号拆分多个正则表达式,跳过位于 {}, [], () 内的逗号,避免破坏类似 \d{1,2} 的量词写法。
|
||||||
|
*/
|
||||||
|
private static List<String> splitPattern(String pattern) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
int braceDepth = 0;
|
||||||
|
int bracketDepth = 0;
|
||||||
|
int parenDepth = 0;
|
||||||
|
int start = 0;
|
||||||
|
int len = pattern.length();
|
||||||
|
for (int i = 0; i < len; i++) {
|
||||||
|
char c = pattern.charAt(i);
|
||||||
|
// 反斜杠转义:跳过下一个字符,避免把 \{ \[ \( 等误判为分组开始
|
||||||
|
if (c == '\\' && i + 1 < len) {
|
||||||
|
i++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
switch (c) {
|
||||||
|
case '{':
|
||||||
|
braceDepth++;
|
||||||
|
break;
|
||||||
|
case '}':
|
||||||
|
if (braceDepth > 0) braceDepth--;
|
||||||
|
break;
|
||||||
|
case '[':
|
||||||
|
bracketDepth++;
|
||||||
|
break;
|
||||||
|
case ']':
|
||||||
|
if (bracketDepth > 0) bracketDepth--;
|
||||||
|
break;
|
||||||
|
case '(':
|
||||||
|
parenDepth++;
|
||||||
|
break;
|
||||||
|
case ')':
|
||||||
|
if (parenDepth > 0) parenDepth--;
|
||||||
|
break;
|
||||||
|
case ',':
|
||||||
|
if (braceDepth == 0 && bracketDepth == 0 && parenDepth == 0) {
|
||||||
|
if (i > start) {
|
||||||
|
result.add(pattern.substring(start, i));
|
||||||
|
}
|
||||||
|
start = i + 1;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (start < len) {
|
||||||
|
result.add(pattern.substring(start));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public boolean filter(String filtered) throws CanalFilterException {
|
public boolean filter(String filtered) throws CanalFilterException {
|
||||||
if (StringUtils.isEmpty(pattern)) {
|
if (StringUtils.isEmpty(pattern)) {
|
||||||
return defaultEmptyValue;
|
return defaultEmptyValue;
|
||||||
|
|||||||
@@ -77,6 +77,28 @@ public class AviaterFilterTest {
|
|||||||
Assert.assertEquals(true, result);
|
Assert.assertEquals(true, result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void test_regex_with_quantifier_comma() {
|
||||||
|
// fix issue #5442: comma inside {n,m} quantifier should not split the pattern
|
||||||
|
AviaterRegexFilter filter = new AviaterRegexFilter("test\\.order_\\d{1,2}");
|
||||||
|
Assert.assertTrue(filter.filter("test.order_5"));
|
||||||
|
Assert.assertTrue(filter.filter("test.order_42"));
|
||||||
|
Assert.assertFalse(filter.filter("test.order_123"));
|
||||||
|
Assert.assertFalse(filter.filter("test.order_"));
|
||||||
|
|
||||||
|
// multiple patterns combined with quantifier should still split correctly
|
||||||
|
AviaterRegexFilter filter2 = new AviaterRegexFilter("test\\.order_\\d{1,2},test\\.user_\\d+");
|
||||||
|
Assert.assertTrue(filter2.filter("test.order_7"));
|
||||||
|
Assert.assertTrue(filter2.filter("test.user_99"));
|
||||||
|
Assert.assertFalse(filter2.filter("test.order_123"));
|
||||||
|
|
||||||
|
// open-ended quantifier {n,}
|
||||||
|
AviaterRegexFilter filter3 = new AviaterRegexFilter("test\\.t_\\d{2,}");
|
||||||
|
Assert.assertTrue(filter3.filter("test.t_12"));
|
||||||
|
Assert.assertTrue(filter3.filter("test.t_123"));
|
||||||
|
Assert.assertFalse(filter3.filter("test.t_1"));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testDisordered() {
|
public void testDisordered() {
|
||||||
AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");
|
AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");
|
||||||
|
|||||||
Reference in New Issue
Block a user