chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
agapple
2014-09-23 15:11:35 +08:00
commit a0defe575b
331 changed files with 53031 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
filter/pom.xml Normal file
View File

@@ -0,0 +1,39 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.19-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.filter</artifactId>
<packaging>jar</packaging>
<name>canal sink module for otter ${project.version}</name>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
</dependency>
<dependency>
<groupId>oro</groupId>
<artifactId>oro</artifactId>
</dependency>
<!-- test dependency -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

13
filter/src/main/java/com/alibaba/otter/canal/filter/CanalEventFilter.java Normal file
View File

@@ -0,0 +1,13 @@
package com.alibaba.otter.canal.filter;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
/**
* 数据过滤机制
*
* @author jianghang 2012-7-20 下午03:51:27
*/
public interface CanalEventFilter<T> {
boolean filter(T event) throws CanalFilterException;
}

48
filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java Normal file
View File

@@ -0,0 +1,48 @@
package com.alibaba.otter.canal.filter;
import java.util.Map;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
/**
* 提供{@linkplain Pattern}的lazy get处理
*
* @author jianghang 2013-1-22 下午09:36:44
* @version 1.0.0
*/
public class PatternUtils {
private static Map<String, Pattern> patterns = new MapMaker().softValues().makeComputingMap(
new Function<String, Pattern>() {
public Pattern apply(
String pattern) {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(
pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK
| Perl5Compiler.READ_ONLY_MASK
| Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e) {
throw new CanalFilterException(
e);
}
}
});
public static Pattern getPattern(String pattern) {
return patterns.get(pattern);
}
public static void clear() {
patterns.clear();
}
}

37
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java Normal file
View File

@@ -0,0 +1,37 @@
package com.alibaba.otter.canal.filter.aviater;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.googlecode.aviator.AviatorEvaluator;
/**
* 基于aviater el表达式的匹配过滤
*
* @author jianghang 2012-7-23 上午10:46:32
*/
public class AviaterELFilter implements CanalEventFilter<CanalEntry.Entry> {
public static final String ROOT_KEY = "entry";
private String expression;
public AviaterELFilter(String expression){
this.expression = expression;
}
public boolean filter(CanalEntry.Entry entry) throws CanalFilterException {
if (StringUtils.isEmpty(expression)) {
return true;
}
Map<String, Object> env = new HashMap<String, Object>();
env.put(ROOT_KEY, entry);
return (Boolean) AviatorEvaluator.execute(expression, env);
}
}

128
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java Normal file
View File

@@ -0,0 +1,128 @@
package com.alibaba.otter.canal.filter.aviater;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.Expression;
/**
* 基于aviater进行tableName正则匹配的过滤算法
*
* @author jianghang 2012-7-20 下午06:01:34
*/
public class AviaterRegexFilter implements CanalEventFilter<String> {
private static final String SPLIT = ",";
private static final String PATTERN_SPLIT = "|";
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
private static final RegexFunction regexFunction = new RegexFunction();
private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
static {
AviatorEvaluator.addFunction(regexFunction);
}
private static final Comparator<String> COMPARATOR = new StringComparator();
final private String pattern;
final private boolean defaultEmptyValue;
public AviaterRegexFilter(String pattern){
this(pattern, true);
}
public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null;
if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>();
} else {
String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss);
}
// 对pattern按照从长到短的排序
// 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
// 的长度不一样
Collections.sort(list, COMPARATOR);
// 对pattern进行头尾完全匹配
list = completionPattern(list);
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
}
public boolean filter(String filtered) throws CanalFilterException {
if (StringUtils.isEmpty(pattern)) {
return defaultEmptyValue;
}
if (StringUtils.isEmpty(filtered)) {
return defaultEmptyValue;
}
Map<String, Object> env = new HashMap<String, Object>();
env.put("pattern", pattern);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
}
/**
* 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
*
* <pre>
* foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
* </pre>
*
* 因此此类对正则表达式进行了从长到短的排序
*
* @author zebin.xuzb 2012-10-22 下午2:02:26
* @version 1.0.0
*/
private static class StringComparator implements Comparator<String> {
@Override
public int compare(String str1, String str2) {
if (str1.length() > str2.length()) {
return -1;
} else if (str1.length() < str2.length()) {
return 1;
} else {
return 0;
}
}
}
/**
* 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
*
* <pre>
* foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
* </pre>
*
* 因此此类对正则表达式进行头尾完全匹配
*
* @author simon
* @version 1.0.0
*/
private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>();
for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^");
stringBuffer.append(pattern);
stringBuffer.append("$");
result.add(stringBuffer.toString());
}
return result;
}
}

53
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java Normal file
View File

@@ -0,0 +1,53 @@
package com.alibaba.otter.canal.filter.aviater;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.filter.CanalEventFilter;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.googlecode.aviator.AviatorEvaluator;
import com.googlecode.aviator.Expression;
/**
* 基于aviater进行tableName简单过滤计算，不支持正则匹配
*
* @author jianghang 2012-7-20 下午05:53:30
*/
public class AviaterSimpleFilter implements CanalEventFilter<String> {
private static final String SPLIT = ",";
private static final String FILTER_EXPRESSION = "include(list,target)";
private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
private final List<String> list;
public AviaterSimpleFilter(String filterExpression){
if (StringUtils.isEmpty(filterExpression)) {
list = new ArrayList<String>();
} else {
String[] ss = filterExpression.toLowerCase().split(SPLIT);
list = Arrays.asList(ss);
}
}
public boolean filter(String filtered) throws CanalFilterException {
if (list.isEmpty()) {
return true;
}
if (StringUtils.isEmpty(filtered)) {
return true;
}
Map<String, Object> env = new HashMap<String, Object>();
env.put("list", list);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
}
}

32
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java Normal file
View File

@@ -0,0 +1,32 @@
package com.alibaba.otter.canal.filter.aviater;
import java.util.Map;
import org.apache.oro.text.regex.Perl5Matcher;
import com.alibaba.otter.canal.filter.PatternUtils;
import com.googlecode.aviator.runtime.function.AbstractFunction;
import com.googlecode.aviator.runtime.function.FunctionUtils;
import com.googlecode.aviator.runtime.type.AviatorBoolean;
import com.googlecode.aviator.runtime.type.AviatorObject;
/**
* 提供aviator regex的代码扩展
*
* @author jianghang 2012-7-23 上午10:29:23
*/
public class RegexFunction extends AbstractFunction {
public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
String pattern = FunctionUtils.getStringValue(arg1, env);
String text = FunctionUtils.getStringValue(arg2, env);
Perl5Matcher matcher = new Perl5Matcher();
boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
return AviatorBoolean.valueOf(isMatch);
}
public String getName() {
return "regex";
}
}

35
filter/src/main/java/com/alibaba/otter/canal/filter/exception/CanalFilterException.java Normal file
View File

@@ -0,0 +1,35 @@
package com.alibaba.otter.canal.filter.exception;
import com.alibaba.otter.canal.common.CanalException;
/**
* canal 异常定义
*
* @author jianghang 2012-6-15 下午04:57:35
* @version 1.0.0
*/
public class CanalFilterException extends CanalException {
private static final long serialVersionUID = -7288830284122672209L;
public CanalFilterException(String errorCode){
super(errorCode);
}
public CanalFilterException(String errorCode, Throwable cause){
super(errorCode, cause);
}
public CanalFilterException(String errorCode, String errorDesc){
super(errorCode + ":" + errorDesc);
}
public CanalFilterException(String errorCode, String errorDesc, Throwable cause){
super(errorCode + ":" + errorDesc, cause);
}
public CanalFilterException(Throwable cause){
super(cause);
}
}

108
filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java Normal file
View File

@@ -0,0 +1,108 @@
package com.alibaba.otter.canal.filter;
import junit.framework.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.filter.aviater.AviaterELFilter;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.filter.aviater.AviaterSimpleFilter;
import com.alibaba.otter.canal.protocol.CanalEntry;
public class AviaterFilterTest {
@Test
public void test_simple() {
AviaterSimpleFilter filter = new AviaterSimpleFilter("s1.t1,s2.t2");
boolean result = filter.filter("s1.t1");
Assert.assertEquals(true, result);
result = filter.filter("s1.t2");
Assert.assertEquals(false, result);
result = filter.filter("");
Assert.assertEquals(true, result);
result = filter.filter("s1.t1,s2.t2");
Assert.assertEquals(false, result);
result = filter.filter("s2.t2");
Assert.assertEquals(true, result);
}
@Test
public void test_regex() {
AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
boolean result = filter.filter("s1.t1");
Assert.assertEquals(true, result);
result = filter.filter("s1.t2");
Assert.assertEquals(true, result);
result = filter.filter("");
Assert.assertEquals(true, result);
result = filter.filter("s12.t1");
Assert.assertEquals(false, result);
result = filter.filter("s2.t2");
Assert.assertEquals(true, result);
result = filter.filter("s3.t2");
Assert.assertEquals(false, result);
AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");
result = filter2.filter("s1.t1");
Assert.assertEquals(true, result);
result = filter2.filter("s1.t2");
Assert.assertEquals(true, result);
result = filter2.filter("s2.t1");
Assert.assertEquals(true, result);
AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");
result = filter3.filter("fooooot");
Assert.assertEquals(true, result);
AviaterRegexFilter filter4 = new AviaterRegexFilter(
"otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
result = filter4.filter("otter1.otter_stability1");
Assert.assertEquals(true, result);
}
@Test
public void testDisordered() {
AviaterRegexFilter filter = new AviaterRegexFilter(
"u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");
boolean result = filter.filter("u.abc");
Assert.assertEquals(true, result);
result = filter.filter("ab.x");
Assert.assertEquals(true, result);
result = filter.filter("ab.xyz1");
Assert.assertEquals(false, result);
result = filter.filter("abc.xyz");
Assert.assertEquals(true, result);
result = filter.filter("uv.xyz");
Assert.assertEquals(true, result);
}
@Test
public void test_el() {
AviaterELFilter filter = new AviaterELFilter("str(entry.entryType) == 'ROWDATA'");
CanalEntry.Entry.Builder entry = CanalEntry.Entry.newBuilder();
entry.setEntryType(CanalEntry.EntryType.ROWDATA);
boolean result = filter.filter(entry.build());
Assert.assertEquals(true, result);
}
}

63
filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java Normal file
View File

@@ -0,0 +1,63 @@
package com.alibaba.otter.canal.filter;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import junit.framework.Assert;
import org.apache.commons.lang.RandomStringUtils;
import org.junit.Test;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
public class MutliAviaterFilterTest {
@Test
public void test_simple() {
int count = 5;
ExecutorService executor = Executors.newFixedThreadPool(count);
final CountDownLatch countDown = new CountDownLatch(count);
final AtomicInteger successed = new AtomicInteger(0);
for (int i = 0; i < count; i++) {
executor.submit(new Runnable() {
public void run() {
try {
for (int i = 0; i < 100; i++) {
doRegexTest();
// try {
// Thread.sleep(10);
// } catch (InterruptedException e) {
// }
}
successed.incrementAndGet();
} finally {
countDown.countDown();
}
}
});
}
try {
countDown.await();
} catch (InterruptedException e) {
}
Assert.assertEquals(count, successed.get());
executor.shutdownNow();
}
private void doRegexTest() {
AviaterRegexFilter filter3 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|"
+ RandomStringUtils.randomAlphabetic(200));
boolean result = filter3.filter("otter1.otter_stability1");
Assert.assertEquals(true, result);
result = filter3.filter("otter2.otter_stability1");
Assert.assertEquals(true, result);
}
}