mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
模块化mq connector (#2562)
* add connector module, add kafka producer connector module for server * add rocketMQ producer connector module for server * add rabbitMQ producer connector module for server * optimize code, add annotation * add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
40
connector/core/pom.xml
Normal file
40
connector/core/pom.xml
Normal file
@@ -0,0 +1,40 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>canal.connector</artifactId>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<version>1.1.5-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.core</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal connector core module for otter ${project.version}</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.googlecode.aviator</groupId>
|
||||
<artifactId>aviator</artifactId>
|
||||
<version>2.2.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>oro</groupId>
|
||||
<artifactId>oro</artifactId>
|
||||
<version>2.0.8</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
<version>2.9.4</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.alibaba.otter.canal.connector.core.config;
|
||||
|
||||
/**
|
||||
* MQ配置常量
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalConstants {
|
||||
|
||||
public static final String ROOT = "canal";
|
||||
|
||||
public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
|
||||
|
||||
public static final String CANAL_MQ_FLAT_MESSAGE = ROOT + "." + "mq.flat.message";
|
||||
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
|
||||
public static final String CANAL_MQ_PARALLEL_THREAD_SIZE = ROOT + "." + "mq.parallel.thread.size";
|
||||
public static final String CANAL_MQ_CANAL_BATCH_SIZE = ROOT + "." + "mq.canal.batch.size";
|
||||
public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT = ROOT + "." + "mq.canal.fetch.timeout";
|
||||
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.access.channel";
|
||||
|
||||
public static final String CANAL_ALIYUN_ACCESS_KEY = ROOT + "." + "aliyun.accessKey";
|
||||
public static final String CANAL_ALIYUN_SECRET_KEY = ROOT + "." + "aliyun.secretKey";
|
||||
public static final String CANAL_ALIYUN_UID = ROOT + "." + "aliyun.uid";
|
||||
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package com.alibaba.otter.canal.connector.core.config;
|
||||
|
||||
/**
|
||||
* MQ配置类
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class MQProperties {
|
||||
|
||||
private boolean flatMessage = true;
|
||||
private boolean databaseHash = true;
|
||||
private boolean filterTransactionEntry = true;
|
||||
private Integer parallelThreadSize = 8;
|
||||
private Integer fetchTimeout = 100;
|
||||
private Integer batchSize = 50;
|
||||
private String accessChannel = "local";
|
||||
|
||||
private String aliyunAccessKey = "";
|
||||
private String aliyunSecretKey = "";
|
||||
private int aliyunUid = 0;
|
||||
|
||||
public boolean isFlatMessage() {
|
||||
return flatMessage;
|
||||
}
|
||||
|
||||
public void setFlatMessage(boolean flatMessage) {
|
||||
this.flatMessage = flatMessage;
|
||||
}
|
||||
|
||||
public boolean isDatabaseHash() {
|
||||
return databaseHash;
|
||||
}
|
||||
|
||||
public void setDatabaseHash(boolean databaseHash) {
|
||||
this.databaseHash = databaseHash;
|
||||
}
|
||||
|
||||
public boolean isFilterTransactionEntry() {
|
||||
return filterTransactionEntry;
|
||||
}
|
||||
|
||||
public void setFilterTransactionEntry(boolean filterTransactionEntry) {
|
||||
this.filterTransactionEntry = filterTransactionEntry;
|
||||
}
|
||||
|
||||
public Integer getParallelThreadSize() {
|
||||
return parallelThreadSize;
|
||||
}
|
||||
|
||||
public void setParallelThreadSize(Integer parallelThreadSize) {
|
||||
this.parallelThreadSize = parallelThreadSize;
|
||||
}
|
||||
|
||||
public Integer getFetchTimeout() {
|
||||
return fetchTimeout;
|
||||
}
|
||||
|
||||
public void setFetchTimeout(Integer fetchTimeout) {
|
||||
this.fetchTimeout = fetchTimeout;
|
||||
}
|
||||
|
||||
public Integer getBatchSize() {
|
||||
return batchSize;
|
||||
}
|
||||
|
||||
public void setBatchSize(Integer batchSize) {
|
||||
this.batchSize = batchSize;
|
||||
}
|
||||
|
||||
public String getAccessChannel() {
|
||||
return accessChannel;
|
||||
}
|
||||
|
||||
public void setAccessChannel(String accessChannel) {
|
||||
this.accessChannel = accessChannel;
|
||||
}
|
||||
|
||||
public String getAliyunAccessKey() {
|
||||
return aliyunAccessKey;
|
||||
}
|
||||
|
||||
public void setAliyunAccessKey(String aliyunAccessKey) {
|
||||
this.aliyunAccessKey = aliyunAccessKey;
|
||||
}
|
||||
|
||||
public String getAliyunSecretKey() {
|
||||
return aliyunSecretKey;
|
||||
}
|
||||
|
||||
public void setAliyunSecretKey(String aliyunSecretKey) {
|
||||
this.aliyunSecretKey = aliyunSecretKey;
|
||||
}
|
||||
|
||||
public int getAliyunUid() {
|
||||
return aliyunUid;
|
||||
}
|
||||
|
||||
public void setAliyunUid(int aliyunUid) {
|
||||
this.aliyunUid = aliyunUid;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.alibaba.otter.canal.connector.core.consumer;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class CommonMessage implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 2611556444074013268L;
|
||||
|
||||
private String database; // 数据库或schema
|
||||
private String table; // 表名
|
||||
private List<String> pkNames;
|
||||
private Boolean isDdl;
|
||||
private String type; // 类型: INSERT UPDATE DELETE
|
||||
// binlog executeTime
|
||||
private Long es; // 执行耗时
|
||||
// dml build timeStamp
|
||||
private Long ts; // 同步时间
|
||||
private String sql; // 执行的sql, dml sql为空
|
||||
private List<Map<String, Object>> data; // 数据列表
|
||||
private List<Map<String, Object>> old; // 旧数据列表, 用于update, size和data的size一一对应
|
||||
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public String getTable() {
|
||||
return table;
|
||||
}
|
||||
|
||||
public void setTable(String table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
public List<String> getPkNames() {
|
||||
return pkNames;
|
||||
}
|
||||
|
||||
public void setPkNames(List<String> pkNames) {
|
||||
this.pkNames = pkNames;
|
||||
}
|
||||
|
||||
public Boolean getIsDdl() {
|
||||
return isDdl;
|
||||
}
|
||||
|
||||
public void setIsDdl(Boolean isDdl) {
|
||||
this.isDdl = isDdl;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public Long getTs() {
|
||||
return ts;
|
||||
}
|
||||
|
||||
public void setTs(Long ts) {
|
||||
this.ts = ts;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(List<Map<String, Object>> data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> getOld() {
|
||||
return old;
|
||||
}
|
||||
|
||||
public void setOld(List<Map<String, Object>> old) {
|
||||
this.old = old;
|
||||
}
|
||||
|
||||
public Long getEs() {
|
||||
return es;
|
||||
}
|
||||
|
||||
public void setEs(Long es) {
|
||||
this.es = es;
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
database = null;
|
||||
table = null;
|
||||
type = null;
|
||||
ts = null;
|
||||
es = null;
|
||||
data = null;
|
||||
old = null;
|
||||
sql = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "CommonMessage{" +
|
||||
"database='" + database + '\'' +
|
||||
", table='" + table + '\'' +
|
||||
", pkNames=" + pkNames +
|
||||
", isDdl=" + isDdl +
|
||||
", type='" + type + '\'' +
|
||||
", es=" + es +
|
||||
", ts=" + ts +
|
||||
", sql='" + sql + '\'' +
|
||||
", data=" + data +
|
||||
", old=" + old +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package com.alibaba.otter.canal.connector.core.filter;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.googlecode.aviator.AviatorEvaluator;
|
||||
import com.googlecode.aviator.Expression;
|
||||
|
||||
/**
|
||||
* 基于aviater进行tableName正则匹配的过滤算法
|
||||
*
|
||||
* @author jianghang 2012-7-20 下午06:01:34
|
||||
*/
|
||||
public class AviaterRegexFilter {
|
||||
|
||||
private static final String SPLIT = ",";
|
||||
private static final String PATTERN_SPLIT = "|";
|
||||
private static final String FILTER_EXPRESSION = "regex(pattern,target)";
|
||||
private static final RegexFunction regexFunction = new RegexFunction();
|
||||
private final Expression exp = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
|
||||
static {
|
||||
AviatorEvaluator.addFunction(regexFunction);
|
||||
}
|
||||
|
||||
private static final Comparator<String> COMPARATOR = new StringComparator();
|
||||
|
||||
final private String pattern;
|
||||
final private boolean defaultEmptyValue;
|
||||
|
||||
public AviaterRegexFilter(String pattern){
|
||||
this(pattern, true);
|
||||
}
|
||||
|
||||
public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
|
||||
this.defaultEmptyValue = defaultEmptyValue;
|
||||
List<String> list = null;
|
||||
if (StringUtils.isEmpty(pattern)) {
|
||||
list = new ArrayList<String>();
|
||||
} else {
|
||||
String[] ss = StringUtils.split(pattern, SPLIT);
|
||||
list = Arrays.asList(ss);
|
||||
}
|
||||
|
||||
// 对pattern按照从长到短的排序
|
||||
// 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
|
||||
// 的长度不一样
|
||||
Collections.sort(list, COMPARATOR);
|
||||
// 对pattern进行头尾完全匹配
|
||||
list = completionPattern(list);
|
||||
this.pattern = StringUtils.join(list, PATTERN_SPLIT);
|
||||
}
|
||||
|
||||
public boolean filter(String filtered) {
|
||||
if (StringUtils.isEmpty(pattern)) {
|
||||
return defaultEmptyValue;
|
||||
}
|
||||
|
||||
if (StringUtils.isEmpty(filtered)) {
|
||||
return defaultEmptyValue;
|
||||
}
|
||||
|
||||
Map<String, Object> env = new HashMap<String, Object>();
|
||||
env.put("pattern", pattern);
|
||||
env.put("target", filtered.toLowerCase());
|
||||
return (Boolean) exp.execute(env);
|
||||
}
|
||||
|
||||
/**
|
||||
* 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
|
||||
*
|
||||
* <pre>
|
||||
* foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
|
||||
* </pre>
|
||||
*
|
||||
* 因此此类对正则表达式进行了从长到短的排序
|
||||
*
|
||||
* @author zebin.xuzb 2012-10-22 下午2:02:26
|
||||
* @version 1.0.0
|
||||
*/
|
||||
private static class StringComparator implements Comparator<String> {
|
||||
|
||||
@Override
|
||||
public int compare(String str1, String str2) {
|
||||
if (str1.length() > str2.length()) {
|
||||
return -1;
|
||||
} else if (str1.length() < str2.length()) {
|
||||
return 1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
|
||||
*
|
||||
* <pre>
|
||||
* foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
|
||||
* </pre>
|
||||
*
|
||||
* 因此此类对正则表达式进行头尾完全匹配
|
||||
*
|
||||
* @author simon
|
||||
* @version 1.0.0
|
||||
*/
|
||||
|
||||
private List<String> completionPattern(List<String> patterns) {
|
||||
List<String> result = new ArrayList<String>();
|
||||
for (String pattern : patterns) {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append("^");
|
||||
stringBuffer.append(pattern);
|
||||
stringBuffer.append("$");
|
||||
result.add(stringBuffer.toString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return pattern;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.alibaba.otter.canal.connector.core.filter;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.oro.text.regex.MalformedPatternException;
|
||||
import org.apache.oro.text.regex.Pattern;
|
||||
import org.apache.oro.text.regex.PatternCompiler;
|
||||
import org.apache.oro.text.regex.Perl5Compiler;
|
||||
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
|
||||
public class PatternUtils {
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> {
|
||||
try {
|
||||
PatternCompiler pc = new Perl5Compiler();
|
||||
return pc.compile(pattern,
|
||||
Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK);
|
||||
} catch (MalformedPatternException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
public static Pattern getPattern(String pattern) {
|
||||
return patterns.get(pattern);
|
||||
}
|
||||
|
||||
public static void clear() {
|
||||
patterns.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.alibaba.otter.canal.connector.core.filter;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.oro.text.regex.Perl5Matcher;
|
||||
|
||||
import com.googlecode.aviator.runtime.function.AbstractFunction;
|
||||
import com.googlecode.aviator.runtime.function.FunctionUtils;
|
||||
import com.googlecode.aviator.runtime.type.AviatorBoolean;
|
||||
import com.googlecode.aviator.runtime.type.AviatorObject;
|
||||
|
||||
/**
|
||||
* 提供aviator regex的代码扩展
|
||||
*
|
||||
* @author jianghang 2012-7-23 上午10:29:23
|
||||
*/
|
||||
public class RegexFunction extends AbstractFunction {
|
||||
|
||||
public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
|
||||
String pattern = FunctionUtils.getStringValue(arg1, env);
|
||||
String text = FunctionUtils.getStringValue(arg2, env);
|
||||
Perl5Matcher matcher = new Perl5Matcher();
|
||||
boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
|
||||
return AviatorBoolean.valueOf(isMatch);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return "regex";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package com.alibaba.otter.canal.connector.core.producer;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.alibaba.otter.canal.connector.core.config.MQProperties;
|
||||
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
|
||||
|
||||
/**
|
||||
* MQ producer 抽象类
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public abstract class AbstractMQProducer implements CanalMQProducer {
|
||||
|
||||
protected MQProperties mqProperties;
|
||||
|
||||
protected ThreadPoolExecutor executor;
|
||||
|
||||
@Override
|
||||
public void init(Properties properties) {
|
||||
// parse canal mq properties
|
||||
loadCanalMqProperties(properties);
|
||||
|
||||
int parallelThreadSize = mqProperties.getParallelThreadSize();
|
||||
executor = new ThreadPoolExecutor(parallelThreadSize,
|
||||
parallelThreadSize,
|
||||
0,
|
||||
TimeUnit.SECONDS,
|
||||
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
|
||||
new NamedThreadFactory("MQParallel"),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
@Override
|
||||
public MQProperties getMqProperties() {
|
||||
return this.mqProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
if (executor != null) {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化配置
|
||||
* <p>
|
||||
* canal.mq.flat.message = true <br/>
|
||||
* canal.mq.database.hash = true <br/>
|
||||
* canal.mq.filter.transaction.entry = true <br/>
|
||||
* canal.mq.parallel.thread.size = 8 <br/>
|
||||
* canal.mq.batch.size = 50 <br/>
|
||||
* canal.mq.timeout = 100 <br/>
|
||||
* canal.mq.access.channel = local <br/>
|
||||
* </p>
|
||||
*
|
||||
* @param properties 总配置对象
|
||||
*/
|
||||
private void loadCanalMqProperties(Properties properties) {
|
||||
String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
|
||||
if (!StringUtils.isEmpty(flatMessage)) {
|
||||
mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
|
||||
}
|
||||
String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
|
||||
if (!StringUtils.isEmpty(databaseHash)) {
|
||||
mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
|
||||
}
|
||||
String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
|
||||
if (!StringUtils.isEmpty(filterTranEntry)) {
|
||||
mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
|
||||
}
|
||||
String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE);
|
||||
if (!StringUtils.isEmpty(parallelThreadSize)) {
|
||||
mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize));
|
||||
}
|
||||
String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
|
||||
if (!StringUtils.isEmpty(batchSize)) {
|
||||
mqProperties.setBatchSize(Integer.parseInt(batchSize));
|
||||
}
|
||||
String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT);
|
||||
if (!StringUtils.isEmpty(timeOut)) {
|
||||
mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
|
||||
}
|
||||
String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
|
||||
if (!StringUtils.isEmpty(accessChannel)) {
|
||||
mqProperties.setAccessChannel(accessChannel);
|
||||
}
|
||||
String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
|
||||
if (!StringUtils.isEmpty(aliyunAccessKey)) {
|
||||
mqProperties.setAliyunAccessKey(aliyunAccessKey);
|
||||
}
|
||||
String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
|
||||
if (!StringUtils.isEmpty(aliyunSecretKey)) {
|
||||
mqProperties.setAliyunAccessKey(aliyunSecretKey);
|
||||
}
|
||||
String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
|
||||
if (!StringUtils.isEmpty(aliyunUid)) {
|
||||
mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.alibaba.otter.canal.connector.core.producer;
|
||||
|
||||
/**
|
||||
* MQ producer destination
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class MQDestination {
|
||||
private String canalDestination;
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
private Integer partitionsNum;
|
||||
private String partitionHash;
|
||||
private String dynamicTopic;
|
||||
|
||||
public String getCanalDestination() {
|
||||
return canalDestination;
|
||||
}
|
||||
|
||||
public void setCanalDestination(String canalDestination) {
|
||||
this.canalDestination = canalDestination;
|
||||
}
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public Integer getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(Integer partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
public Integer getPartitionsNum() {
|
||||
return partitionsNum;
|
||||
}
|
||||
|
||||
public void setPartitionsNum(Integer partitionsNum) {
|
||||
this.partitionsNum = partitionsNum;
|
||||
}
|
||||
|
||||
public String getPartitionHash() {
|
||||
return partitionHash;
|
||||
}
|
||||
|
||||
public void setPartitionHash(String partitionHash) {
|
||||
this.partitionHash = partitionHash;
|
||||
}
|
||||
|
||||
public String getDynamicTopic() {
|
||||
return dynamicTopic;
|
||||
}
|
||||
|
||||
public void setDynamicTopic(String dynamicTopic) {
|
||||
this.dynamicTopic = dynamicTopic;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,674 @@
|
||||
package com.alibaba.otter.canal.connector.core.producer;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
|
||||
import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter;
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.common.collect.MigrateMap;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.InvalidProtocolBufferException;
|
||||
|
||||
/**
|
||||
* process MQ Message utils
|
||||
*
|
||||
* @author agapple 2019年9月29日 下午12:36:26
|
||||
* @since 5.0.0
|
||||
*/
|
||||
public class MQMessageUtils {
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private static Map<String, List<PartitionData>> partitionDatas = MigrateMap
|
||||
.makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> {
|
||||
List<PartitionData> datas = Lists
|
||||
.newArrayList();
|
||||
|
||||
String[] pkHashConfigArray = StringUtils
|
||||
.split(StringUtils.replace(
|
||||
pkHashConfigs,
|
||||
",",
|
||||
";"), ";");
|
||||
// schema.table:id^name
|
||||
for (String pkHashConfig : pkHashConfigArray) {
|
||||
PartitionData data = new PartitionData();
|
||||
int i = pkHashConfig
|
||||
.lastIndexOf(":");
|
||||
if (i > 0) {
|
||||
String pkStr = pkHashConfig
|
||||
.substring(i + 1);
|
||||
if (pkStr.equalsIgnoreCase(
|
||||
"$pk$")) {
|
||||
data.hashMode.autoPkHash = true;
|
||||
} else {
|
||||
data.hashMode.pkNames = Lists
|
||||
.newArrayList(
|
||||
StringUtils
|
||||
.split(
|
||||
pkStr,
|
||||
'^'));
|
||||
}
|
||||
|
||||
pkHashConfig = pkHashConfig
|
||||
.substring(0, i);
|
||||
} else {
|
||||
data.hashMode.tableHash = true;
|
||||
}
|
||||
|
||||
if (!isWildCard(
|
||||
pkHashConfig)) {
|
||||
data.simpleName = pkHashConfig;
|
||||
} else {
|
||||
data.regexFilter = new AviaterRegexFilter(
|
||||
pkHashConfig);
|
||||
}
|
||||
datas.add(data);
|
||||
}
|
||||
|
||||
return datas;
|
||||
});
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap
|
||||
.makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() {
|
||||
|
||||
public List<DynamicTopicData> apply(String pkHashConfigs) {
|
||||
List<DynamicTopicData> datas = Lists
|
||||
.newArrayList();
|
||||
String[] dynamicTopicArray = StringUtils
|
||||
.split(StringUtils.replace(
|
||||
pkHashConfigs,
|
||||
",",
|
||||
";"), ";");
|
||||
// schema.table
|
||||
for (String dynamicTopic : dynamicTopicArray) {
|
||||
DynamicTopicData data = new DynamicTopicData();
|
||||
|
||||
if (!isWildCard(
|
||||
dynamicTopic)) {
|
||||
data.simpleName = dynamicTopic;
|
||||
} else {
|
||||
if (dynamicTopic
|
||||
.contains("\\.")) {
|
||||
data.tableRegexFilter = new AviaterRegexFilter(
|
||||
dynamicTopic);
|
||||
} else {
|
||||
data.schemaRegexFilter = new AviaterRegexFilter(
|
||||
dynamicTopic);
|
||||
}
|
||||
}
|
||||
datas.add(data);
|
||||
}
|
||||
|
||||
return datas;
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 按 schema 或者 schema+table 将 message 分配到对应topic
|
||||
*
|
||||
* @param message 原message
|
||||
* @param defaultTopic 默认topic
|
||||
* @param dynamicTopicConfigs 动态topic规则
|
||||
* @return 分隔后的message map
|
||||
*/
|
||||
public static Map<String, Message> messageTopics(Message message, String defaultTopic, String dynamicTopicConfigs) {
|
||||
List<CanalEntry.Entry> entries;
|
||||
if (message.isRaw()) {
|
||||
List<ByteString> rawEntries = message.getRawEntries();
|
||||
entries = new ArrayList<>(rawEntries.size());
|
||||
for (ByteString byteString : rawEntries) {
|
||||
CanalEntry.Entry entry;
|
||||
try {
|
||||
entry = CanalEntry.Entry.parseFrom(byteString);
|
||||
} catch (InvalidProtocolBufferException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
entries.add(entry);
|
||||
}
|
||||
} else {
|
||||
entries = message.getEntries();
|
||||
}
|
||||
Map<String, Message> messages = new HashMap<>();
|
||||
for (CanalEntry.Entry entry : entries) {
|
||||
// 如果有topic路由,则忽略begin/end事件
|
||||
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String schemaName = entry.getHeader().getSchemaName();
|
||||
String tableName = entry.getHeader().getTableName();
|
||||
|
||||
if (StringUtils.isEmpty(schemaName) || StringUtils.isEmpty(tableName)) {
|
||||
put2MapMessage(messages, message.getId(), defaultTopic, entry);
|
||||
} else {
|
||||
Set<String> topics = matchTopics(schemaName + "." + tableName, dynamicTopicConfigs);
|
||||
if (topics != null) {
|
||||
for (String topic : topics) {
|
||||
put2MapMessage(messages, message.getId(), topic, entry);
|
||||
}
|
||||
} else {
|
||||
topics = matchTopics(schemaName, dynamicTopicConfigs);
|
||||
if (topics != null) {
|
||||
for (String topic : topics) {
|
||||
put2MapMessage(messages, message.getId(), topic, entry);
|
||||
}
|
||||
} else {
|
||||
put2MapMessage(messages, message.getId(), defaultTopic, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* 多线程构造message的rowChanged对象，比如为partition/flastMessage转化等处理 </br>
|
||||
* 因为protobuf对象的序列化和反序列化是cpu密集型，串行执行会有代价
|
||||
*/
|
||||
public static EntryRowData[] buildMessageData(Message message, ThreadPoolExecutor executor) {
|
||||
ExecutorTemplate template = new ExecutorTemplate(executor);
|
||||
if (message.isRaw()) {
|
||||
List<ByteString> rawEntries = message.getRawEntries();
|
||||
final EntryRowData[] datas = new EntryRowData[rawEntries.size()];
|
||||
int i = 0;
|
||||
for (ByteString byteString : rawEntries) {
|
||||
final int index = i;
|
||||
template.submit(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
Entry entry = Entry.parseFrom(byteString);
|
||||
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
|
||||
datas[index] = new EntryRowData();
|
||||
datas[index].entry = entry;
|
||||
datas[index].rowChange = rowChange;
|
||||
} catch (InvalidProtocolBufferException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
template.waitForResult();
|
||||
return datas;
|
||||
} else {
|
||||
final EntryRowData[] datas = new EntryRowData[message.getEntries().size()];
|
||||
int i = 0;
|
||||
for (Entry entry : message.getEntries()) {
|
||||
final int index = i;
|
||||
template.submit(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
|
||||
datas[index] = new EntryRowData();
|
||||
datas[index].entry = entry;
|
||||
datas[index].rowChange = rowChange;
|
||||
} catch (InvalidProtocolBufferException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
template.waitForResult();
|
||||
return datas;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 message 分区
|
||||
*
|
||||
* @param partitionsNum 分区数
|
||||
* @param pkHashConfigs 分区库表主键正则表达式
|
||||
* @param databaseHash 是否取消根据database进行hash
|
||||
* @return 分区message数组
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs,
|
||||
boolean databaseHash) {
|
||||
if (partitionsNum == null) {
|
||||
partitionsNum = 1;
|
||||
}
|
||||
Message[] partitionMessages = new Message[partitionsNum];
|
||||
List<Entry>[] partitionEntries = new List[partitionsNum];
|
||||
for (int i = 0; i < partitionsNum; i++) {
|
||||
// 注意一下并发
|
||||
partitionEntries[i] = Collections.synchronizedList(Lists.newArrayList());
|
||||
}
|
||||
|
||||
for (EntryRowData data : datas) {
|
||||
CanalEntry.Entry entry = data.entry;
|
||||
CanalEntry.RowChange rowChange = data.rowChange;
|
||||
// 如果有分区路由,则忽略begin/end事件
|
||||
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rowChange.getIsDdl()) {
|
||||
partitionEntries[0].add(entry);
|
||||
} else {
|
||||
if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
|
||||
String database = entry.getHeader().getSchemaName();
|
||||
String table = entry.getHeader().getTableName();
|
||||
HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
|
||||
if (hashMode == null) {
|
||||
// 如果都没有匹配，发送到第一个分区
|
||||
partitionEntries[0].add(entry);
|
||||
} else if (hashMode.tableHash) {
|
||||
int hashCode = table.hashCode();
|
||||
int pkHash = Math.abs(hashCode) % partitionsNum;
|
||||
pkHash = Math.abs(pkHash);
|
||||
// tableHash not need split entry message
|
||||
partitionEntries[pkHash].add(entry);
|
||||
} else {
|
||||
// build new entry
|
||||
Entry.Builder builder = Entry.newBuilder(entry);
|
||||
RowChange.Builder rowChangeBuilder = RowChange.newBuilder(rowChange);
|
||||
|
||||
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
|
||||
int hashCode = 0;
|
||||
if (databaseHash) {
|
||||
hashCode = database.hashCode();
|
||||
}
|
||||
CanalEntry.EventType eventType = rowChange.getEventType();
|
||||
List<CanalEntry.Column> columns = null;
|
||||
if (eventType == CanalEntry.EventType.DELETE) {
|
||||
columns = rowData.getBeforeColumnsList();
|
||||
} else {
|
||||
columns = rowData.getAfterColumnsList();
|
||||
}
|
||||
|
||||
if (hashMode.autoPkHash) {
|
||||
// isEmpty use default pkNames
|
||||
for (CanalEntry.Column column : columns) {
|
||||
if (column.getIsKey()) {
|
||||
hashCode = hashCode ^ column.getValue().hashCode();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (CanalEntry.Column column : columns) {
|
||||
if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
|
||||
hashCode = hashCode ^ column.getValue().hashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int pkHash = Math.abs(hashCode) % partitionsNum;
|
||||
pkHash = Math.abs(pkHash);
|
||||
// clear rowDatas
|
||||
rowChangeBuilder.clearRowDatas();
|
||||
rowChangeBuilder.addRowDatas(rowData);
|
||||
builder.clearStoreValue();
|
||||
builder.setStoreValue(rowChangeBuilder.build().toByteString());
|
||||
partitionEntries[pkHash].add(builder.build());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 针对stmt/mixed binlog格式的query事件
|
||||
partitionEntries[0].add(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < partitionsNum; i++) {
|
||||
List<Entry> entriesTmp = partitionEntries[i];
|
||||
if (!entriesTmp.isEmpty()) {
|
||||
partitionMessages[i] = new Message(id, entriesTmp);
|
||||
}
|
||||
}
|
||||
|
||||
return partitionMessages;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将Message转换为FlatMessage
|
||||
*
|
||||
* @return FlatMessage列表
|
||||
* @author agapple 2018年12月11日 下午1:28:32
|
||||
*/
|
||||
public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
|
||||
List<FlatMessage> flatMessages = new ArrayList<>();
|
||||
for (EntryRowData entryRowData : datas) {
|
||||
CanalEntry.Entry entry = entryRowData.entry;
|
||||
CanalEntry.RowChange rowChange = entryRowData.rowChange;
|
||||
// 如果有分区路由,则忽略begin/end事件
|
||||
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// build flatMessage
|
||||
CanalEntry.EventType eventType = rowChange.getEventType();
|
||||
FlatMessage flatMessage = new FlatMessage(id);
|
||||
flatMessages.add(flatMessage);
|
||||
flatMessage.setDatabase(entry.getHeader().getSchemaName());
|
||||
flatMessage.setTable(entry.getHeader().getTableName());
|
||||
flatMessage.setIsDdl(rowChange.getIsDdl());
|
||||
flatMessage.setType(eventType.toString());
|
||||
flatMessage.setEs(entry.getHeader().getExecuteTime());
|
||||
flatMessage.setTs(System.currentTimeMillis());
|
||||
flatMessage.setSql(rowChange.getSql());
|
||||
|
||||
if (!rowChange.getIsDdl()) {
|
||||
Map<String, Integer> sqlType = new LinkedHashMap<>();
|
||||
Map<String, String> mysqlType = new LinkedHashMap<>();
|
||||
List<Map<String, String>> data = new ArrayList<>();
|
||||
List<Map<String, String>> old = new ArrayList<>();
|
||||
|
||||
Set<String> updateSet = new HashSet<>();
|
||||
boolean hasInitPkNames = false;
|
||||
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
|
||||
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
|
||||
&& eventType != CanalEntry.EventType.DELETE) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Map<String, String> row = new LinkedHashMap<>();
|
||||
List<CanalEntry.Column> columns;
|
||||
|
||||
if (eventType == CanalEntry.EventType.DELETE) {
|
||||
columns = rowData.getBeforeColumnsList();
|
||||
} else {
|
||||
columns = rowData.getAfterColumnsList();
|
||||
}
|
||||
|
||||
for (CanalEntry.Column column : columns) {
|
||||
if (!hasInitPkNames && column.getIsKey()) {
|
||||
flatMessage.addPkName(column.getName());
|
||||
}
|
||||
sqlType.put(column.getName(), column.getSqlType());
|
||||
mysqlType.put(column.getName(), column.getMysqlType());
|
||||
if (column.getIsNull()) {
|
||||
row.put(column.getName(), null);
|
||||
} else {
|
||||
row.put(column.getName(), column.getValue());
|
||||
}
|
||||
// 获取update为true的字段
|
||||
if (column.getUpdated()) {
|
||||
updateSet.add(column.getName());
|
||||
}
|
||||
}
|
||||
|
||||
hasInitPkNames = true;
|
||||
if (!row.isEmpty()) {
|
||||
data.add(row);
|
||||
}
|
||||
|
||||
if (eventType == CanalEntry.EventType.UPDATE) {
|
||||
Map<String, String> rowOld = new LinkedHashMap<>();
|
||||
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
|
||||
if (updateSet.contains(column.getName())) {
|
||||
if (column.getIsNull()) {
|
||||
rowOld.put(column.getName(), null);
|
||||
} else {
|
||||
rowOld.put(column.getName(), column.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
// update操作将记录修改前的值
|
||||
if (!rowOld.isEmpty()) {
|
||||
old.add(rowOld);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!sqlType.isEmpty()) {
|
||||
flatMessage.setSqlType(sqlType);
|
||||
}
|
||||
if (!mysqlType.isEmpty()) {
|
||||
flatMessage.setMysqlType(mysqlType);
|
||||
}
|
||||
if (!data.isEmpty()) {
|
||||
flatMessage.setData(data);
|
||||
}
|
||||
if (!old.isEmpty()) {
|
||||
flatMessage.setOld(old);
|
||||
}
|
||||
}
|
||||
}
|
||||
return flatMessages;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将FlatMessage按指定的字段值hash拆分
|
||||
*
|
||||
* @param flatMessage flatMessage
|
||||
* @param partitionsNum 分区数量
|
||||
* @param pkHashConfigs hash映射
|
||||
* @param databaseHash 是否取消根据database进行hash
|
||||
* @return 拆分后的flatMessage数组
|
||||
*/
|
||||
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum, String pkHashConfigs,
|
||||
boolean databaseHash) {
|
||||
if (partitionsNum == null) {
|
||||
partitionsNum = 1;
|
||||
}
|
||||
FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
|
||||
|
||||
if (flatMessage.getIsDdl()) {
|
||||
partitionMessages[0] = flatMessage;
|
||||
} else {
|
||||
if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
|
||||
String database = flatMessage.getDatabase();
|
||||
String table = flatMessage.getTable();
|
||||
HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
|
||||
if (hashMode == null) {
|
||||
// 如果都没有匹配，发送到第一个分区
|
||||
partitionMessages[0] = flatMessage;
|
||||
} else if (hashMode.tableHash) {
|
||||
int hashCode = table.hashCode();
|
||||
int pkHash = Math.abs(hashCode) % partitionsNum;
|
||||
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
|
||||
pkHash = Math.abs(pkHash);
|
||||
partitionMessages[pkHash] = flatMessage;
|
||||
} else {
|
||||
List<String> pkNames = hashMode.pkNames;
|
||||
if (hashMode.autoPkHash) {
|
||||
pkNames = flatMessage.getPkNames();
|
||||
}
|
||||
|
||||
int idx = 0;
|
||||
for (Map<String, String> row : flatMessage.getData()) {
|
||||
int hashCode = 0;
|
||||
if (databaseHash) {
|
||||
hashCode = database.hashCode();
|
||||
}
|
||||
if (pkNames != null) {
|
||||
for (String pkName : pkNames) {
|
||||
String value = row.get(pkName);
|
||||
if (value == null) {
|
||||
value = "";
|
||||
}
|
||||
hashCode = hashCode ^ value.hashCode();
|
||||
}
|
||||
}
|
||||
|
||||
int pkHash = Math.abs(hashCode) % partitionsNum;
|
||||
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
|
||||
pkHash = Math.abs(pkHash);
|
||||
|
||||
FlatMessage flatMessageTmp = partitionMessages[pkHash];
|
||||
if (flatMessageTmp == null) {
|
||||
flatMessageTmp = new FlatMessage(flatMessage.getId());
|
||||
partitionMessages[pkHash] = flatMessageTmp;
|
||||
flatMessageTmp.setDatabase(flatMessage.getDatabase());
|
||||
flatMessageTmp.setTable(flatMessage.getTable());
|
||||
flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
|
||||
flatMessageTmp.setType(flatMessage.getType());
|
||||
flatMessageTmp.setSql(flatMessage.getSql());
|
||||
flatMessageTmp.setSqlType(flatMessage.getSqlType());
|
||||
flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
|
||||
flatMessageTmp.setEs(flatMessage.getEs());
|
||||
flatMessageTmp.setTs(flatMessage.getTs());
|
||||
flatMessageTmp.setPkNames(flatMessage.getPkNames());
|
||||
}
|
||||
List<Map<String, String>> data = flatMessageTmp.getData();
|
||||
if (data == null) {
|
||||
data = new ArrayList<>();
|
||||
flatMessageTmp.setData(data);
|
||||
}
|
||||
data.add(row);
|
||||
if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
|
||||
List<Map<String, String>> old = flatMessageTmp.getOld();
|
||||
if (old == null) {
|
||||
old = new ArrayList<>();
|
||||
flatMessageTmp.setOld(old);
|
||||
}
|
||||
old.add(flatMessage.getOld().get(idx));
|
||||
}
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 针对stmt/mixed binlog格式的query事件
|
||||
partitionMessages[0] = flatMessage;
|
||||
}
|
||||
}
|
||||
return partitionMessages;
|
||||
}
|
||||
|
||||
/**
|
||||
* match return List , not match return null
|
||||
*/
|
||||
public static HashMode getPartitionHashColumns(String name, String pkHashConfigs) {
|
||||
if (StringUtils.isEmpty(pkHashConfigs)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<PartitionData> datas = partitionDatas.get(pkHashConfigs);
|
||||
for (PartitionData data : datas) {
|
||||
if (data.simpleName != null) {
|
||||
if (data.simpleName.equalsIgnoreCase(name)) {
|
||||
return data.hashMode;
|
||||
}
|
||||
} else {
|
||||
if (data.regexFilter.filter(name)) {
|
||||
return data.hashMode;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Set<String> matchTopics(String name, String dynamicTopicConfigs) {
|
||||
String[] router = StringUtils.split(StringUtils.replace(dynamicTopicConfigs, ",", ";"), ";");
|
||||
Set<String> topics = new HashSet<>();
|
||||
for (String item : router) {
|
||||
int i = item.indexOf(":");
|
||||
if (i > -1) {
|
||||
String topic = item.substring(0, i).trim();
|
||||
String topicConfigs = item.substring(i + 1).trim();
|
||||
if (matchDynamicTopic(name, topicConfigs)) {
|
||||
topics.add(topic);
|
||||
// 匹配了一个就退出
|
||||
break;
|
||||
}
|
||||
} else if (matchDynamicTopic(name, item)) {
|
||||
// 匹配了一个就退出
|
||||
topics.add(name.toLowerCase());
|
||||
break;
|
||||
}
|
||||
}
|
||||
return topics.isEmpty() ? null : topics;
|
||||
}
|
||||
|
||||
public static boolean matchDynamicTopic(String name, String dynamicTopicConfigs) {
|
||||
if (StringUtils.isEmpty(dynamicTopicConfigs)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean res = false;
|
||||
List<DynamicTopicData> datas = dynamicTopicDatas.get(dynamicTopicConfigs);
|
||||
for (DynamicTopicData data : datas) {
|
||||
if (data.simpleName != null) {
|
||||
if (data.simpleName.equalsIgnoreCase(name)) {
|
||||
res = true;
|
||||
break;
|
||||
}
|
||||
} else if (name.contains(".")) {
|
||||
if (data.tableRegexFilter != null && data.tableRegexFilter.filter(name)) {
|
||||
res = true;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (data.schemaRegexFilter != null && data.schemaRegexFilter.filter(name)) {
|
||||
res = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
public static boolean checkPkNamesHasContain(List<String> pkNames, String name) {
|
||||
for (String pkName : pkNames) {
|
||||
if (pkName.equalsIgnoreCase(name)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean isWildCard(String value) {
|
||||
// not contaiins '.' ?
|
||||
return StringUtils.containsAny(value,
|
||||
new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' });
|
||||
}
|
||||
|
||||
private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,
|
||||
CanalEntry.Entry entry) {
|
||||
Message message = messageMap.get(topicName);
|
||||
if (message == null) {
|
||||
message = new Message(messageId, new ArrayList<CanalEntry.Entry>());
|
||||
messageMap.put(topicName, message);
|
||||
}
|
||||
message.getEntries().add(entry);
|
||||
}
|
||||
|
||||
public static class PartitionData {
|
||||
|
||||
public String simpleName;
|
||||
public AviaterRegexFilter regexFilter;
|
||||
public HashMode hashMode = new HashMode();
|
||||
}
|
||||
|
||||
public static class HashMode {
|
||||
|
||||
public boolean autoPkHash = false;
|
||||
public boolean tableHash = false;
|
||||
public List<String> pkNames = Lists.newArrayList();
|
||||
}
|
||||
|
||||
public static class DynamicTopicData {
|
||||
|
||||
public String simpleName;
|
||||
public AviaterRegexFilter schemaRegexFilter;
|
||||
public AviaterRegexFilter tableRegexFilter;
|
||||
}
|
||||
|
||||
public static class EntryRowData {
|
||||
|
||||
public Entry entry;
|
||||
public RowChange rowChange;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.alibaba.otter.canal.connector.core.spi;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import com.alibaba.otter.canal.connector.core.config.MQProperties;
|
||||
import com.alibaba.otter.canal.connector.core.util.Callback;
|
||||
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
* MQ producer SPI 接口
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SPI("kafka")
|
||||
public interface CanalMQProducer {
|
||||
|
||||
/**
|
||||
* Init producer.
|
||||
*/
|
||||
void init(Properties properties);
|
||||
|
||||
/**
|
||||
* Get base mq properties
|
||||
*
|
||||
* @return MQProperties
|
||||
*/
|
||||
MQProperties getMqProperties();
|
||||
|
||||
/**
|
||||
* Send canal message to related topic
|
||||
*
|
||||
* @param canalDestination canal mq destination
|
||||
* @param message canal message
|
||||
*/
|
||||
void send(MQDestination canalDestination, Message message, Callback callback);
|
||||
|
||||
/**
|
||||
* Stop MQ producer service
|
||||
*/
|
||||
void stop();
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.alibaba.otter.canal.connector.core.spi;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
|
||||
|
||||
/**
|
||||
* Canal/MQ consumer SPI 接口
|
||||
*
|
||||
* @author rewerma @ 2020-02-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SPI("kafka")
|
||||
public interface CanalMsgConsumer {
|
||||
|
||||
/**
|
||||
* 初始化
|
||||
*
|
||||
* @param properties consumer properties
|
||||
* @param topic topic/destination
|
||||
* @param groupId mq group id
|
||||
*/
|
||||
void init(Properties properties, String topic, String groupId);
|
||||
|
||||
/**
|
||||
* 连接Canal/MQ
|
||||
*/
|
||||
void connect();
|
||||
|
||||
/**
|
||||
* 批量拉取数据
|
||||
*
|
||||
* @param timeout 超时时间
|
||||
* @param unit 时间单位
|
||||
* @return Message列表
|
||||
*/
|
||||
List<CommonMessage> getMessage(Long timeout, TimeUnit unit);
|
||||
|
||||
/**
|
||||
* 提交
|
||||
*/
|
||||
void ack();
|
||||
|
||||
/**
|
||||
* 回滚
|
||||
*/
|
||||
void rollback();
|
||||
|
||||
/**
|
||||
* 断开连接
|
||||
*/
|
||||
void disconnect();
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
package com.alibaba.otter.canal.connector.core.spi;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* SPI 类加载器
|
||||
*
|
||||
* @author rewerma 2018-8-19 下午11:30:49
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class ExtensionLoader<T> {
|
||||
|
||||
private static final Logger logger = LoggerFactory
|
||||
.getLogger(ExtensionLoader.class);
|
||||
|
||||
private static final String SERVICES_DIRECTORY = "META-INF/services/";
|
||||
|
||||
private static final String CANAL_DIRECTORY = "META-INF/canal/";
|
||||
|
||||
private static final String DEFAULT_CLASSLOADER_POLICY = "internal";
|
||||
|
||||
private static final Pattern NAME_SEPARATOR = Pattern
|
||||
.compile("\\s*[,]+\\s*");
|
||||
|
||||
private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS = new ConcurrentHashMap<>();
|
||||
|
||||
private static final ConcurrentMap<Class<?>, Object> EXTENSION_INSTANCES = new ConcurrentHashMap<>();
|
||||
|
||||
private static final ConcurrentMap<String, Object> EXTENSION_KEY_INSTANCE = new ConcurrentHashMap<>();
|
||||
|
||||
private final Class<?> type;
|
||||
|
||||
private final String classLoaderPolicy;
|
||||
|
||||
private final ConcurrentMap<Class<?>, String> cachedNames = new ConcurrentHashMap<>();
|
||||
|
||||
private final Holder<Map<String, Class<?>>> cachedClasses = new Holder<>();
|
||||
|
||||
private final ConcurrentMap<String, Holder<Object>> cachedInstances = new ConcurrentHashMap<>();
|
||||
|
||||
private String cachedDefaultName;
|
||||
|
||||
private ConcurrentHashMap<String, IllegalStateException> exceptions = new ConcurrentHashMap<>();
|
||||
|
||||
private static <T> boolean withExtensionAnnotation(Class<T> type) {
|
||||
return type.isAnnotationPresent(SPI.class);
|
||||
}
|
||||
|
||||
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
|
||||
return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
|
||||
if (type == null) throw new IllegalArgumentException("Extension type == null");
|
||||
if (!type.isInterface()) {
|
||||
throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
|
||||
}
|
||||
if (!withExtensionAnnotation(type)) {
|
||||
throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
|
||||
+ SPI.class.getSimpleName() + " Annotation!");
|
||||
}
|
||||
|
||||
ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
|
||||
if (loader == null) {
|
||||
EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
|
||||
loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
|
||||
}
|
||||
return loader;
|
||||
}
|
||||
|
||||
public ExtensionLoader(Class<?> type){
|
||||
this.type = type;
|
||||
this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY;
|
||||
}
|
||||
|
||||
public ExtensionLoader(Class<?> type, String classLoaderPolicy){
|
||||
this.type = type;
|
||||
this.classLoaderPolicy = classLoaderPolicy;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回指定名字的扩展
|
||||
*
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public T getExtension(String name, String spiDir, String standbyDir) {
|
||||
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
|
||||
if ("true".equals(name)) {
|
||||
return getDefaultExtension(spiDir, standbyDir);
|
||||
}
|
||||
Holder<Object> holder = cachedInstances.get(name);
|
||||
if (holder == null) {
|
||||
cachedInstances.putIfAbsent(name, new Holder<>());
|
||||
holder = cachedInstances.get(name);
|
||||
}
|
||||
Object instance = holder.get();
|
||||
if (instance == null) {
|
||||
synchronized (holder) {
|
||||
instance = holder.get();
|
||||
if (instance == null) {
|
||||
instance = createExtension(name, spiDir, standbyDir);
|
||||
holder.set(instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (T) instance;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public T getExtension(String name, String key, String spiDir, String standbyDir) {
|
||||
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
|
||||
if ("true".equals(name)) {
|
||||
return getDefaultExtension(spiDir, standbyDir);
|
||||
}
|
||||
String extKey = name + "-" + StringUtils.trimToEmpty(key);
|
||||
Holder<Object> holder = cachedInstances.get(extKey);
|
||||
if (holder == null) {
|
||||
cachedInstances.putIfAbsent(extKey, new Holder<>());
|
||||
holder = cachedInstances.get(extKey);
|
||||
}
|
||||
Object instance = holder.get();
|
||||
if (instance == null) {
|
||||
synchronized (holder) {
|
||||
instance = holder.get();
|
||||
if (instance == null) {
|
||||
instance = createExtension(name, key, spiDir, standbyDir);
|
||||
holder.set(instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (T) instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回缺省的扩展，如果没有设置则返回<code>null</code>
|
||||
*/
|
||||
public T getDefaultExtension(String spiDir, String standbyDir) {
|
||||
getExtensionClasses(spiDir, standbyDir);
|
||||
if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
|
||||
return null;
|
||||
}
|
||||
return getExtension(cachedDefaultName, spiDir, standbyDir);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public T createExtension(String name, String spiDir, String standbyDir) {
|
||||
Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
|
||||
if (clazz == null) {
|
||||
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
|
||||
+ ") could not be instantiated: class could not be found");
|
||||
}
|
||||
try {
|
||||
T instance = (T) EXTENSION_INSTANCES.get(clazz);
|
||||
if (instance == null) {
|
||||
EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
|
||||
instance = (T) EXTENSION_INSTANCES.get(clazz);
|
||||
}
|
||||
return instance;
|
||||
} catch (Throwable t) {
|
||||
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
|
||||
+ ") could not be instantiated: " + t.getMessage(),
|
||||
t);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private T createExtension(String name, String key, String spiDir, String standbyDir) {
|
||||
Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
|
||||
if (clazz == null) {
|
||||
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
|
||||
+ ") could not be instantiated: class could not be found");
|
||||
}
|
||||
try {
|
||||
T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
|
||||
if (instance == null) {
|
||||
EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance());
|
||||
instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
|
||||
}
|
||||
return instance;
|
||||
} catch (Throwable t) {
|
||||
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
|
||||
+ ") could not be instantiated: " + t.getMessage(),
|
||||
t);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Class<?>> getExtensionClasses(String spiDir, String standbyDir) {
|
||||
Map<String, Class<?>> classes = cachedClasses.get();
|
||||
if (classes == null) {
|
||||
synchronized (cachedClasses) {
|
||||
classes = cachedClasses.get();
|
||||
if (classes == null) {
|
||||
classes = loadExtensionClasses(spiDir, standbyDir);
|
||||
cachedClasses.set(classes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return classes;
|
||||
}
|
||||
|
||||
private String getJarDirectoryPath() {
|
||||
URL url = Thread.currentThread().getContextClassLoader().getResource("");
|
||||
String dirtyPath;
|
||||
if (url != null) {
|
||||
dirtyPath = url.toString();
|
||||
} else {
|
||||
File file = new File("");
|
||||
dirtyPath = file.getAbsolutePath();
|
||||
}
|
||||
String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
|
||||
// file:/ and
|
||||
// everything
|
||||
// before it
|
||||
jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything
|
||||
// after .jar, if .jar
|
||||
// exists in dirtyPath
|
||||
jarPath = jarPath.replaceAll("%20", " "); // necessary if path has
|
||||
// spaces within
|
||||
if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run
|
||||
// the app using Spring Tools Suit play
|
||||
// button.
|
||||
jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
|
||||
}
|
||||
Path path = Paths.get(jarPath).getParent(); // Paths - from java 8
|
||||
if (path != null) {
|
||||
return path.toString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Map<String, Class<?>> loadExtensionClasses(String spiDir, String standbyDir) {
|
||||
final SPI defaultAnnotation = type.getAnnotation(SPI.class);
|
||||
if (defaultAnnotation != null) {
|
||||
String value = defaultAnnotation.value();
|
||||
if ((value = value.trim()).length() > 0) {
|
||||
String[] names = NAME_SEPARATOR.split(value);
|
||||
if (names.length > 1) {
|
||||
throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
|
||||
+ ": " + Arrays.toString(names));
|
||||
}
|
||||
if (names.length == 1) cachedDefaultName = names[0];
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Class<?>> extensionClasses = new HashMap<>();
|
||||
|
||||
if (spiDir != null && standbyDir != null) {
|
||||
// 1. plugin folder，customized extension classLoader （jar_dir/plugin）
|
||||
String dir = File.separator + this.getJarDirectoryPath() + spiDir; // + "plugin";
|
||||
|
||||
File externalLibDir = new File(dir);
|
||||
if (!externalLibDir.exists()) {
|
||||
externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir);
|
||||
}
|
||||
logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath());
|
||||
if (externalLibDir.exists()) {
|
||||
File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar"));
|
||||
if (files != null) {
|
||||
for (File f : files) {
|
||||
URL url;
|
||||
try {
|
||||
url = f.toURI().toURL();
|
||||
} catch (MalformedURLException e) {
|
||||
throw new RuntimeException("load extension jar failed!", e);
|
||||
}
|
||||
|
||||
ClassLoader parent = Thread.currentThread().getContextClassLoader();
|
||||
URLClassLoader localClassLoader;
|
||||
if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
|
||||
|| DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
|
||||
localClassLoader = new URLClassExtensionLoader(new URL[] { url });
|
||||
} else {
|
||||
localClassLoader = new URLClassLoader(new URL[] { url }, parent);
|
||||
}
|
||||
|
||||
loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
|
||||
loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. load inner extension class with default classLoader
|
||||
ClassLoader classLoader = findClassLoader();
|
||||
loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
|
||||
loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
|
||||
|
||||
return extensionClasses;
|
||||
}
|
||||
|
||||
private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
|
||||
String fileName = dir + type.getName();
|
||||
try {
|
||||
Enumeration<URL> urls;
|
||||
if (classLoader != null) {
|
||||
urls = classLoader.getResources(fileName);
|
||||
} else {
|
||||
urls = ClassLoader.getSystemResources(fileName);
|
||||
}
|
||||
if (urls != null) {
|
||||
while (urls.hasMoreElements()) {
|
||||
URL url = urls.nextElement();
|
||||
try {
|
||||
BufferedReader reader = null;
|
||||
try {
|
||||
reader = new BufferedReader(
|
||||
new InputStreamReader(url.openStream(), StandardCharsets.UTF_8));
|
||||
String line = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
final int ci = line.indexOf('#');
|
||||
if (ci >= 0) line = line.substring(0, ci);
|
||||
line = line.trim();
|
||||
if (line.length() > 0) {
|
||||
try {
|
||||
String name = null;
|
||||
int i = line.indexOf('=');
|
||||
if (i > 0) {
|
||||
name = line.substring(0, i).trim();
|
||||
line = line.substring(i + 1).trim();
|
||||
}
|
||||
if (line.length() > 0) {
|
||||
Class<?> clazz = classLoader.loadClass(line);
|
||||
// Class<?> clazz =
|
||||
// Class.forName(line, true,
|
||||
// classLoader);
|
||||
if (!type.isAssignableFrom(clazz)) {
|
||||
throw new IllegalStateException(
|
||||
"Error when load extension class(interface: " + type
|
||||
+ ", class line: " + clazz.getName()
|
||||
+ "), class " + clazz.getName()
|
||||
+ "is not subtype of interface.");
|
||||
} else {
|
||||
try {
|
||||
clazz.getConstructor(type);
|
||||
} catch (NoSuchMethodException e) {
|
||||
clazz.getConstructor();
|
||||
String[] names = NAME_SEPARATOR.split(name);
|
||||
if (names != null && names.length > 0) {
|
||||
for (String n : names) {
|
||||
if (!cachedNames.containsKey(clazz)) {
|
||||
cachedNames.put(clazz, n);
|
||||
}
|
||||
Class<?> c = extensionClasses.get(n);
|
||||
if (c == null) {
|
||||
extensionClasses.put(n, clazz);
|
||||
} else if (c != clazz) {
|
||||
cachedNames.remove(clazz);
|
||||
throw new IllegalStateException(
|
||||
"Duplicate extension " + type.getName() + " name "
|
||||
+ n + " on "
|
||||
+ c.getName() + " and "
|
||||
+ clazz.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
IllegalStateException e = new IllegalStateException(
|
||||
"Failed to load extension class(interface: " + type + ", class line: "
|
||||
+ line + ") in " + url
|
||||
+ ", cause: "
|
||||
+ t.getMessage(),
|
||||
t);
|
||||
exceptions.put(line, e);
|
||||
}
|
||||
}
|
||||
} // end of while read lines
|
||||
} finally {
|
||||
if (reader != null) {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
|
||||
+ ") in " + url,
|
||||
t);
|
||||
}
|
||||
} // end of while urls
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.error(
|
||||
"Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
|
||||
t);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private static ClassLoader findClassLoader() {
|
||||
return ExtensionLoader.class.getClassLoader();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return this.getClass().getName() + "[" + type.getName() + "]";
|
||||
}
|
||||
|
||||
private static class Holder<T> {
|
||||
|
||||
private volatile T value;
|
||||
|
||||
private void set(T value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
private T get() {
|
||||
return value;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.alibaba.otter.canal.connector.core.spi;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
/**
|
||||
* SPI装载器注解
|
||||
*
|
||||
* @author rewerma @ 2018-10-20
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@Documented
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ ElementType.TYPE })
|
||||
public @interface SPI {
|
||||
|
||||
// Default SPI name
|
||||
String value() default "";
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package com.alibaba.otter.canal.connector.core.spi;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.Enumeration;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
public class URLClassExtensionLoader extends URLClassLoader {
|
||||
public URLClassExtensionLoader(URL[] urls) {
|
||||
super(urls);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<?> loadClass(String name) throws ClassNotFoundException {
|
||||
Class<?> c = findLoadedClass(name);
|
||||
if (c != null) {
|
||||
return c;
|
||||
}
|
||||
|
||||
if (name.startsWith("java.") || name.startsWith("org.slf4j.")
|
||||
|| name.startsWith("org.apache.logging")
|
||||
|| name.startsWith("org.apache.commons.logging.")) {
|
||||
// || name.startsWith("org.apache.hadoop."))
|
||||
// {
|
||||
c = super.loadClass(name);
|
||||
}
|
||||
if (c != null) return c;
|
||||
|
||||
try {
|
||||
// 先加载jar内的class，可避免jar冲突
|
||||
c = findClass(name);
|
||||
} catch (ClassNotFoundException e) {
|
||||
c = null;
|
||||
}
|
||||
if (c != null) {
|
||||
return c;
|
||||
}
|
||||
|
||||
return super.loadClass(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Enumeration<URL> getResources(String name) throws IOException {
|
||||
@SuppressWarnings("unchecked")
|
||||
Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
|
||||
|
||||
tmp[0] = findResources(name); // local class
|
||||
// path first
|
||||
// tmp[1] = super.getResources(name);
|
||||
|
||||
return new CompoundEnumeration<>(tmp);
|
||||
}
|
||||
|
||||
private static class CompoundEnumeration<E> implements Enumeration<E> {
|
||||
|
||||
private Enumeration<E>[] enums;
|
||||
private int index = 0;
|
||||
|
||||
public CompoundEnumeration(Enumeration<E>[] enums){
|
||||
this.enums = enums;
|
||||
}
|
||||
|
||||
private boolean next() {
|
||||
while (this.index < this.enums.length) {
|
||||
if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
++this.index;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean hasMoreElements() {
|
||||
return this.next();
|
||||
}
|
||||
|
||||
public E nextElement() {
|
||||
if (!this.next()) {
|
||||
throw new NoSuchElementException();
|
||||
} else {
|
||||
return this.enums[this.index].nextElement();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.alibaba.otter.canal.connector.core.util;
|
||||
|
||||
/**
|
||||
* MQ 回调类
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public interface Callback {
|
||||
void commit();
|
||||
|
||||
void rollback();
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package com.alibaba.otter.canal.connector.core.util;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.CodedOutputStream;
|
||||
import com.google.protobuf.WireFormat;
|
||||
|
||||
/**
|
||||
* Canal message 序列化工具类
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class CanalMessageSerializerUtil {
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public static byte[] serializer(Message data, boolean filterTransactionEntry) {
|
||||
try {
|
||||
if (data != null) {
|
||||
if (data.getId() != -1) {
|
||||
if (data.isRaw() && !CollectionUtils.isEmpty(data.getRawEntries())) {
|
||||
// for performance
|
||||
List<ByteString> rowEntries = data.getRawEntries();
|
||||
// message size
|
||||
int messageSize = 0;
|
||||
messageSize += CodedOutputStream.computeInt64Size(1, data.getId());
|
||||
|
||||
int dataSize = 0;
|
||||
for (int i = 0; i < rowEntries.size(); i++) {
|
||||
dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntries.get(i));
|
||||
}
|
||||
messageSize += dataSize;
|
||||
messageSize += 1 * rowEntries.size();
|
||||
// packet size
|
||||
int size = 0;
|
||||
size += CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
|
||||
size += CodedOutputStream.computeTagSize(5)
|
||||
+ CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
|
||||
// build data
|
||||
byte[] body = new byte[size];
|
||||
CodedOutputStream output = CodedOutputStream.newInstance(body);
|
||||
output.writeEnum(3, PacketType.MESSAGES.getNumber());
|
||||
|
||||
output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
|
||||
output.writeRawVarint32(messageSize);
|
||||
// message
|
||||
output.writeInt64(1, data.getId());
|
||||
for (int i = 0; i < rowEntries.size(); i++) {
|
||||
output.writeBytes(2, rowEntries.get(i));
|
||||
}
|
||||
output.checkNoSpaceLeft();
|
||||
return body;
|
||||
} else if (!CollectionUtils.isEmpty(data.getEntries())) {
|
||||
// mq模式只会走到非rowEntry模式
|
||||
CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
|
||||
for (CanalEntry.Entry entry : data.getEntries()) {
|
||||
if (filterTransactionEntry
|
||||
&& (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
messageBuilder.addMessages(entry.toByteString());
|
||||
}
|
||||
|
||||
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
|
||||
packetBuilder.setType(PacketType.MESSAGES);
|
||||
packetBuilder.setVersion(1);
|
||||
packetBuilder.setBody(messageBuilder.build().toByteString());
|
||||
return packetBuilder.build().toByteArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Error when serializing message to byte[] ");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static Message deserializer(byte[] data) {
|
||||
return deserializer(data, false);
|
||||
}
|
||||
|
||||
public static Message deserializer(byte[] data, boolean lazyParseEntry) {
|
||||
try {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
|
||||
switch (p.getType()) {
|
||||
case MESSAGES: {
|
||||
if (!p.getCompression().equals(CanalPacket.Compression.NONE)
|
||||
&& !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
|
||||
throw new CanalClientException("compression is not supported in this connector");
|
||||
}
|
||||
|
||||
CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
|
||||
Message result = new Message(messages.getBatchId());
|
||||
if (lazyParseEntry) {
|
||||
// byteString
|
||||
result.setRawEntries(messages.getMessagesList());
|
||||
result.setRaw(true);
|
||||
} else {
|
||||
for (ByteString byteString : messages.getMessagesList()) {
|
||||
result.addEntry(CanalEntry.Entry.parseFrom(byteString));
|
||||
}
|
||||
result.setRaw(false);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case ACK: {
|
||||
CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
|
||||
throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
|
||||
}
|
||||
default: {
|
||||
throw new CanalClientException("unexpected packet type: " + p.getType());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new CanalClientException("deserializer failed", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.alibaba.otter.canal.connector.core.util;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class DateUtil {
|
||||
private static DateTimeZone dateTimeZone;
|
||||
|
||||
static {
|
||||
dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE);
|
||||
}
|
||||
|
||||
/**
|
||||
* 通用日期时间字符解析
|
||||
*
|
||||
* @param datetimeStr 日期时间字符串
|
||||
* @return Date
|
||||
*/
|
||||
public static Date parseDate(String datetimeStr) {
|
||||
if (StringUtils.isEmpty(datetimeStr)) {
|
||||
return null;
|
||||
}
|
||||
datetimeStr = datetimeStr.trim().replace('/', '-');
|
||||
if (datetimeStr.contains("-")) {
|
||||
if (datetimeStr.contains(":")) {
|
||||
datetimeStr = datetimeStr.replace(" ", "T");
|
||||
}
|
||||
} else if (datetimeStr.contains(":")) {
|
||||
datetimeStr = "T" + datetimeStr;
|
||||
} else {
|
||||
if (datetimeStr.length() == 8) {
|
||||
String year = datetimeStr.substring(0, 4);
|
||||
String month = datetimeStr.substring(4, 6);
|
||||
String day = datetimeStr.substring(6, 8);
|
||||
datetimeStr = year + "-" + month + "-" + day;
|
||||
}
|
||||
}
|
||||
|
||||
DateTime dateTime = new DateTime(datetimeStr, dateTimeZone);
|
||||
|
||||
return dateTime.toDate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package com.alibaba.otter.canal.connector.core.util;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.*;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* 类型转换工具类
|
||||
*
|
||||
* @author rewerma 2018-8-19 下午06:14:23
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class JdbcTypeUtil {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
|
||||
|
||||
public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
|
||||
if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
|
||||
return rs.getByte(columnName);
|
||||
} else {
|
||||
return rs.getObject(columnName);
|
||||
}
|
||||
}
|
||||
|
||||
public static Class<?> jdbcType2javaType(int jdbcType) {
|
||||
switch (jdbcType) {
|
||||
case Types.BIT:
|
||||
case Types.BOOLEAN:
|
||||
// return Boolean.class;
|
||||
case Types.TINYINT:
|
||||
return Byte.TYPE;
|
||||
case Types.SMALLINT:
|
||||
return Short.class;
|
||||
case Types.INTEGER:
|
||||
return Integer.class;
|
||||
case Types.BIGINT:
|
||||
return Long.class;
|
||||
case Types.DECIMAL:
|
||||
case Types.NUMERIC:
|
||||
return BigDecimal.class;
|
||||
case Types.REAL:
|
||||
return Float.class;
|
||||
case Types.FLOAT:
|
||||
case Types.DOUBLE:
|
||||
return Double.class;
|
||||
case Types.CHAR:
|
||||
case Types.VARCHAR:
|
||||
case Types.LONGVARCHAR:
|
||||
return String.class;
|
||||
case Types.BINARY:
|
||||
case Types.VARBINARY:
|
||||
case Types.LONGVARBINARY:
|
||||
case Types.BLOB:
|
||||
return byte[].class;
|
||||
case Types.DATE:
|
||||
return Date.class;
|
||||
case Types.TIME:
|
||||
return Time.class;
|
||||
case Types.TIMESTAMP:
|
||||
return Timestamp.class;
|
||||
default:
|
||||
return String.class;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean isText(String columnType) {
|
||||
return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
|
||||
|| "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
|
||||
}
|
||||
|
||||
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
|
||||
if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR
|
||||
|| sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
Object res;
|
||||
switch (sqlType) {
|
||||
case Types.INTEGER:
|
||||
res = Integer.parseInt(value);
|
||||
break;
|
||||
case Types.SMALLINT:
|
||||
res = Short.parseShort(value);
|
||||
break;
|
||||
case Types.BIT:
|
||||
case Types.TINYINT:
|
||||
res = Byte.parseByte(value);
|
||||
break;
|
||||
case Types.BIGINT:
|
||||
if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
|
||||
res = new BigInteger(value);
|
||||
} else {
|
||||
res = Long.parseLong(value);
|
||||
}
|
||||
break;
|
||||
// case Types.BIT:
|
||||
case Types.BOOLEAN:
|
||||
res = !"0".equals(value);
|
||||
break;
|
||||
case Types.DOUBLE:
|
||||
case Types.FLOAT:
|
||||
res = Double.parseDouble(value);
|
||||
break;
|
||||
case Types.REAL:
|
||||
res = Float.parseFloat(value);
|
||||
break;
|
||||
case Types.DECIMAL:
|
||||
case Types.NUMERIC:
|
||||
res = new BigDecimal(value);
|
||||
break;
|
||||
case Types.BINARY:
|
||||
case Types.VARBINARY:
|
||||
case Types.LONGVARBINARY:
|
||||
case Types.BLOB:
|
||||
res = value.getBytes("ISO-8859-1");
|
||||
break;
|
||||
case Types.DATE:
|
||||
if (!value.startsWith("0000-00-00")) {
|
||||
java.util.Date date = DateUtil.parseDate(value);
|
||||
if (date != null) {
|
||||
res = new Date(date.getTime());
|
||||
} else {
|
||||
res = null;
|
||||
}
|
||||
} else {
|
||||
res = null;
|
||||
}
|
||||
break;
|
||||
case Types.TIME: {
|
||||
java.util.Date date = DateUtil.parseDate(value);
|
||||
if (date != null) {
|
||||
res = new Time(date.getTime());
|
||||
} else {
|
||||
res = null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Types.TIMESTAMP:
|
||||
if (!value.startsWith("0000-00-00")) {
|
||||
java.util.Date date = DateUtil.parseDate(value);
|
||||
if (date != null) {
|
||||
res = new Timestamp(date.getTime());
|
||||
} else {
|
||||
res = null;
|
||||
}
|
||||
} else {
|
||||
res = null;
|
||||
}
|
||||
break;
|
||||
case Types.CLOB:
|
||||
default:
|
||||
res = value;
|
||||
break;
|
||||
}
|
||||
return res;
|
||||
} catch (Exception e) {
|
||||
logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.alibaba.otter.canal.connector.core.util;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
* Message对象解析工具类
|
||||
*
|
||||
* @author rewerma 2018-8-19 下午06:14:23
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class MessageUtil {
|
||||
|
||||
public static List<CommonMessage> convert(Message message) {
|
||||
if (message == null) {
|
||||
return null;
|
||||
}
|
||||
List<CanalEntry.Entry> entries = message.getEntries();
|
||||
List<CommonMessage> msgs = new ArrayList<>(entries.size());
|
||||
for (CanalEntry.Entry entry : entries) {
|
||||
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
continue;
|
||||
}
|
||||
|
||||
CanalEntry.RowChange rowChange;
|
||||
try {
|
||||
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
|
||||
e);
|
||||
}
|
||||
|
||||
CanalEntry.EventType eventType = rowChange.getEventType();
|
||||
|
||||
final CommonMessage msg = new CommonMessage();
|
||||
msg.setIsDdl(rowChange.getIsDdl());
|
||||
msg.setDatabase(entry.getHeader().getSchemaName());
|
||||
msg.setTable(entry.getHeader().getTableName());
|
||||
msg.setType(eventType.toString());
|
||||
msg.setEs(entry.getHeader().getExecuteTime());
|
||||
msg.setIsDdl(rowChange.getIsDdl());
|
||||
msg.setTs(System.currentTimeMillis());
|
||||
msg.setSql(rowChange.getSql());
|
||||
msgs.add(msg);
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
List<Map<String, Object>> old = new ArrayList<>();
|
||||
|
||||
if (!rowChange.getIsDdl()) {
|
||||
Set<String> updateSet = new HashSet<>();
|
||||
msg.setPkNames(new ArrayList<>());
|
||||
int i = 0;
|
||||
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
|
||||
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
|
||||
&& eventType != CanalEntry.EventType.DELETE) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
List<CanalEntry.Column> columns;
|
||||
|
||||
if (eventType == CanalEntry.EventType.DELETE) {
|
||||
columns = rowData.getBeforeColumnsList();
|
||||
} else {
|
||||
columns = rowData.getAfterColumnsList();
|
||||
}
|
||||
|
||||
for (CanalEntry.Column column : columns) {
|
||||
if (i == 0) {
|
||||
if (column.getIsKey()) {
|
||||
msg.getPkNames().add(column.getName());
|
||||
}
|
||||
}
|
||||
if (column.getIsNull()) {
|
||||
row.put(column.getName(), null);
|
||||
} else {
|
||||
row.put(column.getName(),
|
||||
JdbcTypeUtil.typeConvert(msg.getTable(),
|
||||
column.getName(),
|
||||
column.getValue(),
|
||||
column.getSqlType(),
|
||||
column.getMysqlType()));
|
||||
}
|
||||
// 获取update为true的字段
|
||||
if (column.getUpdated()) {
|
||||
updateSet.add(column.getName());
|
||||
}
|
||||
}
|
||||
if (!row.isEmpty()) {
|
||||
data.add(row);
|
||||
}
|
||||
|
||||
if (eventType == CanalEntry.EventType.UPDATE) {
|
||||
Map<String, Object> rowOld = new LinkedHashMap<>();
|
||||
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
|
||||
if (updateSet.contains(column.getName())) {
|
||||
if (column.getIsNull()) {
|
||||
rowOld.put(column.getName(), null);
|
||||
} else {
|
||||
rowOld.put(column.getName(),
|
||||
JdbcTypeUtil.typeConvert(msg.getTable(),
|
||||
column.getName(),
|
||||
column.getValue(),
|
||||
column.getSqlType(),
|
||||
column.getMysqlType()));
|
||||
}
|
||||
}
|
||||
}
|
||||
// update操作将记录修改前的值
|
||||
if (!rowOld.isEmpty()) {
|
||||
old.add(rowOld);
|
||||
}
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
if (!data.isEmpty()) {
|
||||
msg.setData(data);
|
||||
}
|
||||
if (!old.isEmpty()) {
|
||||
msg.setOld(old);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return msgs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.alibaba.otter.canal.connector.core.util;
|
||||
|
||||
|
||||
public class TimeZone {
|
||||
public final static String LOCATION_TIME_ZONE; // 当前时区
|
||||
|
||||
static {
|
||||
java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault();
|
||||
int rawOffset = localTimeZone.getRawOffset();
|
||||
String symbol = "+";
|
||||
if (rawOffset < 0) {
|
||||
symbol = "-";
|
||||
}
|
||||
rawOffset = Math.abs(rawOffset);
|
||||
int offsetHour = rawOffset / 3600000;
|
||||
int offsetMinute = rawOffset % 3600000 / 60000;
|
||||
String hour = String.format("%1$02d", offsetHour);
|
||||
String minute = String.format("%1$02d", offsetMinute);
|
||||
LOCATION_TIME_ZONE = symbol + hour + ":" + minute;
|
||||
java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("GMT" + LOCATION_TIME_ZONE));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user