chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge code and fixed issue #2755 #2756 #2735 #2714 #2711 #2710 #2616 #2602 #2585 #2565 #2935 #2434 #2623 #2418 #2387

Browse Source
This commit is contained in:
agapple
2020-08-22 13:07:24 +08:00
parent 90bf9e6de5
commit b79af46054
66 changed files with 459 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java
View File

@@ -12,12 +12,14 @@ public class CanalConstants {
public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
public static final String CANAL_MQ_FLAT_MESSAGE = ROOT + "." + "mq.flat.message";
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.accessChannel";
public static final String CANAL_MQ_CANAL_BATCH_SIZE = ROOT + "." + "mq.canalBatchSize";
public static final String CANAL_MQ_CANAL_GET_TIMEOUT = ROOT + "." + "mq.canalGetTimeout";
public static final String CANAL_MQ_FLAT_MESSAGE = ROOT + "." + "mq.flatMessage";
public static final String CANAL_MQ_DATABASE_HASH = ROOT + "." + "mq.database.hash";
public static final String CANAL_MQ_PARALLEL_THREAD_SIZE = ROOT + "." + "mq.parallel.thread.size";
public static final String CANAL_MQ_CANAL_BATCH_SIZE = ROOT + "." + "mq.canal.batch.size";
public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT = ROOT + "." + "mq.canal.fetch.timeout";
public static final String CANAL_MQ_ACCESS_CHANNEL = ROOT + "." + "mq.access.channel";
public static final String CANAL_MQ_BUILD_THREAD_SIZE = ROOT + "." + "mq.build.thread.size";
public static final String CANAL_MQ_SEND_THREAD_SIZE = ROOT + "." + "mq.send.thread.size";
public static final String CANAL_ALIYUN_ACCESS_KEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRET_KEY = ROOT + "." + "aliyun.secretKey";

37
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java
View File

@@ -8,17 +8,18 @@ package com.alibaba.otter.canal.connector.core.config;
*/
public class MQProperties {
private boolean flatMessage = true;
private boolean databaseHash = true;
private boolean filterTransactionEntry = true;
private Integer parallelThreadSize = 8;
private Integer fetchTimeout = 100;
private Integer batchSize = 50;
private String accessChannel = "local";
private boolean flatMessage = true;
private boolean databaseHash = true;
private boolean filterTransactionEntry = true;
private Integer parallelBuildThreadSize = 8;
private Integer parallelSendThreadSize = 30;
private Integer fetchTimeout = 100;
private Integer batchSize = 50;
private String accessChannel = "local";
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private int aliyunUid = 0;
private String aliyunAccessKey = "";
private String aliyunSecretKey = "";
private int aliyunUid = 0;
public boolean isFlatMessage() {
return flatMessage;
@@ -44,12 +45,20 @@ public class MQProperties {
this.filterTransactionEntry = filterTransactionEntry;
}
public Integer getParallelThreadSize() {
return parallelThreadSize;
public Integer getParallelBuildThreadSize() {
return parallelBuildThreadSize;
}
public void setParallelThreadSize(Integer parallelThreadSize) {
this.parallelThreadSize = parallelThreadSize;
public void setParallelBuildThreadSize(Integer parallelBuildThreadSize) {
this.parallelBuildThreadSize = parallelBuildThreadSize;
}
public Integer getParallelSendThreadSize() {
return parallelSendThreadSize;
}
public void setParallelSendThreadSize(Integer parallelSendThreadSize) {
this.parallelSendThreadSize = parallelSendThreadSize;
}
public Integer getFetchTimeout() {

56
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java
View File

@@ -22,20 +22,30 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
protected MQProperties mqProperties;
protected ThreadPoolExecutor executor;
protected ThreadPoolExecutor sendExecutor;
protected ThreadPoolExecutor buildExecutor;
@Override
public void init(Properties properties) {
// parse canal mq properties
loadCanalMqProperties(properties);
int parallelThreadSize = mqProperties.getParallelThreadSize();
executor = new ThreadPoolExecutor(parallelThreadSize,
parallelThreadSize,
int parallelBuildThreadSize = mqProperties.getParallelBuildThreadSize();
buildExecutor = new ThreadPoolExecutor(parallelBuildThreadSize,
parallelBuildThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
new NamedThreadFactory("MQParallel"),
new ArrayBlockingQueue<Runnable>(parallelBuildThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Builder"),
new ThreadPoolExecutor.CallerRunsPolicy());
int parallelSendThreadSize = mqProperties.getParallelSendThreadSize();
sendExecutor = new ThreadPoolExecutor(parallelSendThreadSize,
parallelSendThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelSendThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Sender"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@@ -46,8 +56,12 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
@Override
public void stop() {
if (executor != null) {
executor.shutdownNow();
if (buildExecutor != null) {
buildExecutor.shutdownNow();
}
if (sendExecutor != null) {
sendExecutor.shutdownNow();
}
}
@@ -57,7 +71,8 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
* canal.mq.flat.message = true <br/>
* canal.mq.database.hash = true <br/>
* canal.mq.filter.transaction.entry = true <br/>
* canal.mq.parallel.thread.size = 8 <br/>
* canal.mq.parallel.build.thread.size = 8 <br/>
* canal.mq.parallel.send.thread.size = 8 <br/>
* canal.mq.batch.size = 50 <br/>
* canal.mq.timeout = 100 <br/>
* canal.mq.access.channel = local <br/>
@@ -70,6 +85,7 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
}
String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
if (!StringUtils.isEmpty(databaseHash)) {
mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
@@ -78,15 +94,19 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
if (!StringUtils.isEmpty(filterTranEntry)) {
mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
}
String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelThreadSize)) {
mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize));
String parallelBuildThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_BUILD_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelBuildThreadSize)) {
mqProperties.setParallelBuildThreadSize(Integer.parseInt(parallelBuildThreadSize));
}
String parallelSendThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_SEND_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelSendThreadSize)) {
mqProperties.setParallelSendThreadSize(Integer.parseInt(parallelSendThreadSize));
}
String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
if (!StringUtils.isEmpty(batchSize)) {
mqProperties.setBatchSize(Integer.parseInt(batchSize));
}
String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT);
String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_GET_TIMEOUT);
if (!StringUtils.isEmpty(timeOut)) {
mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
}
@@ -107,4 +127,14 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
}
}
/**
* 兼容下<=1.1.4的mq配置项
*/
protected void doMoreCompatibleConvert(String oldKey, String newKey, Properties properties) {
String value = properties.getProperty(oldKey);
if (StringUtils.isNotEmpty(value)) {
properties.setProperty(newKey, value);
}
}
}

2
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
View File

@@ -49,7 +49,7 @@ public class MQMessageUtils {
int i = pkHashConfig.lastIndexOf(":");
if (i > 0) {
String pkStr = pkHashConfig.substring(i + 1);
if ("$pk$".equalsIgnoreCase(pkStr)) {
if (pkStr.equalsIgnoreCase("$pk$")) {
data.hashMode.autoPkHash = true;
} else {
data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,

2
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java
View File

@@ -78,7 +78,7 @@ public class JdbcTypeUtil {
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
if (value == null
|| ("".equals(value) && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
|| (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
return null;
}

1
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java
View File

@@ -42,6 +42,7 @@ public class MessageUtil {
CanalEntry.EventType eventType = rowChange.getEventType();
final CommonMessage msg = new CommonMessage();
msg.setIsDdl(rowChange.getIsDdl());
msg.setDatabase(entry.getHeader().getSchemaName());
msg.setTable(entry.getHeader().getTableName());
msg.setType(eventType.toString());