chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

升级rocketmq到4.8.0，并且支持环境变量优先配置 (#3450)

Browse Source
This commit is contained in:
zoemak
2021-04-16 21:35:11 +08:00
committed by GitHub
parent 58448e559d
commit 32d9e58915
7 changed files with 91 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java
View File

@@ -1,17 +1,17 @@
package com.alibaba.otter.canal.connector.core.producer;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.common.utils.PropertiesUtils;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
import org.apache.commons.lang.StringUtils;
import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.config.MQProperties;
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
/**
* MQ producer 抽象类
*
@@ -20,7 +20,7 @@ import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
*/
public abstract class AbstractMQProducer implements CanalMQProducer {
protected MQProperties mqProperties;
protected MQProperties mqProperties;
protected ThreadPoolExecutor sendExecutor;
protected ThreadPoolExecutor buildExecutor;
@@ -32,21 +32,21 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
int parallelBuildThreadSize = mqProperties.getParallelBuildThreadSize();
buildExecutor = new ThreadPoolExecutor(parallelBuildThreadSize,
parallelBuildThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(parallelBuildThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Builder"),
new ThreadPoolExecutor.CallerRunsPolicy());
parallelBuildThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(parallelBuildThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Builder"),
new ThreadPoolExecutor.CallerRunsPolicy());
int parallelSendThreadSize = mqProperties.getParallelSendThreadSize();
sendExecutor = new ThreadPoolExecutor(parallelSendThreadSize,
parallelSendThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(parallelSendThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Sender"),
new ThreadPoolExecutor.CallerRunsPolicy());
parallelSendThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(parallelSendThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Sender"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
@Override
@@ -77,52 +77,52 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
* canal.mq.timeout = 100 <br/>
* canal.mq.access.channel = local <br/>
* </p>
*
*
* @param properties 总配置对象
*/
private void loadCanalMqProperties(Properties properties) {
String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
String flatMessage = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_FLAT_MESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
}
String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
String databaseHash = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_DATABASE_HASH);
if (!StringUtils.isEmpty(databaseHash)) {
mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
}
String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
String filterTranEntry = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
if (!StringUtils.isEmpty(filterTranEntry)) {
mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
}
String parallelBuildThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_BUILD_THREAD_SIZE);
String parallelBuildThreadSize = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_BUILD_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelBuildThreadSize)) {
mqProperties.setParallelBuildThreadSize(Integer.parseInt(parallelBuildThreadSize));
}
String parallelSendThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_SEND_THREAD_SIZE);
String parallelSendThreadSize = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_SEND_THREAD_SIZE);
if (!StringUtils.isEmpty(parallelSendThreadSize)) {
mqProperties.setParallelSendThreadSize(Integer.parseInt(parallelSendThreadSize));
}
String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
String batchSize = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
if (!StringUtils.isEmpty(batchSize)) {
mqProperties.setBatchSize(Integer.parseInt(batchSize));
}
String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_GET_TIMEOUT);
String timeOut = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_CANAL_GET_TIMEOUT);
if (!StringUtils.isEmpty(timeOut)) {
mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
}
String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
String accessChannel = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
if (!StringUtils.isEmpty(accessChannel)) {
mqProperties.setAccessChannel(accessChannel);
}
String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
String aliyunAccessKey = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
if (!StringUtils.isEmpty(aliyunAccessKey)) {
mqProperties.setAliyunAccessKey(aliyunAccessKey);
}
String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
String aliyunSecretKey = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRET_KEY);
if (!StringUtils.isEmpty(aliyunSecretKey)) {
mqProperties.setAliyunAccessKey(aliyunSecretKey);
}
String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
String aliyunUid = PropertiesUtils.getProperty(properties, CanalConstants.CANAL_ALIYUN_UID);
if (!StringUtils.isEmpty(aliyunUid)) {
mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
}
@@ -132,7 +132,7 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
* 兼容下<=1.1.4的mq配置项
*/
protected void doMoreCompatibleConvert(String oldKey, String newKey, Properties properties) {
String value = properties.getProperty(oldKey);
String value = PropertiesUtils.getProperty(properties, oldKey);
if (StringUtils.isNotEmpty(value)) {
properties.setProperty(newKey, value);
}