chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

配置整理

Browse Source
This commit is contained in:
mcy
2018-11-14 11:32:01 +08:00
parent c376f0afd2
commit 68d8aee6c3
8 changed files with 53 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -17,13 +17,13 @@ public class CanalClientConfig {
private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!! private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!!
private String mqServers; // kafka or rocket mq 地址 private String mqServers; // kafka or rocket mq 地址
private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式 private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式
private Integer batchSize; // 批大小 private Integer batchSize; // 批大小
private Integer retry; // 重试次数 private Integer retries; // 重试次数
private Long timeout; // 消费超时时间 private Long timeout; // 消费超时时间
@@ -79,12 +79,12 @@ public class CanalClientConfig {
this.batchSize = batchSize; this.batchSize = batchSize;
} }
public Integer getRetry() { public Integer getRetries() {
return retry; return retries;
} }
public void setRetry(Integer retry) { public void setRetries(Integer retries) {
this.retry = retry; this.retries = retries;
} }
public Long getTimeout() { public Long getTimeout() {

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -45,7 +45,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
while (!running) while (!running)
; ;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); int retry = canalClientConfig.getRetries() == null ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -33,8 +33,6 @@ public class CanalAdapterLoader {
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>(); private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private Map<String, OuterAdapter> outerAdapters = new HashMap<>(); // 配置文件对应adapter
private ExtensionLoader<OuterAdapter> loader; private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){ public CanalAdapterLoader(CanalClientConfig canalClientConfig){

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -43,7 +43,7 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
; ;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); int retry = canalClientConfig.getRetries() == null ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -64,7 +64,7 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
; // waiting until running == true ; // waiting until running == true
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); int retry = canalClientConfig.getRetries() == null ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟 long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟
Integer batchSize = canalClientConfig.getBatchSize(); Integer batchSize = canalClientConfig.getBatchSize();
if (batchSize == null) { if (batchSize == null) {

2
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -16,6 +16,8 @@ canal.conf:
# zookeeperHosts: slave1:2181 # zookeeperHosts: slave1:2181
# mqServers: slave1:6667 #or rocketmq # mqServers: slave1:6667 #or rocketmq
# flatMessage: true # flatMessage: true
retries:
timeout:
# srcDataSources: # srcDataSources:
# defaultDS: # defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true # url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true

57
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -85,23 +85,46 @@ public class CanalLauncher {
private static MQProperties buildMQPosition(Properties properties) { private static MQProperties buildMQPosition(Properties properties) {
MQProperties mqProperties = new MQProperties(); MQProperties mqProperties = new MQProperties();
mqProperties.setServers(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS)); String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
mqProperties.setRetries(Integer.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(servers)) {
CanalConstants.CANAL_MQ_RETRIES))); mqProperties.setServers(servers);
mqProperties.setBatchSize(Integer.valueOf(CanalController.getProperty(properties, }
CanalConstants.CANAL_MQ_BATCHSIZE))); String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
mqProperties.setLingerMs(Integer.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(retires)) {
CanalConstants.CANAL_MQ_LINGERMS))); mqProperties.setRetries(Integer.valueOf(retires));
mqProperties.setBufferMemory(Long.valueOf(CanalController.getProperty(properties, }
CanalConstants.CANAL_MQ_BUFFERMEMORY))); String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
mqProperties.setCanalBatchSize(Integer.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(batchSize)) {
CanalConstants.CANAL_MQ_CANALBATCHSIZE))); mqProperties.setBatchSize(Integer.valueOf(batchSize));
mqProperties.setCanalGetTimeout(Long.valueOf(CanalController.getProperty(properties, }
CanalConstants.CANAL_MQ_CANALGETTIMEOUT))); String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
mqProperties.setFlatMessage(Boolean.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(lingerMs)) {
CanalConstants.CANAL_MQ_FLATMESSAGE))); mqProperties.setLingerMs(Integer.valueOf(lingerMs));
mqProperties.setCompressionType(CanalController.getProperty(properties,CanalConstants.CANAL_MQ_COMPRESSION_TYPE)); }
mqProperties.setAcks(CanalController.getProperty(properties,CanalConstants.CANAL_MQ_ACKS)); String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
if (!StringUtils.isEmpty(bufferMemory)) {
mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
}
String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
if (!StringUtils.isEmpty(canalBatchSize)) {
mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
}
String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
if (!StringUtils.isEmpty(canalGetTimeout)) {
mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
}
String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
}
String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
if (!StringUtils.isEmpty(compressionType)) {
mqProperties.setCompressionType(compressionType);
}
String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks);
}
return mqProperties; return mqProperties;
} }

4
deployer/src/main/resources/canal.properties
View File

@@ -109,5 +109,5 @@ canal.mq.bufferMemory=33554432
canal.mq.canalBatchSize=50 canal.mq.canalBatchSize=50
canal.mq.canalGetTimeout=100 canal.mq.canalGetTimeout=100
canal.mq.flatMessage=true canal.mq.flatMessage=true
#canal.mq.compressionType=none canal.mq.compressionType=
#canal.mq.acks=all canal.mq.acks=