chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1143 from rewerma/master

Browse Source 
配置整理
This commit is contained in:
agapple
2018-11-14 16:54:29 +08:00
committed by GitHub
parent dd04577994 31bf92ec8e
commit 6f3cae5ddf
9 changed files with 69 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -17,13 +17,13 @@ public class CanalClientConfig {
private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!! private String zookeeperHosts; // 集群模式下的zk地址, 如果配置了单机地址则以单机为准!!
private String bootstrapServers; // kafka or rocket mq 地址 private String mqServers; // kafka or rocket mq 地址
private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式 private Boolean flatMessage = true; // 是否已flatMessage模式传输, 只适用于mq模式
private Integer batchSize; // 批大小 private Integer batchSize; // 批大小
private Integer retry; // 重试次数 private Integer retries; // 重试次数
private Long timeout; // 消费超时时间 private Long timeout; // 消费超时时间
@@ -47,12 +47,12 @@ public class CanalClientConfig {
this.zookeeperHosts = zookeeperHosts; this.zookeeperHosts = zookeeperHosts;
} }
public String getBootstrapServers() { public String getMqServers() {
return bootstrapServers; return mqServers;
} }
public void setBootstrapServers(String bootstrapServers) { public void setMqServers(String mqServers) {
this.bootstrapServers = bootstrapServers; this.mqServers = mqServers;
} }
public List<MQTopic> getMqTopics() { public List<MQTopic> getMqTopics() {
@@ -79,12 +79,12 @@ public class CanalClientConfig {
this.batchSize = batchSize; this.batchSize = batchSize;
} }
public Integer getRetry() { public Integer getRetries() {
return retry; return retries;
} }
public void setRetry(Integer retry) { public void setRetries(Integer retries) {
this.retry = retry; this.retries = retries;
} }
public Long getTimeout() { public Long getTimeout() {

3
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java
View File

@@ -180,9 +180,6 @@ public class ExtensionLoader<T> {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private T createExtension(String name, String key) { private T createExtension(String name, String key) {
// System.out.println("xxxxxxxxxxxxx");
// getExtensionClasses().forEach((k, v) -> logger.info("fffff: " + k +
// " " + v.getName()));
Class<?> clazz = getExtensionClasses().get(name); Class<?> clazz = getExtensionClasses().get(name);
if (clazz == null) { if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -45,7 +45,8 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
while (!running) while (!running)
; ;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

6
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -33,8 +33,6 @@ public class CanalAdapterLoader {
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>(); private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private Map<String, OuterAdapter> outerAdapters = new HashMap<>(); // 配置文件对应adapter
private ExtensionLoader<OuterAdapter> loader; private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){ public CanalAdapterLoader(CanalClientConfig canalClientConfig){
@@ -97,7 +95,7 @@ public class CanalAdapterLoader {
canalOuterAdapterGroups.add(canalOuterAdapters); canalOuterAdapterGroups.add(canalOuterAdapters);
if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) { if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getBootstrapServers(), canalClientConfig.getMqServers(),
topic.getTopic(), topic.getTopic(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,
@@ -106,7 +104,7 @@ public class CanalAdapterLoader {
rocketMQWorker.start(); rocketMQWorker.start();
} else if ("kafka".equalsIgnoreCase(topic.getMqMode())) { } else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
canalClientConfig.getBootstrapServers(), canalClientConfig.getMqServers(),
topic.getTopic(), topic.getTopic(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -43,7 +43,8 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
; ;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) { while (running) {

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -64,7 +64,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
; // waiting until running == true ; // waiting until running == true
ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry(); int retry = canalClientConfig.getRetries() == null
|| canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟 long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟
Integer batchSize = canalClientConfig.getBatchSize(); Integer batchSize = canalClientConfig.getBatchSize();
if (batchSize == null) { if (batchSize == null) {

14
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -14,18 +14,20 @@ spring:
canal.conf: canal.conf:
canalServerHost: 127.0.0.1:11111 canalServerHost: 127.0.0.1:11111
# zookeeperHosts: slave1:2181 # zookeeperHosts: slave1:2181
# bootstrapServers: slave1:6667 #or rocketmq # mqServers: slave1:6667 #or rocketmq
# flatMessage: true # flatMessage: true
retries: 0
timeout:
# srcDataSources: # srcDataSources:
# defaultDS: # defaultDS:
# url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true # url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
# username: root # username: root
# password: 121212 # password: 121212
canalInstances: # canalInstances:
- instance: example # - instance: example
groups: # groups:
- outAdapters: # - outAdapters:
- name: logger # - name: logger
# - name: rdb # - name: rdb
# key: oracle1 # key: oracle1
# properties: # properties:

57
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
View File

@@ -85,23 +85,46 @@ public class CanalLauncher {
private static MQProperties buildMQPosition(Properties properties) { private static MQProperties buildMQPosition(Properties properties) {
MQProperties mqProperties = new MQProperties(); MQProperties mqProperties = new MQProperties();
mqProperties.setServers(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS)); String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
mqProperties.setRetries(Integer.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(servers)) {
CanalConstants.CANAL_MQ_RETRIES))); mqProperties.setServers(servers);
mqProperties.setBatchSize(Integer.valueOf(CanalController.getProperty(properties, }
CanalConstants.CANAL_MQ_BATCHSIZE))); String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
mqProperties.setLingerMs(Integer.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(retires)) {
CanalConstants.CANAL_MQ_LINGERMS))); mqProperties.setRetries(Integer.valueOf(retires));
mqProperties.setBufferMemory(Long.valueOf(CanalController.getProperty(properties, }
CanalConstants.CANAL_MQ_BUFFERMEMORY))); String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
mqProperties.setCanalBatchSize(Integer.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(batchSize)) {
CanalConstants.CANAL_MQ_CANALBATCHSIZE))); mqProperties.setBatchSize(Integer.valueOf(batchSize));
mqProperties.setCanalGetTimeout(Long.valueOf(CanalController.getProperty(properties, }
CanalConstants.CANAL_MQ_CANALGETTIMEOUT))); String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
mqProperties.setFlatMessage(Boolean.valueOf(CanalController.getProperty(properties, if (!StringUtils.isEmpty(lingerMs)) {
CanalConstants.CANAL_MQ_FLATMESSAGE))); mqProperties.setLingerMs(Integer.valueOf(lingerMs));
mqProperties.setCompressionType(CanalController.getProperty(properties,CanalConstants.CANAL_MQ_COMPRESSION_TYPE)); }
mqProperties.setAcks(CanalController.getProperty(properties,CanalConstants.CANAL_MQ_ACKS)); String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
if (!StringUtils.isEmpty(bufferMemory)) {
mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
}
String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
if (!StringUtils.isEmpty(canalBatchSize)) {
mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
}
String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
if (!StringUtils.isEmpty(canalGetTimeout)) {
mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
}
String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
if (!StringUtils.isEmpty(flatMessage)) {
mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
}
String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
if (!StringUtils.isEmpty(compressionType)) {
mqProperties.setCompressionType(compressionType);
}
String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
if (!StringUtils.isEmpty(acks)) {
mqProperties.setAcks(acks);
}
return mqProperties; return mqProperties;
} }

2
deployer/src/main/resources/canal.properties
View File

@@ -109,3 +109,5 @@ canal.mq.bufferMemory=33554432
canal.mq.canalBatchSize=50 canal.mq.canalBatchSize=50
canal.mq.canalGetTimeout=100 canal.mq.canalGetTimeout=100
canal.mq.flatMessage=true canal.mq.flatMessage=true
canal.mq.compressionType=
canal.mq.acks=