chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加kafka自定义properties

Browse Source
This commit is contained in:
mcy
2019-02-05 19:20:36 +08:00
parent f6d03bcb37
commit ad62275cd7
5 changed files with 47 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
View File

@@ -51,6 +51,7 @@ public class CanalConstants {
public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction";
public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey";
public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey";
public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties";
public static String getInstanceModeKey(String destination) { public static String getInstanceModeKey(String destination) {
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

12
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.deployer; package com.alibaba.otter.canal.deployer;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@@ -162,7 +163,16 @@ public class CanalStater {
if (!StringUtils.isEmpty(transaction)) { if (!StringUtils.isEmpty(transaction)) {
mqProperties.setTransaction(Boolean.valueOf(transaction)); mqProperties.setTransaction(Boolean.valueOf(transaction));
} }
for (Object key : properties.keySet()) {
key = StringUtils.trim(key.toString());
if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) {
String value = CanalController.getProperty(properties, (String) key);
String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1);
mqProperties.getProperties().put(subKey, value);
}
}
return mqProperties; return mqProperties;
} }
} }

1
deployer/src/main/resources/canal.properties
View File

@@ -117,3 +117,4 @@ canal.mq.compressionType = none
canal.mq.acks = all canal.mq.acks = all
# use transaction for kafka flatMessage batch produce # use transaction for kafka flatMessage batch produce
canal.mq.transaction = false canal.mq.transaction = false
#canal.mq.properties. =

43
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.common; package com.alibaba.otter.canal.common;
import java.util.Properties;
/** /**
* kafka 配置项 * kafka 配置项
* *
@@ -8,22 +10,23 @@ package com.alibaba.otter.canal.common;
*/ */
public class MQProperties { public class MQProperties {
private String servers = "127.0.0.1:6667"; private String servers = "127.0.0.1:6667";
private int retries = 0; private int retries = 0;
private int batchSize = 16384; private int batchSize = 16384;
private int lingerMs = 1; private int lingerMs = 1;
private int maxRequestSize = 1048576; private int maxRequestSize = 1048576;
private long bufferMemory = 33554432L; private long bufferMemory = 33554432L;
private boolean filterTransactionEntry = true; private boolean filterTransactionEntry = true;
private String producerGroup = "Canal-Producer"; private String producerGroup = "Canal-Producer";
private int canalBatchSize = 50; private int canalBatchSize = 50;
private Long canalGetTimeout = 100L; private Long canalGetTimeout = 100L;
private boolean flatMessage = true; private boolean flatMessage = true;
private String compressionType = "none"; private String compressionType = "none";
private String acks = "all"; private String acks = "all";
private String aliyunAccessKey = ""; private String aliyunAccessKey = "";
private String aliyunSecretKey = ""; private String aliyunSecretKey = "";
private boolean transaction = false; // 是否开启事务 private boolean transaction = false; // 是否开启事务
private Properties properties = new Properties();
public static class CanalDestination { public static class CanalDestination {
@@ -210,4 +213,12 @@ public class MQProperties {
public void setTransaction(boolean transaction) { public void setTransaction(boolean transaction) {
this.transaction = transaction; this.transaction = transaction;
} }
public Properties getProperties() {
return properties;
}
public void setProperties(Properties properties) {
this.properties = properties;
}
} }

7
server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java
View File

@@ -49,7 +49,12 @@ public class CanalKafkaProducer implements CanalMQProducer {
properties.put("max.request.size", kafkaProperties.getMaxRequestSize()); properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
properties.put("buffer.memory", kafkaProperties.getBufferMemory()); properties.put("buffer.memory", kafkaProperties.getBufferMemory());
properties.put("key.serializer", StringSerializer.class.getName()); properties.put("key.serializer", StringSerializer.class.getName());
if(kafkaProperties.getTransaction()){
if (!kafkaProperties.getProperties().isEmpty()) {
properties.putAll(kafkaProperties.getProperties());
}
if (kafkaProperties.getTransaction()) {
properties.put("transactional.id", "canal-transactional-id"); properties.put("transactional.id", "canal-transactional-id");
} else { } else {
properties.put("retries", kafkaProperties.getRetries()); properties.put("retries", kafkaProperties.getRetries());