chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加adapter相关参数: retry timeout batchSize

Browse Source
This commit is contained in:
mcy
2018-10-30 14:13:51 +08:00
parent 11ac253c46
commit e4e23c9ecd
18 changed files with 513 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java
View File

@@ -133,7 +133,7 @@ public class SimpleCanalConnector implements CanalConnector {
runningMonitor.stop();
}
} else {
doDisconnnect();
doDisconnect();
}
}
@@ -190,7 +190,7 @@ public class SimpleCanalConnector implements CanalConnector {
}
}
private void doDisconnnect() throws CanalClientException {
private void doDisconnect() throws CanalClientException {
if (readableChannel != null) {
quietlyClose(readableChannel);
readableChannel = null;
@@ -434,7 +434,7 @@ public class SimpleCanalConnector implements CanalConnector {
public void processActiveExit() {
mutex.set(false);
doDisconnnect();
doDisconnect();
}
});

8
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -42,7 +42,8 @@ public class KafkaCanalConnector implements CanalMQConnector {
private volatile boolean running = false;
private boolean flatMessage;
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, boolean flatMessage){
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
boolean flatMessage){
this.topic = topic;
this.partition = partition;
this.flatMessage = flatMessage;
@@ -55,7 +56,10 @@ public class KafkaCanalConnector implements CanalMQConnector {
properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
properties.put("session.timeout.ms", "30000"); // 默认为30秒
properties.put("max.poll.records", "100");
if (batchSize == null) {
batchSize = 100;
}
properties.put("max.poll.records", batchSize.toString());
properties.put("key.deserializer", StringDeserializer.class.getName());
if (!flatMessage) {
properties.put("value.deserializer", MessageDeserializer.class.getName());

50
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java
View File

@@ -1,50 +0,0 @@
package com.alibaba.otter.canal.client.kafka;
/**
* canal kafka connectors创建工具类
*
* @author machengyuan @ 2018-6-12
* @version 1.0.0
*/
public class KafkaCanalConnectors {
/**
* 创建kafka客户端链接，独立运行不注册zk信息
*
* @param servers
* @param topic
* @param partition
* @param groupId
* @return
*/
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) {
return new KafkaCanalConnector(servers, topic, partition, groupId, false);
}
/**
* 创建kafka客户端链接，独立运行不注册zk信息
*
* @param servers
* @param topic
* @param groupId
* @return
*/
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
return new KafkaCanalConnector(servers, topic, null, groupId, false);
}
/**
* 创建kafka客户端链接
*
* @param servers
* @param topic
* @param partition
* @param groupId
* @param flatMessage
* @return
*/
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition,
String groupId, boolean flatMessage) {
return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage);
}
}

24
client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnectors.java
View File

@@ -1,24 +0,0 @@
package com.alibaba.otter.canal.client.rocketmq;
/**
* RocketMQ connector provider.
*/
public class RocketMQCanalConnectors {
/**
* Create RocketMQ connector
*
* @param nameServers name servers for RocketMQ
* @param topic
* @param groupId
* @return {@link RocketMQCanalConnector}
*/
public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId) {
return new RocketMQCanalConnector(nameServers, topic, groupId, false);
}
public static RocketMQCanalConnector newRocketMQConnector(String nameServers, String topic, String groupId,
boolean flatMessage) {
return new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage);
}
}

6
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java
View File

@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -36,12 +35,13 @@ public class CanalKafkaClientExample {
};
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false);
connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
}
public static void main(String[] args) {
try {
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
AbstractKafkaTest.zkServers,
AbstractKafkaTest.servers,
AbstractKafkaTest.topic,
AbstractKafkaTest.partition,

2
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java
View File

@@ -30,7 +30,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
public void testKafkaConsumer() {
final ExecutorService executor = Executors.newFixedThreadPool(1);
final KafkaCanalConnector connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId);
final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
executor.submit(new Runnable() {

3
client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/CanalRocketMQClientExample.java
View File

@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectors;
import com.alibaba.otter.canal.client.running.kafka.AbstractKafkaTest;
import com.alibaba.otter.canal.protocol.Message;
@@ -37,7 +36,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
};
public CanalRocketMQClientExample(String nameServers, String topic, String groupId){
connector = RocketMQCanalConnectors.newRocketMQConnector(nameServers, topic, groupId);
connector = new RocketMQCanalConnector(nameServers, topic, groupId, false);
}
public static void main(String[] args) {