chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加adapter相关参数: retry timeout batchSize

Browse Source
This commit is contained in:
mcy
2018-10-30 14:13:51 +08:00
parent 11ac253c46
commit e4e23c9ecd
18 changed files with 513 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java
View File

@@ -2,11 +2,10 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.*;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalMQConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,6 +116,52 @@ public abstract class AbstractCanalAdapterWorker {
});
}
protected void mqWriteOutData(int retry, long timeout, boolean flatMessage, CanalMQConnector connector,
ExecutorService workerExecutor) {
for (int i = 0; i < retry; i++) {
try {
List<?> messages;
if (!flatMessage) {
messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
} else {
messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null) {
Future<Boolean> future = workerExecutor.submit(() -> {
for (final Object message : messages) {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
}
return true;
});
try {
future.get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
future.cancel(true);
throw e;
}
}
connector.ack();
break;
} catch (Throwable e) {
if (i == retry - 1) {
connector.ack();
}
logger.error(e.getMessage(), e);
try {
TimeUnit.SECONDS.sleep(1L);
} catch (InterruptedException e1) {
// ignore
}
}
}
}
public void start() {
if (!running) {
thread = new Thread(this::process);

54
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java
View File

@@ -1,14 +1,14 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.*;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
@@ -20,17 +20,24 @@ import com.alibaba.otter.canal.protocol.Message;
*/
public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private CanalClientConfig canalClientConfig;
private KafkaCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.canalDestination = topic;
this.flatMessage = flatMessage;
this.connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
this.connector = new KafkaCanalConnector(bootstrapServers,
topic,
null,
groupId,
canalClientConfig.getBatchSize(),
flatMessage);
// connector.setSessionTimeout(1L, TimeUnit.MINUTES);
}
@@ -38,6 +45,10 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
protected void process() {
while (!running)
;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
@@ -47,35 +58,12 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
while (running) {
try {
Boolean status = syncSwitch.status(canalDestination);
if (status != null && !status) {
connector.disconnect();
break;
}
List<?> messages;
if (!flatMessage) {
messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
} else {
messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null) {
for (final Object message : messages) {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
}
}
connector.ack();
} catch (CommitFailedException e) {
logger.warn(e.getMessage());
} catch (Throwable e) {
logger.error(e.getMessage(), e);
TimeUnit.SECONDS.sleep(1L);
Boolean status = syncSwitch.status(canalDestination);
if (status != null && !status) {
connector.disconnect();
break;
}
mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);

16
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -67,9 +67,15 @@ public class CanalAdapterLoader {
}
CanalAdapterWorker worker;
if (sa != null) {
worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups);
worker = new CanalAdapterWorker(canalClientConfig,
instance.getInstance(),
sa,
canalOuterAdapterGroups);
} else if (zkHosts != null) {
worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups);
worker = new CanalAdapterWorker(canalClientConfig,
instance.getInstance(),
zkHosts,
canalOuterAdapterGroups);
} else {
throw new RuntimeException("No canal server connector found");
}
@@ -90,7 +96,8 @@ public class CanalAdapterLoader {
}
canalOuterAdapterGroups.add(canalOuterAdapters);
if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig.getBootstrapServers(),
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getBootstrapServers(),
topic.getTopic(),
group.getGroupId(),
canalOuterAdapterGroups,
@@ -98,7 +105,8 @@ public class CanalAdapterLoader {
canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
} else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig.getBootstrapServers(),
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
canalClientConfig.getBootstrapServers(),
topic.getTopic(),
group.getGroupId(),
canalOuterAdapterGroups,

49
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java
View File

@@ -1,13 +1,16 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnectors;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
@@ -18,17 +21,19 @@ import com.alibaba.otter.canal.protocol.Message;
*/
public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private CanalClientConfig canalClientConfig;
private RocketMQCanalConnector connector;
private String topic;
private boolean flatMessage;
public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId,
List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.topic = topic;
this.flatMessage = flatMessage;
this.canalDestination = topic;
this.connector = RocketMQCanalConnectors.newRocketMQConnector(nameServers, topic, groupId, flatMessage);
this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, flatMessage);
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
}
@@ -36,6 +41,11 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
protected void process() {
while (!running)
;
ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry();
long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
while (running) {
try {
syncSwitch.get(canalDestination);
@@ -45,33 +55,12 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
connector.subscribe();
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
try {
Boolean status = syncSwitch.status(canalDestination);
if (status != null && !status) {
connector.disconnect();
break;
}
List<?> messages;
if (!flatMessage) {
messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
} else {
messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null) {
for (final Object message : messages) {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
}
}
connector.ack();
} catch (Throwable e) {
logger.error(e.getMessage(), e);
TimeUnit.SECONDS.sleep(1L);
Boolean status = syncSwitch.status(canalDestination);
if (status != null && !status) {
connector.disconnect();
break;
}
mqWriteOutData(retry, timeout, flatMessage, connector, workerExecutor);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);

96
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java
View File

@@ -2,12 +2,12 @@ package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.SocketAddress;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.*;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
@@ -20,10 +20,12 @@ import com.alibaba.otter.canal.protocol.Message;
*/
public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
private static final int BATCH_SIZE = 50;
private static final int SO_TIMEOUT = 0;
private static final int BATCH_SIZE = 50;
private static final int SO_TIMEOUT = 0;
private CanalConnector connector;
private CanalConnector connector;
private CanalClientConfig canalClientConfig;
/**
* 单台client适配器worker的构造方法
@@ -32,9 +34,10 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
* @param address canal-server地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(String canalDestination, SocketAddress address,
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalClientConfig = canalClientConfig;
this.canalDestination = canalDestination;
connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
}
@@ -46,10 +49,11 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
* @param zookeeperHosts zookeeper地址
* @param canalOuterAdapters 外部适配器组
*/
public CanalAdapterWorker(String canalDestination, String zookeeperHosts,
public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
List<List<OuterAdapter>> canalOuterAdapters){
super(canalOuterAdapters);
this.canalDestination = canalDestination;
this.canalClientConfig = canalClientConfig;
connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
}
@@ -58,6 +62,15 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
protected void process() {
while (!running)
; // waiting until running == true
ExecutorService workerExecutor = Executors.newSingleThreadExecutor();
int retry = canalClientConfig.getRetry() == null ? 1 : canalClientConfig.getRetry();
long timeout = canalClientConfig.getTimeout() == null ? 300000 : canalClientConfig.getTimeout(); // 默认超时5分钟
Integer batchSize = canalClientConfig.getBatchSize();
if (batchSize == null) {
batchSize = BATCH_SIZE;
}
while (running) {
try {
syncSwitch.get(canalDestination);
@@ -77,35 +90,50 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
break;
}
// server配置canal.instance.network.soTimeout(默认: 30s)
// 范围内未与server交互，server将关闭本次socket连接
Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据
long batchId = message.getId();
try {
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
Thread.sleep(1000);
} else {
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} batchSize: {} ",
this.canalDestination,
batchId,
size);
for (int i = 0; i < retry; i++) {
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
try {
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
Thread.sleep(500);
} else {
Future<Boolean> future = workerExecutor.submit(() -> {
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} batchSize: {} ",
canalDestination,
batchId,
size);
}
long begin = System.currentTimeMillis();
writeOut(message);
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} elapsed time: {} ms",
canalDestination,
batchId,
System.currentTimeMillis() - begin);
}
return true;
});
try {
future.get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
future.cancel(true);
throw e;
}
}
long begin = System.currentTimeMillis();
writeOut(message);
if (logger.isDebugEnabled()) {
logger.debug("destination: {} batchId: {} elapsed time: {} ms",
this.canalDestination,
batchId,
System.currentTimeMillis() - begin);
connector.ack(batchId); // 提交确认
break;
} catch (Exception e) {
if (i != retry - 1) {
connector.rollback(batchId); // 处理失败, 回滚数据
} else {
connector.ack(batchId);
}
logger.error("sync error!", e);
Thread.sleep(500);
}
connector.ack(batchId); // 提交确认
} catch (Exception e) {
connector.rollback(batchId); // 处理失败, 回滚数据
logger.error("sync error!", e);
Thread.sleep(500);
}
}
@@ -124,6 +152,8 @@ public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
}
}
}
workerExecutor.shutdown();
}
@Override

31
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -15,27 +15,30 @@ hbase.zookeeper.property.clientPort: 2181
hbase.zookeeper.znode.parent: /hbase
canal.conf:
canalServerHost: 127.0.0.1:11111
# canalServerHost: 127.0.0.1:11111
# zookeeperHosts: slave1:2181
# bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876
bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: true
canalInstances:
- instance: example
groups:
- outAdapters:
- name: logger
retry: 3
timeout: 20000
batchSize: 50
# canalInstances:
# - instance: example
# groups:
# - outAdapters:
# - name: logger
# - name: hbase
# properties:
# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
# mqTopics:
# - mqMode: kafka
# topic: example
# groups:
# - groupId: g2
# outAdapters:
# - name: logger
mqTopics:
- mqMode: kafka
topic: example
groups:
- groupId: g2
outAdapters:
- name: logger
# mqTopics:
# - mqMode: rocketmq
# topic: example