chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

增加扁平Message用于卡夫卡数据投递

Browse Source
This commit is contained in:
mcy
2018-09-14 11:22:47 +08:00
parent a6798c61df
commit e8e0032404
19 changed files with 686 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java
View File

@@ -7,10 +7,13 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -47,9 +50,16 @@ public abstract class AbstractCanalAdapterWorker {
public Boolean call() {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
for (CanalOuterAdapter c : adapters) {
for (final CanalOuterAdapter c : adapters) {
long begin = System.currentTimeMillis();
c.writeOut(message);
MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
@Override
public void accept(Dml dml) {
c.writeOut(dml);
}
});
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
c.getClass().getName(),
@@ -71,7 +81,49 @@ public abstract class AbstractCanalAdapterWorker {
logger.error("Outer adapter write failed");
}
} catch (InterruptedException | ExecutionException e) {
logger.error(e.getMessage(), e);
// ignore
}
}
}
}
protected void writeOut(final FlatMessage flatMessage) {
List<Future<Boolean>> futures = new ArrayList<>();
// 组间适配器并行运行
for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
final List<CanalOuterAdapter> adapters = outerAdapters;
futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
@Override
public Boolean call() {
try {
// 组内适配器穿行运行，尽量不要配置组内适配器
for (CanalOuterAdapter c : adapters) {
long begin = System.currentTimeMillis();
Dml dml = MessageUtil.flatMessage2Dml(flatMessage);
c.writeOut(dml);
if (logger.isDebugEnabled()) {
logger.debug("{} elapsed time: {}",
c.getClass().getName(),
(System.currentTimeMillis() - begin));
}
}
return true;
} catch (Exception e) {
return false;
}
}
}));
// 等待所有适配器写入完成
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
for (Future<Boolean> f : futures) {
try {
if (!f.get()) {
logger.error("Outer adapter write failed");
}
} catch (InterruptedException | ExecutionException e) {
// ignore
}
}
}

99
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java
View File

@@ -12,6 +12,7 @@ import org.apache.kafka.common.errors.WakeupException;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -26,14 +27,17 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
private String topic;
public CanalAdapterKafkaWorker(String zkServers, String bootstrapServers, String topic, String groupId,
List<List<CanalOuterAdapter>> canalOuterAdapters){
private boolean flatMessage;
public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
List<List<CanalOuterAdapter>> canalOuterAdapters, boolean flatMessage){
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
this.topic = topic;
this.canalDestination = topic;
connector = KafkaCanalConnectors.newKafkaConnector(zkServers, bootstrapServers, topic, null, groupId);
connector.setSessionTimeout(5L, TimeUnit.MINUTES);
this.flatMessage = flatMessage;
connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
// connector.setSessionTimeout(1L, TimeUnit.MINUTES);
// super.initSwitcher(topic);
}
@@ -104,56 +108,47 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
try {
// switcher.get(); //等待开关开启
final Message message = connector.getWithoutAck();
List<?> messages;
if (!flatMessage) {
messages = connector.getWithoutAck();
} else {
messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS);
}
if (messages != null) {
for (final Object message : messages) {
executing.set(true);
if (message != null) {
executor.submit(new Runnable() {
executing.set(true);
if (message != null) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
if (message instanceof FlatMessage) {
writeOut((FlatMessage) message);
} else {
writeOut((Message) message);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
executing.compareAndSet(true, false);
}
}
});
@Override
public void run() {
try {
if (logger.isDebugEnabled()) {
logger.debug("topic: {} batchId: {} batchSize: {} ",
topic,
message.getId(),
message.getEntries().size());
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
long currentTS = System.currentTimeMillis();
while (executing.get()) {
// 大于10秒未消费完ack一次keep alive
if (System.currentTimeMillis() - currentTS > 10000) {
connector.ack();
currentTS = System.currentTimeMillis();
}
long begin = System.currentTimeMillis();
writeOut(message);
long now = System.currentTimeMillis();
if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) {
logger.error("topic: {} batchId {} elapsed time: {} ms",
topic,
message.getId(),
now - begin);
}
if (logger.isDebugEnabled()) {
logger.debug("topic: {} batchId {} elapsed time: {} ms",
topic,
message.getId(),
now - begin);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
executing.compareAndSet(true, false);
}
}
});
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
long currentTS = System.currentTimeMillis();
while (executing.get()) {
// 大于1分钟未消费完ack一次keep alive
if (System.currentTimeMillis() - currentTS > 60000) {
connector.ack();
currentTS = System.currentTimeMillis();
}
}
} else {
connector.ack();
}
connector.ack();
} catch (CommitFailedException e) {
logger.warn(e.getMessage());
} catch (Exception e) {
@@ -161,19 +156,23 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
TimeUnit.SECONDS.sleep(1L);
}
}
} catch (Exception e) {
} catch (
Exception e) {
logger.error(e.getMessage(), e);
}
}
executor.shutdown();
try {
try
{
connector.unsubscribe();
} catch (WakeupException e) {
// No-op. Continue process
}
connector.disconnnect();
connector.disconnect();
logger.info("=============> Disconnect topic: {} <=============", this.topic);
}
}

7
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java
View File

@@ -59,6 +59,8 @@ public class CanalAdapterLoader {
}
String zkHosts = this.canalClientConfig.getZookeeperHosts();
boolean flatMessage = this.canalClientConfig.getFlatMessage();
// if (zkHosts == null && sa == null) {
// throw new RuntimeException("Blank config property: canalServerHost or
// zookeeperHosts");
@@ -104,11 +106,12 @@ public class CanalAdapterLoader {
canalOuterAdapterGroups.add(canalOuterAdapters);
// String zkServers = canalClientConfig.getZookeeperHosts();
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(zkHosts,
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(
canalClientConfig.getBootstrapServers(),
kafkaTopic.getTopic(),
group.getGroupId(),
canalOuterAdapterGroups);
canalOuterAdapterGroups,
flatMessage);
canalKafkaWorkers.put(kafkaTopic.getTopic() + "-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start();
logger.info("Start adapter for canal-client kafka topic: {} succeed",

31
client-launcher/src/main/resources/canal-client.yml
View File

@@ -1,21 +1,22 @@
canalServerHost: 127.0.0.1:11111
#canalServerHost: 127.0.0.1:11111
#zookeeperHosts: slave1:2181
#bootstrapServers: slave1:6667,slave2:6667
bootstrapServers: slave1:6667,slave2:6667
flatMessage: true
canalInstances:
- instance: example
adapterGroups:
- outAdapters:
- name: logger
- name: hbase
hosts: slave1:2181
properties: {znodeParent: "/hbase-unsecure"}
#kafkaTopics:
#- topic: example
# groups:
# - groupId: example_g1
# outAdapters:
#canalInstances:
#- instance: example
# adapterGroups:
# - outAdapters:
# - name: logger
# - name: hbase
# hosts: slave1:2181
# properties: {znodeParent: "/hbase-unsecure"}
kafkaTopics:
- topic: example
groups:
- groupId: egroup
outAdapters:
- name: logger
# - name: hbase
# hosts: slave1:2181
# properties: {znodeParent: "/hbase-unsecure"}