chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add flat message support to rocketmq

Browse Source
This commit is contained in:
duhengforever
2018-10-28 16:09:31 +08:00
parent 80d3781d81
commit b484593869
8 changed files with 222 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java
View File

@@ -105,7 +105,7 @@ public class CanalAdapterLoader {
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig.getBootstrapServers(),
topic.getTopic(),
group.getGroupId(),
canalOuterAdapterGroups);
canalOuterAdapterGroups, canalClientConfig.getFlatMessage());
canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start();
} else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {

34
client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterRocketMQWorker.java
View File

@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.client.adapter.loader;
import com.alibaba.otter.canal.protocol.FlatMessage;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -23,16 +24,19 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
private RocketMQCanalConnector connector;
private String topic;
private String topic;
private boolean flatMessage;
public CanalAdapterRocketMQWorker(String nameServers, String topic, String groupId,
List<List<CanalOuterAdapter>> canalOuterAdapters){
List<List<CanalOuterAdapter>> canalOuterAdapters, boolean flatMessage) {
logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
this.canalOuterAdapters = canalOuterAdapters;
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
this.topic = topic;
this.flatMessage = flatMessage;
this.canalDestination = topic;
connector = RocketMQCanalConnectorProvider.newRocketMQConnector(nameServers, topic, groupId);
connector = RocketMQCanalConnectorProvider.newRocketMQConnector(nameServers, topic, groupId, flatMessage);
}
@Override
@@ -80,20 +84,32 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
while (running) {
try {
// switcher.get(); //等待开关开启
final Message message = connector.getWithoutAck(1);
Object message = null;
if (!flatMessage) {
message = connector.getWithoutAck(1);
} else {
message = connector.getFlatMessageWithoutAck();
}
if (message != null) {
final Object msg = message;
executor.submit(new Runnable() {
@Override
public void run() {
try {
writeOut(message, topic);
if (msg != null) {
if (msg instanceof Message) {
Message receive = (Message) msg;
writeOut(receive, topic);
connector.ack(receive.getId());
} else {
FlatMessage receive = (FlatMessage) msg;
writeOut(receive);
connector.ack(receive.getId());
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
connector.ack(message.getId());
}
});
} else {

2
client-launcher/src/main/resources/canal-client.yml
View File

@@ -1,7 +1,7 @@
#canalServerHost: 127.0.0.1:11111
#zookeeperHosts: slave1:2181
bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: true
flatMessage: false
#canalInstances:
#- instance: example