chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support seamless docking with Alibaba Cloud RocketMQ commercial service (#1849)

Browse Source
This commit is contained in:
Heng Du
2019-05-27 23:51:32 +08:00
committed by agapple
parent 3b9ff26d1a
commit d0a83aad27
16 changed files with 276 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client/pom.xml
View File

@@ -106,8 +106,8 @@
<artifactId>rocketmq-client</artifactId>
</dependency>
<dependency>
<groupId>com.aliyun.openservices</groupId>
<artifactId>aliware-apache-rocketmq-cloud</artifactId>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
</dependency>
<!-- 客户端要使用请单独引入kafka-clients依赖 -->
<dependency>

46
client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java
View File

@@ -6,6 +6,9 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
@@ -24,8 +27,6 @@ import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.aliyun.openservices.apache.api.impl.authority.SessionCredentials;
import com.aliyun.openservices.apache.api.impl.rocketmq.ClientRPCHook;
import com.google.common.collect.Lists;
/**
@@ -40,7 +41,8 @@ import com.google.common.collect.Lists;
*/
public class RocketMQCanalConnector implements CanalMQConnector {
private static final Logger logger = LoggerFactory.getLogger(RocketMQCanalConnector.class);
private static final Logger logger = LoggerFactory.getLogger(RocketMQCanalConnector.class);
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
private String nameServer;
private String topic;
@@ -54,6 +56,26 @@ public class RocketMQCanalConnector implements CanalMQConnector {
private volatile ConsumerBatchMessage lastGetBatchMessage = null;
private String accessKey;
private String secretKey;
private String customizedTraceTopic;
private boolean enableMessageTrace = false;
private String accessChannel;
private String namespace;
public RocketMQCanalConnector(String nameServer, String topic, String groupName, String accessKey,
String secretKey, Integer batchSize, boolean flatMessage, boolean enableMessageTrace,
String customizedTraceTopic, String accessChannel, String namespace) {
this(nameServer, topic, groupName, accessKey, secretKey, batchSize, flatMessage, enableMessageTrace, customizedTraceTopic, accessChannel);
this.namespace = namespace;
}
public RocketMQCanalConnector(String nameServer, String topic, String groupName, String accessKey,
String secretKey, Integer batchSize, boolean flatMessage, boolean enableMessageTrace,
String customizedTraceTopic, String accessChannel) {
this(nameServer, topic, groupName, accessKey, secretKey, batchSize, flatMessage);
this.enableMessageTrace = enableMessageTrace;
this.customizedTraceTopic = customizedTraceTopic;
this.accessChannel = accessChannel;
}
public RocketMQCanalConnector(String nameServer, String topic, String groupName, Integer batchSize,
boolean flatMessage){
@@ -73,16 +95,24 @@ public class RocketMQCanalConnector implements CanalMQConnector {
}
public void connect() throws CanalClientException {
RPCHook rpcHook = null;
if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
SessionCredentials sessionCredentials = new SessionCredentials();
sessionCredentials.setAccessKey(accessKey);
sessionCredentials.setSecretKey(secretKey);
rpcHook = new ClientRPCHook(sessionCredentials);
rpcHook = new AclClientRPCHook(sessionCredentials);
}
rocketMQConsumer = new DefaultMQPushConsumer(groupName, rpcHook, new AllocateMessageQueueAveragely());
rocketMQConsumer = new DefaultMQPushConsumer(groupName, rpcHook, new AllocateMessageQueueAveragely(), enableMessageTrace, customizedTraceTopic);
rocketMQConsumer.setVipChannelEnabled(false);
if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) {
rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD);
}
if (!StringUtils.isEmpty(this.namespace)) {
rocketMQConsumer.setNamespace(this.namespace);
}
if (!StringUtils.isBlank(nameServer)) {
rocketMQConsumer.setNamesrvAddr(nameServer);
}
@@ -131,7 +161,9 @@ public class RocketMQCanalConnector implements CanalMQConnector {
}
private boolean process(List<MessageExt> messageExts) {
logger.info("Get Message:{}", messageExts);
if (logger.isDebugEnabled()) {
logger.debug("Get Message: {}", messageExts);
}
List messageList = Lists.newArrayList();
for (MessageExt messageExt : messageExts) {
byte[] data = messageExt.getBody();