mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
模块化mq connector (#2562)
* add connector module, add kafka producer connector module for server * add rocketMQ producer connector module for server * add rabbitMQ producer connector module for server * optimize code, add annotation * add canal message consumer connector for canal-adapter module
This commit is contained in:
rewerma
104
connector/rocketmq-connector/pom.xml
Normal file
104
connector/rocketmq-connector/pom.xml
Normal file
@@ -0,0 +1,104 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>canal.connector</artifactId>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<version>1.1.5-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.rocketmq</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal connector rocketMQ module for otter ${project.version}</name>
|
||||
|
||||
<properties>
|
||||
<rocketmq_version>4.5.2</rocketmq_version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>connector.core</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-client</artifactId>
|
||||
<version>${rocketmq_version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-acl</artifactId>
|
||||
<version>${rocketmq_version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4</version>
|
||||
<configuration>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>1.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<tasks>
|
||||
<copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
|
||||
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
|
||||
<include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar"/>
|
||||
</fileset>
|
||||
</copy>
|
||||
<copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
|
||||
<fileset dir="${project.basedir}/target" erroronmissingdir="true">
|
||||
<include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar" />
|
||||
</fileset>
|
||||
</copy>
|
||||
</tasks>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.alibaba.otter.canal.connector.rocketmq.config;
|
||||
|
||||
/**
|
||||
* RocketMQ 配置常量类
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class RocketMQConstants {
|
||||
|
||||
public static final String ROOT = "rocketmq";
|
||||
|
||||
public static final String ROCKETMQ_PRODUCER_GROUP = ROOT + "." + "producer.group";
|
||||
public static final String ROCKETMQ_ENABLE_MESSAGE_TRACE = ROOT + "." + "enable.message.trace";
|
||||
public static final String ROCKETMQ_CUSTOMIZED_TRACE_TOPIC = ROOT + "." + "customized.trace.topic";
|
||||
public static final String ROCKETMQ_NAMESPACE = ROOT + "." + "namespace";
|
||||
public static final String ROCKETMQ_NAMESRV_ADDR = ROOT + "." + "namesrv.addr";
|
||||
public static final String ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED = ROOT + "." + "retry.times.when.send.failed";
|
||||
public static final String ROCKETMQ_VIP_CHANNEL_ENABLED = ROOT + "." + "vip.channel.enabled";
|
||||
|
||||
public static final String ROCKETMQ_ACCESS_CHANNEL = ROOT + "." + "access.channel";
|
||||
public static final String ROCKETMQ_BATCH_SIZE = ROOT + "." + "batch.size";
|
||||
public static final String ROCKETMQ_SUBSCRIBE_FILTER = ROOT + "." + "subscribe.filter";
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package com.alibaba.otter.canal.connector.rocketmq.config;
|
||||
|
||||
import com.alibaba.otter.canal.connector.core.config.MQProperties;
|
||||
|
||||
/**
|
||||
* RocketMQ 配置类
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class RocketMQProducerConfig extends MQProperties {
|
||||
|
||||
private String producerGroup;
|
||||
private boolean enableMessageTrace = false;
|
||||
private String customizedTraceTopic;
|
||||
private String namespace;
|
||||
private String namesrvAddr;
|
||||
private int retryTimesWhenSendFailed = 0;
|
||||
private boolean vipChannelEnabled = false;
|
||||
|
||||
public String getProducerGroup() {
|
||||
return producerGroup;
|
||||
}
|
||||
|
||||
public void setProducerGroup(String producerGroup) {
|
||||
this.producerGroup = producerGroup;
|
||||
}
|
||||
|
||||
public boolean isEnableMessageTrace() {
|
||||
return enableMessageTrace;
|
||||
}
|
||||
|
||||
public void setEnableMessageTrace(boolean enableMessageTrace) {
|
||||
this.enableMessageTrace = enableMessageTrace;
|
||||
}
|
||||
|
||||
public String getCustomizedTraceTopic() {
|
||||
return customizedTraceTopic;
|
||||
}
|
||||
|
||||
public void setCustomizedTraceTopic(String customizedTraceTopic) {
|
||||
this.customizedTraceTopic = customizedTraceTopic;
|
||||
}
|
||||
|
||||
public String getNamespace() {
|
||||
return namespace;
|
||||
}
|
||||
|
||||
public void setNamespace(String namespace) {
|
||||
this.namespace = namespace;
|
||||
}
|
||||
|
||||
public String getNamesrvAddr() {
|
||||
return namesrvAddr;
|
||||
}
|
||||
|
||||
public void setNamesrvAddr(String namesrvAddr) {
|
||||
this.namesrvAddr = namesrvAddr;
|
||||
}
|
||||
|
||||
public int getRetryTimesWhenSendFailed() {
|
||||
return retryTimesWhenSendFailed;
|
||||
}
|
||||
|
||||
public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) {
|
||||
this.retryTimesWhenSendFailed = retryTimesWhenSendFailed;
|
||||
}
|
||||
|
||||
public boolean isVipChannelEnabled() {
|
||||
return vipChannelEnabled;
|
||||
}
|
||||
|
||||
public void setVipChannelEnabled(boolean vipChannelEnabled) {
|
||||
this.vipChannelEnabled = vipChannelEnabled;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package com.alibaba.otter.canal.connector.rocketmq.consumer;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.rocketmq.acl.common.AclClientRPCHook;
|
||||
import org.apache.rocketmq.acl.common.SessionCredentials;
|
||||
import org.apache.rocketmq.client.AccessChannel;
|
||||
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
|
||||
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
|
||||
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
|
||||
import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
|
||||
import org.apache.rocketmq.client.exception.MQClientException;
|
||||
import org.apache.rocketmq.common.message.MessageExt;
|
||||
import org.apache.rocketmq.remoting.RPCHook;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
|
||||
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
|
||||
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
|
||||
import com.alibaba.otter.canal.connector.core.spi.SPI;
|
||||
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
|
||||
import com.alibaba.otter.canal.connector.core.util.MessageUtil;
|
||||
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
/**
|
||||
* RocketMQ consumer SPI 实现
|
||||
*
|
||||
* @author rewerma 2020-02-01
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SPI("rocketmq")
|
||||
public class CanalRocketMQConsumer implements CanalMsgConsumer {
|
||||
|
||||
private static final Logger logger = LoggerFactory
|
||||
.getLogger(CanalRocketMQConsumer.class);
|
||||
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
|
||||
|
||||
private String nameServer;
|
||||
private String topic;
|
||||
private String groupName;
|
||||
private volatile boolean connected = false;
|
||||
private DefaultMQPushConsumer rocketMQConsumer;
|
||||
private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
|
||||
private int batchSize = -1;
|
||||
private long batchProcessTimeout = 60 * 1000;
|
||||
private boolean flatMessage;
|
||||
private volatile ConsumerBatchMessage<CommonMessage> lastGetBatchMessage = null;
|
||||
private String accessKey;
|
||||
private String secretKey;
|
||||
private String customizedTraceTopic;
|
||||
private boolean enableMessageTrace = false;
|
||||
private String accessChannel;
|
||||
private String namespace;
|
||||
private String filter = "*";
|
||||
|
||||
@Override
|
||||
public void init(Properties properties, String topic, String groupName) {
|
||||
this.topic = topic;
|
||||
this.groupName = groupName;
|
||||
this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
|
||||
this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
|
||||
this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
|
||||
this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
|
||||
String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
|
||||
if (StringUtils.isNotEmpty(enableMessageTrace)) {
|
||||
this.enableMessageTrace = Boolean.parseBoolean(enableMessageTrace);
|
||||
}
|
||||
this.customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
|
||||
this.accessChannel = properties.getProperty(RocketMQConstants.ROCKETMQ_ACCESS_CHANNEL);
|
||||
this.namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
|
||||
this.nameServer = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
|
||||
String batchSize = properties.getProperty(RocketMQConstants.ROCKETMQ_BATCH_SIZE);
|
||||
if (StringUtils.isNotEmpty(batchSize)) {
|
||||
this.batchSize = Integer.parseInt(batchSize);
|
||||
}
|
||||
String subscribeFilter = properties.getProperty(RocketMQConstants.ROCKETMQ_SUBSCRIBE_FILTER);
|
||||
if (StringUtils.isNotEmpty(subscribeFilter)) {
|
||||
this.filter = subscribeFilter;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void connect() {
|
||||
RPCHook rpcHook = null;
|
||||
if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
|
||||
SessionCredentials sessionCredentials = new SessionCredentials();
|
||||
sessionCredentials.setAccessKey(accessKey);
|
||||
sessionCredentials.setSecretKey(secretKey);
|
||||
rpcHook = new AclClientRPCHook(sessionCredentials);
|
||||
}
|
||||
|
||||
rocketMQConsumer = new DefaultMQPushConsumer(groupName,
|
||||
rpcHook,
|
||||
new AllocateMessageQueueAveragely(),
|
||||
enableMessageTrace,
|
||||
customizedTraceTopic);
|
||||
rocketMQConsumer.setVipChannelEnabled(false);
|
||||
if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) {
|
||||
rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD);
|
||||
}
|
||||
|
||||
if (!StringUtils.isEmpty(this.namespace)) {
|
||||
rocketMQConsumer.setNamespace(this.namespace);
|
||||
}
|
||||
|
||||
if (!StringUtils.isBlank(nameServer)) {
|
||||
rocketMQConsumer.setNamesrvAddr(nameServer);
|
||||
}
|
||||
if (batchSize != -1) {
|
||||
rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
|
||||
}
|
||||
|
||||
try {
|
||||
if (rocketMQConsumer == null) {
|
||||
this.connect();
|
||||
}
|
||||
rocketMQConsumer.subscribe(this.topic, this.filter);
|
||||
rocketMQConsumer.registerMessageListener((MessageListenerOrderly) (messageExts, context) -> {
|
||||
context.setAutoCommit(true);
|
||||
boolean isSuccess = process(messageExts);
|
||||
if (isSuccess) {
|
||||
return ConsumeOrderlyStatus.SUCCESS;
|
||||
} else {
|
||||
return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
|
||||
}
|
||||
});
|
||||
rocketMQConsumer.start();
|
||||
} catch (MQClientException ex) {
|
||||
connected = false;
|
||||
logger.error("Start RocketMQ consumer error", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean process(List<MessageExt> messageExts) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Get Message: {}", messageExts);
|
||||
}
|
||||
List<CommonMessage> messageList = Lists.newArrayList();
|
||||
for (MessageExt messageExt : messageExts) {
|
||||
byte[] data = messageExt.getBody();
|
||||
if (data != null) {
|
||||
try {
|
||||
if (!flatMessage) {
|
||||
Message message = CanalMessageSerializerUtil.deserializer(data);
|
||||
messageList.addAll(MessageUtil.convert(message));
|
||||
} else {
|
||||
CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
|
||||
messageList.add(commonMessage);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
logger.error("Add message error", ex);
|
||||
throw new CanalClientException(ex);
|
||||
}
|
||||
} else {
|
||||
logger.warn("Received message data is null");
|
||||
}
|
||||
}
|
||||
ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
|
||||
try {
|
||||
messageBlockingQueue.put(batchMessage);
|
||||
} catch (InterruptedException e) {
|
||||
logger.error("Put message to queue error", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
boolean isCompleted;
|
||||
try {
|
||||
isCompleted = batchMessage.waitFinish(batchProcessTimeout);
|
||||
} catch (InterruptedException e) {
|
||||
logger.error("Interrupted when waiting messages to be finished.", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
boolean isSuccess = batchMessage.isSuccess();
|
||||
return isCompleted && isSuccess;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
|
||||
try {
|
||||
if (this.lastGetBatchMessage != null) {
|
||||
throw new CanalClientException("mq get/ack not support concurrent & async ack");
|
||||
}
|
||||
|
||||
ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
|
||||
if (batchMessage != null) {
|
||||
this.lastGetBatchMessage = batchMessage;
|
||||
return batchMessage.getData();
|
||||
}
|
||||
} catch (InterruptedException ex) {
|
||||
logger.warn("Get message timeout", ex);
|
||||
throw new CanalClientException("Failed to fetch the data after: " + timeout);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rollback() {
|
||||
try {
|
||||
if (this.lastGetBatchMessage != null) {
|
||||
this.lastGetBatchMessage.fail();
|
||||
}
|
||||
} finally {
|
||||
this.lastGetBatchMessage = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ack() {
|
||||
try {
|
||||
if (this.lastGetBatchMessage != null) {
|
||||
this.lastGetBatchMessage.ack();
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
if (this.lastGetBatchMessage != null) {
|
||||
this.lastGetBatchMessage.fail();
|
||||
}
|
||||
} finally {
|
||||
this.lastGetBatchMessage = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void disconnect() {
|
||||
rocketMQConsumer.unsubscribe(topic);
|
||||
rocketMQConsumer.shutdown();
|
||||
connected = false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.alibaba.otter.canal.connector.rocketmq.consumer;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class ConsumerBatchMessage<T> {
|
||||
|
||||
private final List<T> data;
|
||||
private CountDownLatch latch;
|
||||
private boolean hasFailure = false;
|
||||
|
||||
public ConsumerBatchMessage(List<T> data){
|
||||
this.data = data;
|
||||
latch = new CountDownLatch(1);
|
||||
}
|
||||
|
||||
public boolean waitFinish(long timeout) throws InterruptedException {
|
||||
return latch.await(timeout, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public boolean isSuccess() {
|
||||
return !hasFailure;
|
||||
}
|
||||
|
||||
public List<T> getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Countdown if the sub message is successful.
|
||||
*/
|
||||
public void ack() {
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Countdown and fail-fast if the sub message is failed.
|
||||
*/
|
||||
public void fail() {
|
||||
hasFailure = true;
|
||||
// fail fast
|
||||
long count = latch.getCount();
|
||||
for (int i = 0; i < count; i++) {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package com.alibaba.otter.canal.connector.rocketmq.producer;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.rocketmq.acl.common.AclClientRPCHook;
|
||||
import org.apache.rocketmq.acl.common.SessionCredentials;
|
||||
import org.apache.rocketmq.client.AccessChannel;
|
||||
import org.apache.rocketmq.client.exception.MQClientException;
|
||||
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
|
||||
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
|
||||
import org.apache.rocketmq.client.producer.DefaultMQProducer;
|
||||
import org.apache.rocketmq.client.producer.SendResult;
|
||||
import org.apache.rocketmq.common.message.Message;
|
||||
import org.apache.rocketmq.common.message.MessageQueue;
|
||||
import org.apache.rocketmq.remoting.RPCHook;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import com.alibaba.otter.canal.common.CanalException;
|
||||
import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
|
||||
import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
|
||||
import com.alibaba.otter.canal.connector.core.util.Callback;
|
||||
import com.alibaba.otter.canal.connector.core.producer.MQDestination;
|
||||
import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
|
||||
import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
|
||||
import com.alibaba.otter.canal.connector.core.spi.SPI;
|
||||
import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
|
||||
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
|
||||
import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
|
||||
/**
|
||||
* RocketMQ Producer SPI 实现
|
||||
*
|
||||
* @author rewerma 2020-01-27
|
||||
* @version 1.0.0
|
||||
*/
|
||||
@SPI("rocketmq")
|
||||
public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
|
||||
|
||||
private DefaultMQProducer defaultMQProducer;
|
||||
private static final String CLOUD_ACCESS_CHANNEL = "cloud";
|
||||
|
||||
@Override
|
||||
public void init(Properties properties) {
|
||||
RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
|
||||
this.mqProperties = rocketMQProperties;
|
||||
super.init(properties);
|
||||
loadRocketMQProperties(properties);
|
||||
|
||||
RPCHook rpcHook = null;
|
||||
if (mqProperties.getAliyunAccessKey().length() > 0
|
||||
&& mqProperties.getAliyunSecretKey().length() > 0) {
|
||||
SessionCredentials sessionCredentials = new SessionCredentials();
|
||||
sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
|
||||
sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
|
||||
rpcHook = new AclClientRPCHook(sessionCredentials);
|
||||
}
|
||||
|
||||
defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
|
||||
rpcHook,
|
||||
rocketMQProperties.isEnableMessageTrace(),
|
||||
rocketMQProperties.getCustomizedTraceTopic());
|
||||
if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
|
||||
defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
|
||||
}
|
||||
if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) {
|
||||
defaultMQProducer.setNamespace(rocketMQProperties.getNamespace());
|
||||
}
|
||||
defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr());
|
||||
defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed());
|
||||
defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled());
|
||||
logger.info("##Start RocketMQ producer##");
|
||||
try {
|
||||
defaultMQProducer.start();
|
||||
} catch (MQClientException ex) {
|
||||
throw new CanalException("Start RocketMQ producer error", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void loadRocketMQProperties(Properties properties) {
|
||||
RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
|
||||
|
||||
String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
|
||||
if (!StringUtils.isEmpty(producerGroup)) {
|
||||
rocketMQProperties.setProducerGroup(producerGroup);
|
||||
}
|
||||
String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
|
||||
if (!StringUtils.isEmpty(enableMessageTrace)) {
|
||||
rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
|
||||
}
|
||||
String customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
|
||||
if (!StringUtils.isEmpty(customizedTraceTopic)) {
|
||||
rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
|
||||
}
|
||||
String namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
|
||||
if (!StringUtils.isEmpty(namespace)) {
|
||||
rocketMQProperties.setNamespace(namespace);
|
||||
}
|
||||
String namesrvAddr = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
|
||||
if (!StringUtils.isEmpty(namesrvAddr)) {
|
||||
rocketMQProperties.setNamesrvAddr(namesrvAddr);
|
||||
}
|
||||
String retry = properties.getProperty(RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED);
|
||||
if (!StringUtils.isEmpty(retry)) {
|
||||
rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
|
||||
}
|
||||
String vipChannelEnabled = properties.getProperty(RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
|
||||
if (!StringUtils.isEmpty(vipChannelEnabled)) {
|
||||
rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
|
||||
ExecutorTemplate template = new ExecutorTemplate(executor);
|
||||
try {
|
||||
if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
|
||||
// 动态topic
|
||||
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
|
||||
.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
|
||||
|
||||
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
|
||||
String topicName = entry.getKey().replace('.', '_');
|
||||
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
|
||||
template.submit(() -> {
|
||||
try {
|
||||
send(destination, topicName, messageSub);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template.waitForResult();
|
||||
} else {
|
||||
send(destination, destination.getTopic(), message);
|
||||
}
|
||||
|
||||
callback.commit();
|
||||
} catch (Throwable e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
callback.rollback();
|
||||
} finally {
|
||||
template.clear();
|
||||
}
|
||||
}
|
||||
|
||||
public void send(final MQDestination destination, String topicName,
|
||||
com.alibaba.otter.canal.protocol.Message message) {
|
||||
if (!mqProperties.isFlatMessage()) {
|
||||
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
|
||||
// 并发构造
|
||||
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
|
||||
// 串行分区
|
||||
com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
|
||||
message.getId(),
|
||||
destination.getPartitionsNum(),
|
||||
destination.getPartitionHash(),
|
||||
mqProperties.isDatabaseHash());
|
||||
int length = messages.length;
|
||||
|
||||
ExecutorTemplate template = new ExecutorTemplate(executor);
|
||||
for (int i = 0; i < length; i++) {
|
||||
com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
|
||||
if (dataPartition != null) {
|
||||
final int index = i;
|
||||
template.submit(() -> {
|
||||
Message data = new Message(topicName,
|
||||
CanalMessageSerializerUtil.serializer(dataPartition,
|
||||
mqProperties.isFilterTransactionEntry()));
|
||||
sendMessage(data, index);
|
||||
});
|
||||
}
|
||||
}
|
||||
// 等所有分片发送完毕
|
||||
template.waitForResult();
|
||||
} else {
|
||||
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
|
||||
Message data = new Message(topicName,
|
||||
CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
|
||||
sendMessage(data, partition);
|
||||
}
|
||||
} else {
|
||||
// 并发构造
|
||||
MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
|
||||
// 串行分区
|
||||
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
|
||||
// 初始化分区合并队列
|
||||
if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
|
||||
List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
|
||||
for (int i = 0; i < destination.getPartitionsNum(); i++) {
|
||||
partitionFlatMessages.add(new ArrayList<>());
|
||||
}
|
||||
|
||||
for (FlatMessage flatMessage : flatMessages) {
|
||||
FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
|
||||
destination.getPartitionsNum(),
|
||||
destination.getPartitionHash(),
|
||||
mqProperties.isDatabaseHash());
|
||||
int length = partitionFlatMessage.length;
|
||||
for (int i = 0; i < length; i++) {
|
||||
partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
|
||||
}
|
||||
}
|
||||
|
||||
ExecutorTemplate template = new ExecutorTemplate(executor);
|
||||
for (int i = 0; i < partitionFlatMessages.size(); i++) {
|
||||
final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
|
||||
if (flatMessagePart != null) {
|
||||
final int index = i;
|
||||
template.submit(() -> {
|
||||
List<Message> messages = flatMessagePart.stream()
|
||||
.map(flatMessage -> new Message(topicName,
|
||||
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
|
||||
.collect(Collectors.toList());
|
||||
// 批量发送
|
||||
sendMessage(messages, index);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 批量等所有分区的结果
|
||||
template.waitForResult();
|
||||
} else {
|
||||
final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
|
||||
List<Message> messages = flatMessages.stream()
|
||||
.map(flatMessage -> new Message(topicName,
|
||||
JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
|
||||
.collect(Collectors.toList());
|
||||
// 批量发送
|
||||
sendMessage(messages, partition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void sendMessage(Message message, int partition) {
|
||||
try {
|
||||
SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> {
|
||||
if (partition > mqs.size()) {
|
||||
return mqs.get(partition % mqs.size());
|
||||
} else {
|
||||
return mqs.get(partition);
|
||||
}
|
||||
}, null);
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Send Message Result: {}", sendResult);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private void sendMessage(List<Message> messages, int partition) {
|
||||
if (messages.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 获取一下messageQueue
|
||||
DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
|
||||
TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
|
||||
if (topicInfo == null) {
|
||||
for (Message message : messages) {
|
||||
sendMessage(message, partition);
|
||||
}
|
||||
} else {
|
||||
// 批量发送
|
||||
List<MessageQueue> queues = topicInfo.getMessageQueueList();
|
||||
int size = queues.size();
|
||||
if (size <= 0) {
|
||||
// 可能是第一次创建
|
||||
for (Message message : messages) {
|
||||
sendMessage(message, partition);
|
||||
}
|
||||
} else {
|
||||
MessageQueue queue;
|
||||
if (partition > size) {
|
||||
queue = queues.get(partition % size);
|
||||
} else {
|
||||
queue = queues.get(partition);
|
||||
}
|
||||
|
||||
try {
|
||||
SendResult sendResult = this.defaultMQProducer.send(messages, queue);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Send Message Result: {}", sendResult);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
logger.info("## Stop RocketMQ producer##");
|
||||
this.defaultMQProducer.shutdown();
|
||||
super.stop();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
rocketmq=com.alibaba.otter.canal.connector.rocketmq.producer.CanalRocketMQProducer
|
||||
@@ -0,0 +1 @@
|
||||
rocketmq=com.alibaba.otter.canal.connector.rocketmq.consumer.CanalRocketMQConsumer
|
||||
Reference in New Issue
Block a user