chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rabbitmq(amqp)MQ支持 (#2156)

Browse Source
This commit is contained in:
耐小心
2019-09-16 12:43:02 +08:00
committed by agapple
parent 5ec2087257
commit 79552e0a37
21 changed files with 817 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
server/pom.xml
View File

@@ -45,6 +45,14 @@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-acl</artifactId>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.mq-amqp</groupId>
<artifactId>mq-amqp-client</artifactId>
</dependency>
<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
<dependency>
<groupId>org.jboss.netty</groupId>

45
server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java
View File

@@ -33,6 +33,11 @@ public class MQProperties {
private boolean kerberosEnable = false; // kafka集群是否启动Kerberos认证
private String kerberosKrb5FilePath = ""; // 启动Kerberos认证时配置为krb5.conf文件的路径
private String kerberosJaasFilePath = ""; // 启动Kerberos认证时配置为jaas.conf文件的路径
private String username = ""; // rabbitmq 账号
private String password = ""; // rabbitmq 密码
private String vhost = ""; // rabbitmq 密码
private long aliyunUID = 0; // aliyun 用户ID rabbitmq 阿里云需要使用
private String exchange = ""; // rabbitmq 交换机
public static class CanalDestination {
@@ -276,6 +281,46 @@ public class MQProperties {
this.kerberosJaasFilePath = kerberosJaasFilePath;
}
public void setPassword(String password) {
this.password = password;
}
public String getPassword() {
return password;
}
public void setUsername(String username) {
this.username = username;
}
public String getUsername() {
return username;
}
public void setVhost(String vhost) {
this.vhost = vhost;
}
public String getVhost() {
return vhost;
}
public long getAliyunUID() {
return aliyunUID;
}
public void setAliyunUID(long aliyunUID) {
this.aliyunUID = aliyunUID;
}
public String getExchange() {
return exchange;
}
public void setExchange(String exchange) {
this.exchange = exchange;
}
@Override
public String toString() {
return "MQProperties{" + "servers='" + servers + '\'' + ", retries=" + retries + ", batchSize=" + batchSize

55
server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java Normal file
View File

@@ -0,0 +1,55 @@
/**
* aliyun amqp协议 账号类
* 暂不支持STS授权情况
*/
package com.alibaba.otter.canal.rabbitmq;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import com.alibaba.mq.amqp.utils.UserUtils;
import com.rabbitmq.client.impl.CredentialsProvider;
public class AliyunCredentialsProvider implements CredentialsProvider {
/**
* Access Key ID
*/
private final String AliyunAccessKey;
/**
* Access Key Secret
*/
private final String AliyunAccessSecret;
/**
* 资源主账号ID
*/
private final long resourceOwnerId;
public AliyunCredentialsProvider(final String accessKey, final String accessSecret, final long resourceOwnerId) {
this.AliyunAccessKey = accessKey;
this.AliyunAccessSecret = accessSecret;
this.resourceOwnerId = resourceOwnerId;
}
@Override
public String getUsername() {
return UserUtils.getUserName(AliyunAccessKey, resourceOwnerId);
}
@Override
public String getPassword() {
try {
return UserUtils.getPassord(AliyunAccessSecret);
} catch (InvalidKeyException | NoSuchAlgorithmException ignored) {
}
return null;
}
}

119
server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java Normal file
View File

@@ -0,0 +1,119 @@
package com.alibaba.otter.canal.rabbitmq;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.otter.canal.common.CanalMessageSerializer;
import com.alibaba.otter.canal.common.MQMessageUtils;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.spi.CanalMQProducer;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
public class CanalRabbitMQProducer implements CanalMQProducer {
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
private MQProperties mqProperties;
private Connection connect;
private Channel channel;
@Override
public void init(MQProperties mqProperties) {
this.mqProperties = mqProperties;
ConnectionFactory factory = new ConnectionFactory();
factory.setHost(mqProperties.getServers());
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
&& mqProperties.getAliyunUID() > 0) {
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
mqProperties.getAliyunSecretKey(),
mqProperties.getAliyunUID()));
} else {
factory.setUsername(mqProperties.getUsername());
factory.setPassword(mqProperties.getPassword());
}
factory.setVirtualHost(mqProperties.getVhost());
try {
connect = factory.newConnection();
channel = connect.createChannel();
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Start RabbitMQ producer error", ex);
}
}
@Override
public void send(MQProperties.CanalDestination canalDestination, Message message,
Callback callback) throws IOException {
try {
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
// 动态topic
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
.messageTopics(message, canalDestination.getTopic(), canalDestination.getDynamicTopic());
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
String topicName = entry.getKey().replace('.', '_');
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
send(canalDestination, topicName, messageSub);
}
} else {
send(canalDestination, canalDestination.getTopic(), message);
}
callback.commit();
} catch (Throwable e) {
callback.rollback();
}
}
private void send(MQProperties.CanalDestination canalDestination, String topicName,
Message messageSub) throws Exception {
if (!mqProperties.getFlatMessage()) {
byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry());
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
} else {
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(messageSub);
if (flatMessages != null) {
for (FlatMessage flatMessage : flatMessages) {
byte[] message = JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue).getBytes();
if (logger.isDebugEnabled()) {
logger.debug("send message:{} to destination:{}",
message,
canalDestination.getCanalDestination());
}
sendMessage(topicName, message);
}
}
}
}
private void sendMessage(String queueName, byte[] message) throws Exception {
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
channel.basicPublish(mqProperties.getExchange(), queueName, null, message);
}
@Override
public void stop() {
logger.info("## Stop RabbitMQ producer##");
try {
this.connect.close();
this.channel.close();
} catch (IOException | TimeoutException ex) {
throw new CanalServerException("Stop RabbitMQ producer error", ex);
}
}
}