mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
rabbitmq(amqp)MQ支持 (#2156)
This commit is contained in:
耐小心
@@ -33,6 +33,11 @@ public class MQProperties {
|
||||
private boolean kerberosEnable = false; // kafka集群是否启动Kerberos认证
|
||||
private String kerberosKrb5FilePath = ""; // 启动Kerberos认证时配置为krb5.conf文件的路径
|
||||
private String kerberosJaasFilePath = ""; // 启动Kerberos认证时配置为jaas.conf文件的路径
|
||||
private String username = ""; // rabbitmq 账号
|
||||
private String password = ""; // rabbitmq 密码
|
||||
private String vhost = ""; // rabbitmq 密码
|
||||
private long aliyunUID = 0; // aliyun 用户ID rabbitmq 阿里云需要使用
|
||||
private String exchange = ""; // rabbitmq 交换机
|
||||
|
||||
public static class CanalDestination {
|
||||
|
||||
@@ -276,6 +281,46 @@ public class MQProperties {
|
||||
this.kerberosJaasFilePath = kerberosJaasFilePath;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setVhost(String vhost) {
|
||||
this.vhost = vhost;
|
||||
}
|
||||
|
||||
public String getVhost() {
|
||||
return vhost;
|
||||
}
|
||||
|
||||
public long getAliyunUID() {
|
||||
return aliyunUID;
|
||||
}
|
||||
|
||||
public void setAliyunUID(long aliyunUID) {
|
||||
this.aliyunUID = aliyunUID;
|
||||
}
|
||||
|
||||
public String getExchange() {
|
||||
return exchange;
|
||||
}
|
||||
|
||||
public void setExchange(String exchange) {
|
||||
this.exchange = exchange;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "MQProperties{" + "servers='" + servers + '\'' + ", retries=" + retries + ", batchSize=" + batchSize
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* aliyun amqp协议 账号类
|
||||
* 暂不支持STS授权情况
|
||||
*/
|
||||
package com.alibaba.otter.canal.rabbitmq;
|
||||
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
|
||||
import com.alibaba.mq.amqp.utils.UserUtils;
|
||||
import com.rabbitmq.client.impl.CredentialsProvider;
|
||||
|
||||
|
||||
public class AliyunCredentialsProvider implements CredentialsProvider {
|
||||
|
||||
|
||||
/**
|
||||
* Access Key ID
|
||||
*/
|
||||
private final String AliyunAccessKey;
|
||||
|
||||
/**
|
||||
* Access Key Secret
|
||||
*/
|
||||
private final String AliyunAccessSecret;
|
||||
|
||||
/**
|
||||
* 资源主账号ID
|
||||
*/
|
||||
private final long resourceOwnerId;
|
||||
|
||||
|
||||
public AliyunCredentialsProvider(final String accessKey, final String accessSecret, final long resourceOwnerId) {
|
||||
this.AliyunAccessKey = accessKey;
|
||||
this.AliyunAccessSecret = accessSecret;
|
||||
this.resourceOwnerId = resourceOwnerId;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getUsername() {
|
||||
return UserUtils.getUserName(AliyunAccessKey, resourceOwnerId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPassword() {
|
||||
try {
|
||||
return UserUtils.getPassord(AliyunAccessSecret);
|
||||
} catch (InvalidKeyException | NoSuchAlgorithmException ignored) {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package com.alibaba.otter.canal.rabbitmq;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import com.alibaba.otter.canal.common.CanalMessageSerializer;
|
||||
import com.alibaba.otter.canal.common.MQMessageUtils;
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.server.exception.CanalServerException;
|
||||
import com.alibaba.otter.canal.spi.CanalMQProducer;
|
||||
import com.rabbitmq.client.Channel;
|
||||
import com.rabbitmq.client.Connection;
|
||||
import com.rabbitmq.client.ConnectionFactory;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
public class CanalRabbitMQProducer implements CanalMQProducer {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
|
||||
private MQProperties mqProperties;
|
||||
private Connection connect;
|
||||
private Channel channel;
|
||||
|
||||
@Override
|
||||
public void init(MQProperties mqProperties) {
|
||||
this.mqProperties = mqProperties;
|
||||
ConnectionFactory factory = new ConnectionFactory();
|
||||
factory.setHost(mqProperties.getServers());
|
||||
if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
|
||||
&& mqProperties.getAliyunUID() > 0) {
|
||||
factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
|
||||
mqProperties.getAliyunSecretKey(),
|
||||
mqProperties.getAliyunUID()));
|
||||
} else {
|
||||
factory.setUsername(mqProperties.getUsername());
|
||||
factory.setPassword(mqProperties.getPassword());
|
||||
}
|
||||
factory.setVirtualHost(mqProperties.getVhost());
|
||||
try {
|
||||
connect = factory.newConnection();
|
||||
channel = connect.createChannel();
|
||||
// channel.exchangeDeclare(mqProperties.getExchange(), "topic");
|
||||
} catch (IOException | TimeoutException ex) {
|
||||
throw new CanalServerException("Start RabbitMQ producer error", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void send(MQProperties.CanalDestination canalDestination, Message message,
|
||||
Callback callback) throws IOException {
|
||||
try {
|
||||
if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
|
||||
// 动态topic
|
||||
Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
|
||||
.messageTopics(message, canalDestination.getTopic(), canalDestination.getDynamicTopic());
|
||||
|
||||
for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
|
||||
String topicName = entry.getKey().replace('.', '_');
|
||||
com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
|
||||
send(canalDestination, topicName, messageSub);
|
||||
}
|
||||
} else {
|
||||
send(canalDestination, canalDestination.getTopic(), message);
|
||||
}
|
||||
callback.commit();
|
||||
} catch (Throwable e) {
|
||||
callback.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
private void send(MQProperties.CanalDestination canalDestination, String topicName,
|
||||
Message messageSub) throws Exception {
|
||||
if (!mqProperties.getFlatMessage()) {
|
||||
byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry());
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
|
||||
}
|
||||
sendMessage(topicName, message);
|
||||
} else {
|
||||
List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(messageSub);
|
||||
if (flatMessages != null) {
|
||||
for (FlatMessage flatMessage : flatMessages) {
|
||||
byte[] message = JSON.toJSONString(flatMessage, SerializerFeature.WriteMapNullValue).getBytes();
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message:{} to destination:{}",
|
||||
message,
|
||||
canalDestination.getCanalDestination());
|
||||
}
|
||||
sendMessage(topicName, message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void sendMessage(String queueName, byte[] message) throws Exception {
|
||||
// tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
|
||||
channel.basicPublish(mqProperties.getExchange(), queueName, null, message);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
logger.info("## Stop RabbitMQ producer##");
|
||||
try {
|
||||
this.connect.close();
|
||||
this.channel.close();
|
||||
} catch (IOException | TimeoutException ex) {
|
||||
throw new CanalServerException("Stop RabbitMQ producer error", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user