模块化mq connector (#2562)

* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java

@@ -0,0 +1,20 @@
+package com.alibaba.otter.canal.connector.rabbitmq.config;
+
+/**
+ * RabbitMQ 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RabbitMQConstants {
+
+    public static final String ROOT                      = "rabbitmq";
+
+    public static final String RABBITMQ_HOST             = ROOT + "." + "host";
+    public static final String RABBITMQ_EXCHANGE         = ROOT + "." + "exchange";
+    public static final String RABBITMQ_VIRTUAL_HOST     = ROOT + "." + "virtual.host";
+    public static final String RABBITMQ_USERNAME         = ROOT + "." + "username";
+    public static final String RABBITMQ_PASSWORD         = ROOT + "." + "password";
+
+    public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId";
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java

@@ -0,0 +1,58 @@
+package com.alibaba.otter.canal.connector.rabbitmq.config;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * RabbitMQ 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RabbitMQProducerConfig extends MQProperties {
+
+    private String host;
+    private String virtualHost;
+    private String exchange;
+    private String username;
+    private String password;
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public String getVirtualHost() {
+        return virtualHost;
+    }
+
+    public void setVirtualHost(String virtualHost) {
+        this.virtualHost = virtualHost;
+    }
+
+    public String getExchange() {
+        return exchange;
+    }
+
+    public void setExchange(String exchange) {
+        this.exchange = exchange;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java

@@ -0,0 +1,214 @@
+package com.alibaba.otter.canal.connector.rabbitmq.consumer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
+import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+import com.rabbitmq.client.*;
+
+/**
+ * RabbitMQ consumer SPI 实现
+ *
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("rabbitmq")
+public class CanalRabbitMQConsumer implements CanalMsgConsumer {
+
+    private static final Logger                                logger              = LoggerFactory
+        .getLogger(CanalRabbitMQConsumer.class);
+
+    // 链接地址
+    private String                                             nameServer;
+    // 主机名
+    private String                                             vhost;
+    private String                                             queueName;
+
+    // 一些鉴权信息
+    private String                                             accessKey;
+    private String                                             secretKey;
+    private Long                                               resourceOwnerId;
+    private String                                             username;
+    private String                                             password;
+
+    private boolean                                            flatMessage;
+
+    private Connection                                         connect;
+    private Channel                                            channel;
+
+    private long                                               batchProcessTimeout = 60 * 1000;
+    private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
+    private volatile ConsumerBatchMessage<CommonMessage>       lastGetBatchMessage = null;
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        this.nameServer = properties.getProperty("rabbitmq.host");
+        this.vhost = properties.getProperty("rabbitmq.virtual.host");
+        this.queueName = topic;
+        this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        this.username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
+        this.password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
+        Long resourceOwnerIdPro = (Long) properties.get(RabbitMQConstants.RABBITMQ_RESOURCE_OWNERID);
+        if (resourceOwnerIdPro != null) {
+            this.resourceOwnerId = resourceOwnerIdPro;
+        }
+        this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
+    }
+
+    @Override
+    public void connect() {
+        ConnectionFactory factory = new ConnectionFactory();
+        if (accessKey.length() > 0 && secretKey.length() > 0) {
+            factory.setCredentialsProvider(new AliyunCredentialsProvider(accessKey, secretKey, resourceOwnerId));
+        } else {
+            factory.setUsername(username);
+            factory.setPassword(password);
+        }
+        factory.setHost(nameServer);
+        factory.setAutomaticRecoveryEnabled(true);
+        factory.setNetworkRecoveryInterval(5000);
+        factory.setVirtualHost(vhost);
+        try {
+            connect = factory.newConnection();
+            channel = connect.createChannel();
+        } catch (IOException | TimeoutException e) {
+            throw new CanalClientException("Start RabbitMQ producer error", e);
+        }
+
+        // 不存在连接 则重新连接
+        if (connect == null) {
+            this.connect();
+        }
+
+        Consumer consumer = new DefaultConsumer(channel) {
+
+            @Override
+            public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties,
+                                       byte[] body) throws IOException {
+
+                if (body != null) {
+                    channel.basicAck(envelope.getDeliveryTag(), process(body));
+                }
+            }
+        };
+        try {
+            channel.basicConsume(queueName, false, consumer);
+        } catch (IOException e) {
+            throw new CanalClientException("error", e);
+        }
+    }
+
+    private boolean process(byte[] messageData) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Get Message: {}", new String(messageData));
+        }
+        List<CommonMessage> messageList = Lists.newArrayList();
+        if (!flatMessage) {
+            Message message = CanalMessageSerializerUtil.deserializer(messageData);
+            messageList.addAll(MessageUtil.convert(message));
+        } else {
+            CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
+            messageList.add(commonMessage);
+        }
+        ConsumerBatchMessage<CommonMessage>  batchMessage = new ConsumerBatchMessage<>(messageList);
+        try {
+            messageBlockingQueue.put(batchMessage);
+        } catch (InterruptedException e) {
+            logger.error("Put message to queue error", e);
+            throw new RuntimeException(e);
+        }
+        boolean isCompleted;
+        try {
+            isCompleted = batchMessage.waitFinish(batchProcessTimeout);
+        } catch (InterruptedException e) {
+            logger.error("Interrupted when waiting messages to be finished.", e);
+            throw new RuntimeException(e);
+        }
+        boolean isSuccess = batchMessage.isSuccess();
+        return isCompleted && isSuccess;
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
+            }
+
+            ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
+            if (batchMessage != null) {
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
+            }
+        } catch (InterruptedException ex) {
+            logger.warn("Get message timeout", ex);
+            throw new CanalClientException("Failed to fetch the data after: " + timeout);
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.ack();
+            }
+        } catch (Throwable e) {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        if (connect != null) {
+            try {
+                connect.close();
+            } catch (IOException e) {
+                throw new CanalClientException("stop connect error", e);
+            }
+        }
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException | TimeoutException e) {
+                throw new CanalClientException("stop channel error", e);
+            }
+        }
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.connector.rabbitmq.consumer;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class ConsumerBatchMessage<T> {
+
+    private final List<T>  data;
+    private CountDownLatch         latch;
+    private boolean                hasFailure = false;
+
+    public ConsumerBatchMessage(List<T> data){
+        this.data = data;
+        latch = new CountDownLatch(1);
+    }
+
+    public boolean waitFinish(long timeout) throws InterruptedException {
+        return latch.await(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    public boolean isSuccess() {
+        return !hasFailure;
+    }
+
+    public List<T> getData() {
+        return data;
+    }
+
+    /**
+     * Countdown if the sub message is successful.
+     */
+    public void ack() {
+        latch.countDown();
+    }
+
+    /**
+     * Countdown and fail-fast if the sub message is failed.
+     */
+    public void fail() {
+        hasFailure = true;
+        // fail fast
+        long count = latch.getCount();
+        for (int i = 0; i < count; i++) {
+            latch.countDown();
+        }
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.connector.rabbitmq.producer;
+
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
+import com.alibaba.mq.amqp.utils.UserUtils;
+import com.rabbitmq.client.impl.CredentialsProvider;
+
+/**
+ * aliyun amqp协议 账号类
+ * 暂不支持STS授权情况
+ *
+ * @version 1.0.0
+ */
+public class AliyunCredentialsProvider implements CredentialsProvider {
+
+    /**
+     * Access Key ID
+     */
+    private final String AliyunAccessKey;
+
+    /**
+     * Access Key Secret
+     */
+    private final String AliyunAccessSecret;
+
+    /**
+     * 资源主账号ID
+     */
+    private final long   resourceOwnerId;
+
+    public AliyunCredentialsProvider(final String accessKey, final String accessSecret, final long resourceOwnerId){
+        this.AliyunAccessKey = accessKey;
+        this.AliyunAccessSecret = accessSecret;
+        this.resourceOwnerId = resourceOwnerId;
+    }
+
+    @Override
+    public String getUsername() {
+        return UserUtils.getUserName(AliyunAccessKey, resourceOwnerId);
+    }
+
+    @Override
+    public String getPassword() {
+        try {
+            return UserUtils.getPassord(AliyunAccessSecret);
+        } catch (InvalidKeyException | NoSuchAlgorithmException ignored) {
+        }
+        return null;
+    }
+
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java

@@ -0,0 +1,180 @@
+package com.alibaba.otter.canal.connector.rabbitmq.producer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+/**
+ * RabbitMQ Producer SPI 实现
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("rabbitmq")
+public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
+
+    private Connection          connect;
+    private Channel             channel;
+
+    @Override
+    public void init(Properties properties) {
+        RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig();
+        this.mqProperties = rabbitMQProperties;
+        super.init(properties);
+        loadRabbitMQProperties(properties);
+
+        ConnectionFactory factory = new ConnectionFactory();
+        factory.setHost(rabbitMQProperties.getHost());
+        if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
+            && mqProperties.getAliyunUid() > 0) {
+            factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
+                mqProperties.getAliyunSecretKey(),
+                mqProperties.getAliyunUid()));
+        } else {
+            factory.setUsername(rabbitMQProperties.getUsername());
+            factory.setPassword(rabbitMQProperties.getPassword());
+        }
+        factory.setVirtualHost(rabbitMQProperties.getVirtualHost());
+        try {
+            connect = factory.newConnection();
+            channel = connect.createChannel();
+            // channel.exchangeDeclare(mqProperties.getExchange(), "topic");
+        } catch (IOException | TimeoutException ex) {
+            throw new CanalException("Start RabbitMQ producer error", ex);
+        }
+    }
+
+    private void loadRabbitMQProperties(Properties properties) {
+        RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
+
+        String host = properties.getProperty(RabbitMQConstants.RABBITMQ_HOST);
+        if (!StringUtils.isEmpty(host)) {
+            rabbitMQProperties.setHost(host);
+        }
+        String vhost = properties.getProperty(RabbitMQConstants.RABBITMQ_VIRTUAL_HOST);
+        if (!StringUtils.isEmpty(vhost)) {
+            rabbitMQProperties.setVirtualHost(vhost);
+        }
+        String exchange = properties.getProperty(RabbitMQConstants.RABBITMQ_EXCHANGE);
+        if (!StringUtils.isEmpty(exchange)) {
+            rabbitMQProperties.setExchange(exchange);
+        }
+        String username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
+        if (!StringUtils.isEmpty(username)) {
+            rabbitMQProperties.setUsername(username);
+        }
+        String password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
+        if (!StringUtils.isEmpty(password)) {
+            rabbitMQProperties.setPassword(password);
+        }
+    }
+
+    @Override
+    public void send(final MQDestination destination, Message message, Callback callback) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map<String, Message> messageMap = MQMessageUtils
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
+
+                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                    final String topicName = entry.getKey().replace('.', '_');
+                    final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+
+                    template.submit(new Runnable() {
+
+                        @Override
+                        public void run() {
+                            send(destination, topicName, messageSub);
+                        }
+                    });
+                }
+
+                template.waitForResult();
+            } else {
+                send(destination, destination.getTopic(), message);
+            }
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    private void send(MQDestination canalDestination, String topicName, Message messageSub) {
+        if (!mqProperties.isFlatMessage()) {
+            byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry());
+            if (logger.isDebugEnabled()) {
+                logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
+            }
+            sendMessage(topicName, message);
+        } else {
+            // 并发构造
+            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
+            for (FlatMessage flatMessage : flatMessages) {
+                byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
+                if (logger.isDebugEnabled()) {
+                    logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
+                }
+                sendMessage(topicName, message);
+            }
+        }
+
+    }
+
+    private void sendMessage(String queueName, byte[] message) {
+        // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
+        try {
+            RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
+            channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message);
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RabbitMQ producer##");
+        try {
+            this.connect.close();
+            this.channel.close();
+            super.stop();
+        } catch (IOException | TimeoutException ex) {
+            throw new CanalException("Stop RabbitMQ producer error", ex);
+        }
+
+        super.stop();
+    }
+}

connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.producer.CanalRabbitMQProducer

connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.consumer.CanalRabbitMQConsumer