rabbitmq(amqp)MQ支持 (#2156)

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -33,6 +33,13 @@ public class CanalClientConfig {
     // aliyun ak/sk
     private String             accessKey;
     private String             secretKey;
+    // rabbitmq 账号密码
+    private String             username;
+    private String             password;
+    // rabbitmq vhost
+    private String             vhost         = "/";
+
+    private Long               resourceOwnerId;
     // 是否启用消息轨迹
     private boolean            enableMessageTrace;
     // 在使用阿里云商业化mq服务时，如果想使用云上消息轨迹功能，请设置此配置为true
@@ -132,6 +139,38 @@ public class CanalClientConfig {
         this.secretKey = secretKey;
     }
 
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getVhost() {
+        return vhost;
+    }
+
+    public void setVhost(String vhost) {
+        this.vhost = vhost;
+    }
+
+    public Long getResourceOwnerId() {
+        return resourceOwnerId;
+    }
+
+    public void setResourceOwnerId(Long resourceOwnerId) {
+        this.resourceOwnerId = resourceOwnerId;
+    }
+
     public List<CanalAdapter> getCanalAdapters() {
         return canalAdapters;
     }
@@ -176,7 +215,7 @@ public class CanalClientConfig {
 
         private String      instance; // 实例名
 
-        private List<Group> groups;  // 适配器分组列表
+        private List<Group> groups;   // 适配器分组列表
 
         public String getInstance() {
             return instance;

client-adapter/launcher/pom.xml

@@ -71,6 +71,16 @@
             <version>1.1.1</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.rabbitmq</groupId>
+            <artifactId>amqp-client</artifactId>
+            <version>5.5.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.mq-amqp</groupId>
+            <artifactId>mq-amqp-client</artifactId>
+            <version>1.0.3</version>
+        </dependency>
         <!-- jdbc -->
         <dependency>
             <groupId>mysql</groupId>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -142,6 +142,29 @@ public class CanalAdapterLoader {
                         canalAdapter.getInstance() + "-" + group.getGroupId());
                 }
             }
+        } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
+            // 初始化canal-client-rabbitMQ的适配器
+            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+                    List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
+                    List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
+                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                        loadAdapter(config, canalOuterAdapters);
+                    }
+                    canalOuterAdapterGroups.add(canalOuterAdapters);
+                    CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
+                        canalOuterAdapterGroups,
+                        canalAdapter.getInstance(),
+                        group.getGroupId(),
+                        canalClientConfig.getFlatMessage());
+                    canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
+                    rabbitMQWork.start();
+
+                    logger.info("Start adapter for canal-client mq topic: {} succeed",
+                        canalAdapter.getInstance() + "-" + group.getGroupId());
+                }
+            }
+            // CanalAdapterRabbitMQWork
         }
     }
 

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java

@@ -0,0 +1,92 @@
+package com.alibaba.otter.canal.adapter.launcher.loader;
+
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector;
+import org.apache.kafka.common.errors.WakeupException;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker {
+
+    private RabbitMQCanalConnector connector;
+    private String                 topic;
+    private boolean                flatMessage;
+
+    public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List<List<OuterAdapter>> canalOuterAdapters,
+                                      String topic, String groupId, boolean flatMessage){
+        super(canalOuterAdapters);
+        this.canalClientConfig = canalClientConfig;
+        this.topic = topic;
+        this.flatMessage = flatMessage;
+        this.canalDestination = topic;
+        this.groupId = groupId;
+        this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(),
+            canalClientConfig.getVhost(),
+            topic,
+            canalClientConfig.getAccessKey(),
+            canalClientConfig.getSecretKey(),
+            canalClientConfig.getUsername(),
+            canalClientConfig.getPassword(),
+            canalClientConfig.getResourceOwnerId(),
+            flatMessage);
+    }
+
+    @Override
+    protected void process() {
+        while (!running) {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException ignored) {
+            }
+        }
+
+        ExecutorService workerExecutor = Util.newSingleThreadExecutor(5000L);
+        int retry = canalClientConfig.getRetries() == null
+                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
+        while (running) {
+            try {
+                syncSwitch.get(canalDestination);
+                logger.info("=============> Start to connect topic: {} <=============", this.topic);
+                connector.connect();
+                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
+                connector.subscribe();
+                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
+                while (running) {
+                    boolean status = syncSwitch.status(canalDestination);
+                    if (!status) {
+                        connector.disconnect();
+                        break;
+                    }
+                    if (retry == -1) {
+                        retry = Integer.MAX_VALUE;
+                    }
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector, workerExecutor)) {
+                            break;
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        workerExecutor.shutdown();
+
+        try {
+            connector.unsubscribe();
+        } catch (WakeupException e) {
+            // No-op. Continue process
+        }
+        connector.disconnect();
+        logger.info("=============> Disconnect topic: {} <=============", this.topic);
+
+    }
+}

client-adapter/launcher/src/main/resources/application.yml

@@ -18,6 +18,9 @@ canal.conf:
   timeout:
   accessKey:
   secretKey:
+  username:
+  password:
+  vhost:
 #  srcDataSources:
 #    defaultDS:
 #      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true