add support for polardbx multi binlog streams (#4660)

* AbstractCanalInstance的subscribeChange方法，增加对AbstractEventParser的类型判断，以支持用户的自定义CanalEventParser * MysqlEventParser对Gtid模式的处理需要考虑兼容性，支持系统平滑的从非gtid模式升级为gtid模式 * add support for polardbx multi binlog streams #4657 * support auto generate instance names #4657
2026-11-04 00:30:15 +08:00 · 2023-03-23 13:46:10 +08:00
parent 5d7d19fdb1
commit a04a1912a8
13 changed files with 142 additions and 25 deletions
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
@@ -30,6 +30,7 @@ public class CanalConstants {
    public static final String CANAL_WITHOUT_NETTY                  = ROOT + "." + "withoutNetty";

    public static final String CANAL_DESTINATIONS                   = ROOT + "." + "destinations";
+    public static final String CANAL_DESTINATIONS_EXPR              = ROOT + "." + "destinations.expr";
    public static final String CANAL_AUTO_SCAN                      = ROOT + "." + "auto.scan";
    public static final String CANAL_AUTO_SCAN_INTERVAL             = ROOT + "." + "auto.scan.interval";
    public static final String CANAL_CONF_DIR                       = ROOT + "." + "conf.dir";
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
@@ -1,18 +1,5 @@
 package com.alibaba.otter.canal.deployer;

-import java.util.Map;
-import java.util.Properties;
-
-import org.I0Itec.zkclient.IZkStateListener;
-import org.I0Itec.zkclient.exception.ZkNoNodeException;
-import org.I0Itec.zkclient.exception.ZkNodeExistsException;
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
 import com.alibaba.otter.canal.common.utils.AddressUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
@@ -36,6 +23,26 @@ import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
 import com.google.common.collect.MigrateMap;
+import org.I0Itec.zkclient.IZkStateListener;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static com.alibaba.otter.canal.deployer.CanalConstants.CANAL_DESTINATIONS;
+import static com.alibaba.otter.canal.deployer.CanalConstants.CANAL_DESTINATIONS_EXPR;

 /**
 * canal调度控制器
@@ -390,7 +397,7 @@ public class CanalController {
    }

    private void initInstanceConfig(Properties properties) {
-        String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+        String destinationStr = getDestinations(properties);
        String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT);

        for (String destination : destinations) {
@@ -461,6 +468,40 @@ public class CanalController {
        return StringUtils.trim(value);
    }

+    public static String getDestinations(Properties properties) {
+        String expr = getProperty(properties, CANAL_DESTINATIONS_EXPR);
+        if (StringUtils.isNotBlank(expr)) {
+            return parseExpr(expr);
+        } else {
+            return getProperty(properties, CANAL_DESTINATIONS);
+        }
+    }
+
+    private static String parseExpr(String expr) {
+        String prefix = StringUtils.substringBefore(expr, "{");
+        String range = StringUtils.substringAfter(expr, "{");
+        range = StringUtils.substringBefore(range, "}");
+
+        String regex = "(\\d+)-(\\d+)";
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(range);
+        if (matcher.find()) {
+            String head = matcher.group(1);
+            String tail = matcher.group(2);
+            int start = Integer.parseInt(head);
+            int end = Integer.parseInt(tail);
+
+            List<String> list = new ArrayList<>();
+            for (int i = start; i <= end; i++) {
+                String d = prefix + i;
+                list.add(d);
+            }
+            return list.stream().map(Object::toString).collect(Collectors.joining(","));
+        } else {
+            throw new CanalServerException("invalid destinations expr " + expr);
+        }
+    }
+
    public void start() throws Throwable {
        logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port);
        // 创建整个canal的工作节点
--- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java
+++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java
@@ -102,7 +102,7 @@ public class CanalStarter {

        if (canalMQProducer != null) {
            canalMQStarter = new CanalMQStarter(canalMQProducer);
-            String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+            String destinations = CanalController.getDestinations(properties);
            canalMQStarter.start(destinations);
            controller.setCanalMQStarter(canalMQStarter);
        }
--- a/deployer/src/main/resources/example/instance.properties
+++ b/deployer/src/main/resources/example/instance.properties
@@ -56,4 +56,7 @@ canal.mq.partition=0
 #canal.mq.partitionsNum=3
 #canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
 #canal.mq.partitionHash=test.table:id^name,.*\\..*
+#
+# multi stream for polardbx
+canal.instance.multi.stream.on=false
 #################################################
--- a/deployer/src/main/resources/spring/default-instance.xml
+++ b/deployer/src/main/resources/spring/default-instance.xml
@@ -197,6 +197,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>

 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
--- a/deployer/src/main/resources/spring/file-instance.xml
+++ b/deployer/src/main/resources/spring/file-instance.xml
@@ -183,6 +183,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>

 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
--- a/deployer/src/main/resources/spring/group-instance.xml
+++ b/deployer/src/main/resources/spring/group-instance.xml
@@ -170,6 +170,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>

 	<bean id="eventParser2" parent="baseEventParser">
@@ -278,6 +279,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>

    <bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">
--- a/deployer/src/main/resources/spring/memory-instance.xml
+++ b/deployer/src/main/resources/spring/memory-instance.xml
@@ -171,6 +171,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />

 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>

 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">