fixed format

* add support for listener name which is described in https://pulsar.apache.org/docs/next/concepts-multiple-advertised-listeners/

* upgrade for CVE-2023-20883  CVE-2022-1471  CVE-2023-20860

* add   enableChunking

* downgrade  pulsar client version

* fix

* 增加压缩算法

admin/pom.xml

@@ -36,7 +36,7 @@
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>
-                <version>2.5.4</version>
+                <version>2.5.15</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -48,7 +48,12 @@
             <dependency>
                 <groupId>io.ebean</groupId>
                 <artifactId>ebean</artifactId>
-                <version>11.41.1</version>
+                <version>11.45.1</version>
+            </dependency>
+            <dependency>
+                <groupId>org.yaml</groupId>
+                <artifactId>snakeyaml</artifactId>
+                <version>2.0</version>
             </dependency>
         </dependencies>
     </dependencyManagement>

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseEtlService.java

@@ -13,10 +13,7 @@ import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
 import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig.DbMapping;
 import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
+import com.alibaba.otter.canal.client.adapter.support.*;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.Util;
 
 /**
  * ClickHouse ETL 操作业务类
@@ -42,7 +39,8 @@ public class ClickHouseEtlService extends AbstractEtlService {
      */
     public EtlResult importData(List<String> params) {
         DbMapping dbMapping = config.getDbMapping();
-        String sql = "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable();
+        DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType());
         return importData(sql, params);
     }
 

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java

@@ -178,8 +178,7 @@ public class Util {
                         // ignore
                     }
                 }
-                }
+            });
-        );
     }
 
     public static ThreadPoolExecutor newSingleDaemonThreadExecutor(long keepAliveTime) {
@@ -327,4 +326,56 @@ public class Util {
 
         return null;
     }
+
+    /**
+     * Check if the time string has millisecond or microsecond units
+     * 
+     * @param timeStr time string
+     * @return boolean
+     */
+    public static boolean isAccuracyOverSecond(String timeStr) {
+        if (StringUtils.isEmpty(timeStr)) {
+            return false;
+        }
+        String[] times = StringUtils.split(timeStr, ".");
+        return times.length > 1 && !times[times.length - 1].isEmpty();
+    }
+
+    /**
+     * Check if the datetime string has microsecond or nanosecond units
+     * 
+     * @param datetimeStr datetime string
+     * @return boolean
+     */
+    public static boolean isAccuracyOverMillisecond(String datetimeStr) {
+        if (StringUtils.isEmpty(datetimeStr)) {
+            return false;
+        }
+        String[] times = StringUtils.split(datetimeStr, ".");
+        return times.length > 1 && times[times.length - 1].length() > 3;
+    }
+
+    /**
+     * MySQL has fractional seconds support for TIME, DATETIME, and TIMESTAMP
+     * values, with up to microseconds (6 digits) precision. ISO-8601 standard
+     * format is with up to nanoseconds (9 digits) precision, which is sufficient
+     * for storing MySQL time-related data.
+     * 
+     * @param datetimeStr datetime string
+     * @return LocalDateTime
+     */
+    public static LocalDateTime parseISOLocalDateTime(String datetimeStr) {
+        if (StringUtils.isEmpty(datetimeStr)) {
+            return null;
+        }
+        datetimeStr = datetimeStr.trim();
+        try {
+            // Replace SPACE in middle with `T` to comply with ISO-8601 standard format
+            return LocalDateTime.parse(datetimeStr.replace(" ", "T"));
+        } catch (Exception e) {
+            logger.error("Convert datetime string to ISOLocalDateTime fail:{}", datetimeStr, e);
+        }
+        return null;
+    }
+
 }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java

@@ -4,9 +4,11 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+
 import org.apache.commons.lang.StringUtils;
 import org.springframework.boot.context.properties.bind.Bindable;
 import org.springframework.boot.context.properties.bind.Binder;
@@ -18,10 +20,14 @@ import org.springframework.core.env.PropertySource;
 import org.springframework.core.io.ByteArrayResource;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.core.io.Resource;
+import org.springframework.util.PropertyPlaceholderHelper;
 
 public class YamlUtils {
 
     public static <T> T resourceYmlToObj(String resource, String prefix, Class<T> clazz) {
+        if (!StringUtils.startsWithIgnoreCase(resource, "classpath:")) {
+            resource = "classpath:" + resource;
+        }
         ClassPathResource classPathResource = new ClassPathResource(resource);
 
         String content;
@@ -87,7 +93,13 @@ public class YamlUtils {
             }
 
             ConfigurationPropertySource sources = new MapConfigurationPropertySource(properties);
-            Binder binder = new Binder(sources);
+            PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}");
+            Binder binder = new Binder(Arrays.asList(sources), value -> {
+                if (value instanceof String) {
+                    return propertyPlaceholderHelper.replacePlaceholders((String) value, baseProperties);
+                }
+                return value;
+            });
             return binder.bind(prefix, Bindable.of(clazz)).get();
         } catch (Exception e) {
             throw new RuntimeException(e);

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java

@@ -2,11 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -318,11 +314,15 @@ public class ES7xTemplate implements ESTemplate {
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
+            if (columnItem.getOwner() == null || columnItem.getColumnName() == null) {
+                continue;
+            }
+
             if (!columnItem.getOwner().equals(owner)) {
                 continue;
             }
-            String columnName = columnItem.getColumnName();
 
+            String columnName = columnItem.getColumnName();
             if (fieldItem.getFieldName().equals(idFieldName)) {
                 resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
             }

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ES8xTemplate.java

@@ -65,8 +65,9 @@ public class ES8xTemplate implements ESTemplate {
         if (mapping.getId() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                ESUpdateRequest updateRequest = esConnection.new ES8xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest updateRequest = esConnection.new ES8xUpdateRequest(mapping.getIndex(), pkVal.toString())
-                    pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+                    .setDoc(esFieldData)
+                    .setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     updateRequest.setRouting(parentVal);
                 }

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ESConnection.java

@@ -1,18 +1,29 @@
 package com.alibaba.otter.canal.client.adapter.es8x.support;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
 import java.util.Arrays;
 import java.util.Map;
 
+import javax.net.ssl.SSLContext;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -51,6 +62,47 @@ public class ESConnection {
     private RestHighLevelClient restHighLevelClient;
 
     public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException{
+        String caPath = properties.get("security.ca.path");
+        if (StringUtils.isNotEmpty(caPath)) {
+            connectEsWithCa(hosts, properties, caPath);
+        } else {
+            connectEsWithoutCa(hosts, properties);
+        }
+    }
+
+    private void connectEsWithCa(String[] hosts, Map<String, String> properties, String caPath) {
+        Path caCertificatePath = Paths.get(caPath);
+        try (InputStream is = Files.newInputStream(caCertificatePath)) {
+            CertificateFactory factory = CertificateFactory.getInstance("X.509");
+            Certificate trustedCa = factory.generateCertificate(is);
+            KeyStore trustStore = KeyStore.getInstance("pkcs12");
+            trustStore.load(null, null);
+            trustStore.setCertificateEntry("ca", trustedCa);
+            SSLContextBuilder sslContextBuilder = SSLContexts.custom().loadTrustMaterial(trustStore, null);
+            final SSLContext sslContext = sslContextBuilder.build();
+
+            HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
+            RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
+            String nameAndPwd = properties.get("security.auth");
+            if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
+                String[] nameAndPwdArr = nameAndPwd.split(":");
+                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+                credentialsProvider.setCredentials(AuthScope.ANY,
+                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
+                restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> {
+                    httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+                    return httpClientBuilder.setSSLContext(sslContext);
+                });
+            }
+            restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build())
+                .setApiCompatibilityMode(true)
+                .build();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void connectEsWithoutCa(String[] hosts, Map<String, String> properties) {
         HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
         RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
         String nameAndPwd = properties.get("security.auth");
@@ -280,9 +332,7 @@ public class ESConnection {
         private BulkRequest        bulkRequest;
 
         public ES8xBulkRequest(){
-
             bulkRequest = new BulkRequest();
-
         }
 
         public void resetBulk() {

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduEtlService.java

@@ -2,11 +2,7 @@ package com.alibaba.otter.canal.client.adapter.kudu.service;
 
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.sql.DataSource;
@@ -45,13 +41,13 @@ public class KuduEtlService extends AbstractEtlService {
         boolean flag = kuduTemplate.tableExists(kuduMapping.getTargetTable());
         // 表不存在，停止导入
         if (!flag) {
-            logger.info("{} is don't hava,please check your kudu table !", kuduMapping.getTargetTable());
+            logger.info("{} is don't exist , please check your kudu table !", kuduMapping.getTargetTable());
             errMsg.add(kuduMapping.getTargetTable() + " is don't hava,please check your kudu table !");
             etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
             return etlResult;
         }
         logger.info("{} etl is starting!", kuduMapping.getTargetTable());
-        String sql = "SELECT * FROM " + kuduMapping.getDatabase() + "." + kuduMapping.getTable();
+        String sql = "SELECT * FROM " + SyncUtil.getDbTableName(kuduMapping);
         return importData(sql, params);
     }
 

client-adapter/launcher/pom.xml

@@ -17,7 +17,7 @@
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>
-                <version>2.5.4</version>
+                <version>2.5.15</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>

client-adapter/launcher/src/main/bin/restart.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+args=$@
+
 case $(uname) in
 Linux)
   bin_abs_path=$(readlink -f $(dirname $0))
@@ -9,5 +11,5 @@ Linux)
   ;;
 esac
 
-sh "$bin_abs_path"/stop.sh
+sh "$bin_abs_path"/stop.sh $args
-sh "$bin_abs_path"/startup.sh
+sh "$bin_abs_path"/startup.sh $args

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -52,7 +52,7 @@ public class AdapterProcessor {
     private SyncSwitch                      syncSwitch;
 
     public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
-        List<List<OuterAdapter>> canalOuterAdapters) {
+                            List<List<OuterAdapter>> canalOuterAdapters){
         this.canalClientConfig = canalClientConfig;
         this.canalDestination = destination;
         this.groupId = groupId;
@@ -63,8 +63,11 @@ public class AdapterProcessor {
 
         // load connector consumer
         ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
-        canalMsgConsumer = new ProxyCanalMsgConsumer(loader
+        // see https://github.com/alibaba/canal/pull/5175
-            .getExtension(canalClientConfig.getMode().toLowerCase(), destination, CONNECTOR_SPI_DIR,
+        String key = destination + "_" + groupId;
+        canalMsgConsumer = new ProxyCanalMsgConsumer(loader.getExtension(canalClientConfig.getMode().toLowerCase(),
+            key,
+            CONNECTOR_SPI_DIR,
             CONNECTOR_STANDBY_SPI_DIR));
 
         Properties properties = canalClientConfig.getConsumerProperties();
@@ -224,7 +227,7 @@ public class AdapterProcessor {
                                     logger.error("finish turn off switch of destination:" + canalDestination);
                                 } else {
                                     canalMsgConsumer.ack();
-                                    logger.error(e.getMessage() + " Error sync but ACK!");
+                                    logger.error(e.getMessage() + " Error sync but ACK!", e);
                                 }
                             }
                             Thread.sleep(500);

client-adapter/launcher/src/main/resources/application.yml

@@ -91,6 +91,7 @@ canal.conf:
 #        hosts: 127.0.0.1:9300 # 127.0.0.1:9200 for rest mode
 #        properties:
 #          mode: transport # or rest
+#          # security.ca.path: /etc/es8/ca.crt
 #          # security.auth: test:123456 #  only used for rest mode
 #          cluster.name: elasticsearch
 #      - name: kudu

client-adapter/pom.xml

@@ -117,7 +117,7 @@
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot</artifactId>
-                <version>2.5.4</version>
+                <version>2.5.15</version>
             </dependency>
             <dependency>
                 <groupId>com.h2database</groupId>
@@ -238,7 +238,7 @@
             <dependency>
                 <groupId>org.yaml</groupId>
                 <artifactId>snakeyaml</artifactId>
-                <version>1.29</version>
+                <version>2.0</version>
             </dependency>
             <!-- test -->
             <dependency>

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.rdb.service;
 
-import java.sql.Connection;
+import java.sql.*;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -17,10 +13,7 @@ import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
 import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping;
 import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
+import com.alibaba.otter.canal.client.adapter.support.*;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.Util;
 
 /**
  * RDB ETL 操作业务类
@@ -44,7 +37,9 @@ public class RdbEtlService extends AbstractEtlService {
      */
     public EtlResult importData(List<String> params) {
         DbMapping dbMapping = config.getDbMapping();
-        String sql = "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable();
+        // 获取源数据源，根据数据库类型拼装数据库名和表名
+        DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType());
         return importData(sql, params);
     }
 

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java

@@ -1,21 +1,23 @@
 package com.alibaba.otter.canal.client.adapter.rdb.support;
 
-import com.alibaba.druid.DbType;
-import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.Reader;
 import java.io.StringReader;
 import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
 import java.sql.*;
+import java.time.LocalDateTime;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.DbType;
+import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+
 public class SyncUtil {
     private static final Logger logger  = LoggerFactory.getLogger(SyncUtil.class);
 
@@ -220,12 +222,17 @@ public class SyncUtil {
                     pstmt.setTime(i, new java.sql.Time(((java.util.Date) value).getTime()));
                 } else if (value instanceof String) {
                     String v = (String) value;
+                    if(Util.isAccuracyOverSecond(v)) {
+                        //the java.sql.time doesn't support for even millisecond, only setObject works here.
+                        pstmt.setObject(i, v);
+                    }else {
                         java.util.Date date = Util.parseDate(v);
                         if (date != null) {
                             pstmt.setTime(i, new Time(date.getTime()));
                         } else {
                             pstmt.setNull(i, type);
                         }
+                    }
                 } else {
                     pstmt.setNull(i, type);
                 }
@@ -238,12 +245,23 @@ public class SyncUtil {
                 } else if (value instanceof String) {
                     String v = (String) value;
                     if (!v.startsWith("0000-00-00")) {
+                        if(Util.isAccuracyOverMillisecond(v)){
+                            //convert to ISO-8601 standard format, with up to nanoseconds (9 digits) precision
+                            LocalDateTime isoDatetime = Util.parseISOLocalDateTime(v);
+                            if (isoDatetime != null) {
+                                pstmt.setTimestamp(i, Timestamp.valueOf(isoDatetime));
+                            } else {
+                                //if can't convert, set to null
+                                pstmt.setNull(i, type);
+                            }
+                        }else {
                             java.util.Date date = Util.parseDate(v);
                             if (date != null) {
                                 pstmt.setTimestamp(i, new Timestamp(date.getTime()));
                             } else {
                                 pstmt.setNull(i, type);
                             }
+                        }
                     } else {
                         pstmt.setObject(i, value);
                     }

client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java

@@ -51,7 +51,7 @@ public class CanalMessageDeserializer {
                 }
             }
         } catch (Exception e) {
-            throw new CanalClientException("deserializer failed", e);
+            throw new CanalClientException("deserializer failed by " + e.getMessage(), e);
         }
     }
 }

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -78,14 +78,15 @@ public class KafkaCanalConnector implements CanalMQConnector {
             return;
         }
 
-        connected = true;
         if (kafkaConsumer == null && !flatMessage) {
             kafkaConsumer = new KafkaConsumer<>(properties);
-
         }
+
         if (kafkaConsumer2 == null && flatMessage) {
             kafkaConsumer2 = new KafkaConsumer<>(properties);
         }
+
+        connected = true;
     }
 
     /**

client/src/main/java/com/alibaba/otter/canal/client/pulsarmq/PulsarMQCanalConnector.java

@@ -1,5 +1,13 @@
 package com.alibaba.otter.canal.client.pulsarmq;
 
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.client.api.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.otter.canal.client.CanalMQConnector;
 import com.alibaba.otter.canal.client.CanalMessageDeserializer;
@@ -9,13 +17,6 @@ import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.pulsar.client.api.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 /**
  * PulsarMQ的连接
@@ -24,13 +25,15 @@ import java.util.concurrent.TimeUnit;
  * 注意点:
  * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
  * </pre>
+ * 
  * todo 重复消费的概率相当高。一次批处理中，只要有一个消息处理失败，则该批次全部重试
  *
  * @since 1.1.1
  */
 public class PulsarMQCanalConnector implements CanalMQConnector {
 
-    private static final Logger logger = LoggerFactory.getLogger(PulsarMQCanalConnector.class);
+    private static final Logger       logger                     = LoggerFactory
+        .getLogger(PulsarMQCanalConnector.class);
 
     private volatile Messages<byte[]> lastGetBatchMessage;
 
@@ -58,6 +61,12 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
      * 角色认证token
      */
     private String                    roleToken;
+
+    /**
+     * listener name
+     */
+    private String                    listenerName;
+
     /**
      * 订阅客户端名称
      */
@@ -67,8 +76,7 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
      */
     private int                       batchSize                  = 30;
     /**
-     * 与{@code batchSize}一起决定批量获取的数据大小
+     * 与{@code batchSize}一起决定批量获取的数据大小 当：
-     * 当：
      * <p>
      * 1. {@code batchSize} 条消息未消费时<br/>
      * 2. 距上一次批量消费时间达到{@code batchTimeoutSeconds}秒时
@@ -126,8 +134,13 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
      * @author chad
      * @since 1 by chad at 2021/9/18 完善
      */
-    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic
+    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic,
-            , String subscriptName) {
+                                  String subscriptName){
+        this(isFlatMessage, serviceUrl, roleToken, topic, subscriptName, null);
+    }
+
+    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic,
+                                  String subscriptName, String listenerName){
         this.isFlatMessage = isFlatMessage;
         this.serviceUrl = serviceUrl;
         this.roleToken = roleToken;
@@ -136,6 +149,7 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
         if (StringUtils.isEmpty(this.subscriptName)) {
             throw new RuntimeException("Pulsar Consumer subscriptName required");
         }
+        this.listenerName = listenerName;
     }
 
     /**
@@ -145,10 +159,31 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
      * @author chad
      * @since 1 by chad at 2021/9/18 完善
      */
-    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic
+    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic,
-            , String subscriptName, int batchSize, int getBatchTimeoutSeconds, int batchProcessTimeoutSeconds
+                                  String subscriptName, int batchSize, int getBatchTimeoutSeconds,
-            , int redeliveryDelaySeconds, int ackTimeoutSeconds, boolean isRetry, boolean isRetryDLQUpperCase
+                                  int batchProcessTimeoutSeconds, int redeliveryDelaySeconds, int ackTimeoutSeconds,
-            , int maxRedeliveryCount) {
+                                  boolean isRetry, boolean isRetryDLQUpperCase, int maxRedeliveryCount){
+        this(isFlatMessage,
+            serviceUrl,
+            roleToken,
+            topic,
+            subscriptName,
+            batchSize,
+            getBatchTimeoutSeconds,
+            batchProcessTimeoutSeconds,
+            redeliveryDelaySeconds,
+            ackTimeoutSeconds,
+            isRetry,
+            isRetryDLQUpperCase,
+            maxRedeliveryCount,
+            null);
+    }
+
+    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic,
+                                  String subscriptName, int batchSize, int getBatchTimeoutSeconds,
+                                  int batchProcessTimeoutSeconds, int redeliveryDelaySeconds, int ackTimeoutSeconds,
+                                  boolean isRetry, boolean isRetryDLQUpperCase, int maxRedeliveryCount,
+                                  String listenerName){
         this.isFlatMessage = isFlatMessage;
         this.serviceUrl = serviceUrl;
         this.roleToken = roleToken;
@@ -165,16 +200,20 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
         this.isRetry = isRetry;
         this.isRetryDLQUpperCase = isRetryDLQUpperCase;
         this.maxRedeliveryCount = maxRedeliveryCount;
+        this.listenerName = listenerName;
     }
 
     @Override
     public void connect() throws CanalClientException {
         // 连接创建客户端
         try {
-            pulsarClient = PulsarClient.builder()
+            ClientBuilder builder = PulsarClient.builder()
                 .serviceUrl(serviceUrl)
-                    .authentication(AuthenticationFactory.token(roleToken))
+                .authentication(AuthenticationFactory.token(roleToken));
-                    .build();
+            if (StringUtils.isNotEmpty(listenerName)) {
+                builder.listenerName(listenerName);
+            }
+            pulsarClient = builder.build();
         } catch (PulsarClientException e) {
             throw new RuntimeException(e);
         }
@@ -226,8 +265,7 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
             // 调用consumer.negativeAcknowledge(message) （即nack）来表示消费失败的消息
             // 在指定的时间进行重新消费，默认是1分钟。
             .negativeAckRedeliveryDelay(this.redeliveryDelaySeconds, TimeUnit.SECONDS)
-                .subscriptionName(this.subscriptName)
+            .subscriptionName(this.subscriptName);
-        ;
         if (this.isRetry) {
             DeadLetterPolicy.DeadLetterPolicyBuilder dlqBuilder = DeadLetterPolicy.builder()
                 // 最大重试次数
@@ -240,29 +278,26 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
                 dlqBuilder.deadLetterTopic(dlqTopic);
             }
 
-            //默认关闭，如果需要重试则开启
+            // 默认关闭，如果需要重试则开启
-            builder.enableRetry(true)
+            builder.enableRetry(true).deadLetterPolicy(dlqBuilder.build());
-                    .deadLetterPolicy(dlqBuilder.build());
         }
 
         // ack超时
         builder.ackTimeout(this.ackTimeoutSeconds, TimeUnit.SECONDS);
 
         // pulsar批量获取消息设置
-        builder.batchReceivePolicy(new BatchReceivePolicy.Builder()
+        builder.batchReceivePolicy(new BatchReceivePolicy.Builder().maxNumMessages(this.batchSize)
-                .maxNumMessages(this.batchSize)
             .timeout(this.getBatchTimeoutSeconds, TimeUnit.SECONDS)
             .build());
 
         try {
             this.consumer = builder.subscribe();
+            connected = true;
         } catch (PulsarClientException e) {
             throw new RuntimeException(e);
         }
-        connected = true;
     }
 
-
     @Override
     public void subscribe() throws CanalClientException {
         this.subscribe(null);
@@ -299,7 +334,8 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
     }
 
     /**
-     * 关心业务执行结果，业务侧根据执行结果调用 {@link PulsarMQCanalConnector#ack()}或{@link PulsarMQCanalConnector#rollback()}
+     * 关心业务执行结果，业务侧根据执行结果调用
+     * {@link PulsarMQCanalConnector#ack()}或{@link PulsarMQCanalConnector#rollback()}
      * <p>
      * 本方法示支持多线程，在MQ保障顺序的前提下，也无法提供单Topic多线程
      * </p>
@@ -339,7 +375,8 @@ public class PulsarMQCanalConnector implements CanalMQConnector {
      * @return java.util.List<T>
      * @date 2021/9/14 15:20
      * @author chad
-     * @since 1 by chad at 2021/9/14 供{@link PulsarMQCanalConnector#getListWithoutAck(Long, TimeUnit)}
+     * @since 1 by chad at 2021/9/14
+     * 供{@link PulsarMQCanalConnector#getListWithoutAck(Long, TimeUnit)}
      * 和{@link PulsarMQCanalConnector#getFlatListWithoutAck(Long, TimeUnit)}调用
      */
     private <T> List<T> getListWithoutAck() {

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java

@@ -6,7 +6,6 @@ import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
-import com.alibaba.otter.canal.client.ConsumerBatchMessage;
 import org.apache.commons.lang.StringUtils;
 import org.apache.rocketmq.acl.common.AclClientRPCHook;
 import org.apache.rocketmq.acl.common.SessionCredentials;
@@ -25,6 +24,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.otter.canal.client.CanalMQConnector;
 import com.alibaba.otter.canal.client.CanalMessageDeserializer;
+import com.alibaba.otter.canal.client.ConsumerBatchMessage;
 import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
@@ -155,11 +155,10 @@ public class RocketMQCanalConnector implements CanalMQConnector {
                 }
             });
             rocketMQConsumer.start();
-        } catch (MQClientException ex) {
-            connected = false;
-            logger.error("Start RocketMQ consumer error", ex);
-        }
             connected = true;
+        } catch (MQClientException ex) {
+            throw new RuntimeException("Start RocketMQ consumer error", ex);
+        }
     }
 
     private boolean process(List<MessageExt> messageExts) {

common/pom.xml

@@ -87,6 +87,10 @@
 			<groupId>com.google.code.findbugs</groupId>
 			<artifactId>jsr305</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>com.alibaba</groupId>
+			<artifactId>druid</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework</groupId>
 			<artifactId>spring-test</artifactId>

common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java

@@ -7,6 +7,7 @@ import java.net.ServerSocket;
 import java.util.Enumeration;
 import java.util.regex.Pattern;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -75,4 +76,27 @@ public class AddressUtils {
         logger.error("Could not get local host ip address, will use 127.0.0.1 instead.");
         return localAddress;
     }
+
+    /**
+     * 拆分IP地址和端口号
+     *
+     * @param text ip地址和端口号，ip和端口号以英文冒号(:)分隔;
+     * 
+     *     <pre>
+     *             ipv4 127.0.0.1:3306
+     *             ipv6 [::1]:3306
+     *     </pre>
+     *
+     * @return
+     */
+    public static String[] splitIPAndPort(String text) {
+        text = text.replace("[", "").replace("]", "");
+        int idx = text.lastIndexOf(':');
+        if (idx > 0) {
+            String ip = text.substring(0, idx);
+            String port = text.substring(idx + 1);
+            return new String[] { ip, port };
+        }
+        return ArrayUtils.EMPTY_STRING_ARRAY;
+    }
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java

@@ -1,5 +1,7 @@
 package com.alibaba.otter.canal.common.utils;
 
+import com.alibaba.druid.filter.config.ConfigTools;
+
 import java.io.File;
 
 /**
@@ -50,4 +52,8 @@ public class CommonUtils {
 
         return dirFile.delete();
     }
+
+    public static String decryptDruidPassword(String publicKeyText, String cipherText) throws Exception {
+        return ConfigTools.decrypt(publicKeyText, cipherText);
+    }
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/ExecutorTemplate.java

@@ -44,7 +44,7 @@ public class ExecutorTemplate {
         check(future);
     }
 
-    public void submit(Callable<Exception> task) {
+    public void submit(Callable task) {
         Future future = executor.submit(task);
         futures.add(future);
         check(future);

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java

@@ -79,7 +79,7 @@ public class CanalMessageSerializerUtil {
                 }
             }
         } catch (Exception e) {
-            throw new RuntimeException("Error when serializing message to byte[] ");
+            throw new RuntimeException("Error when serializing message to byte[] by " + e.getMessage() , e);
         }
         return null;
     }
@@ -125,7 +125,7 @@ public class CanalMessageSerializerUtil {
                 }
             }
         } catch (Exception e) {
-            throw new CanalClientException("deserializer failed", e);
+            throw new CanalClientException("deserializer failed by " + e.getMessage(), e);
         }
     }
 }

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQConstants.java

@@ -61,4 +61,18 @@ public class PulsarMQConstants {
      */
     public static final String PULSARMQ_ADMIN_SERVER_URL          = ROOT + "." + "adminServerUrl";
 
+    /**
+     * Pulsar 监听器名字
+     */
+    public static final String PULSARMQ_LISTENER_NAME             = ROOT + "." + "listenerName";
+
+    /**
+     * Pulsar 开启chunking
+     */
+    public static final String PULSARMQ_ENABLE_CHUNKING           = ROOT + "." + "enableChunking";
+
+    /**
+     * Pulsar 压缩算法
+     */
+    public static final String PULSARMQ_COMPRESSION_TYPE          = ROOT + "." + "compressionType";
 }

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQProducerConfig.java

@@ -37,6 +37,21 @@ public class PulsarMQProducerConfig extends MQProperties {
      */
     private String  adminServerUrl;
 
+    /**
+     * listener name
+     */
+    private String  listenerName;
+
+    /**
+     * enableChunking
+     */
+    private boolean enableChunking;
+
+    /**
+     * compressionType
+     */
+    private String  compressionType;
+
     public String getServerUrl() {
         return serverUrl;
     }
@@ -68,4 +83,28 @@ public class PulsarMQProducerConfig extends MQProperties {
     public void setAdminServerUrl(String adminServerUrl) {
         this.adminServerUrl = adminServerUrl;
     }
+
+    public String getListenerName() {
+        return listenerName;
+    }
+
+    public void setListenerName(String listenerName) {
+        this.listenerName = listenerName;
+    }
+
+    public void setEnableChunking(boolean enableChunking) {
+        this.enableChunking = enableChunking;
+    }
+
+    public boolean getEnableChunking() {
+        return this.enableChunking;
+    }
+
+    public void setCompressionType(String compressionType) {
+        this.compressionType = compressionType;
+    }
+
+    public String getCompressionType() {
+        return this.compressionType;
+    }
 }

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumer.java

@@ -54,6 +54,12 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
      * 角色认证token
      */
     private String                    roleToken;
+
+    /**
+     * listener name
+     */
+    private String                    listenerName;
+
     /**
      * 订阅客户端名称
      */
@@ -110,6 +116,7 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
         }
         this.serviceUrl = properties.getProperty(PulsarMQConstants.PULSARMQ_SERVER_URL);
         this.roleToken = properties.getProperty(PulsarMQConstants.PULSARMQ_ROLE_TOKEN);
+        this.listenerName = properties.getProperty(PulsarMQConstants.PULSARMQ_LISTENER_NAME);
         this.subscriptName = properties.getProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME);
         // 采用groupId作为subscriptName，避免所有的都是同一个订阅者名称
         if (StringUtils.isEmpty(this.subscriptName)) {
@@ -165,6 +172,9 @@ public class CanalPulsarMQConsumer implements CanalMsgConsumer {
             if (StringUtils.isNotEmpty(roleToken)) {
                 builder.authentication(AuthenticationFactory.token(roleToken));
             }
+            if (StringUtils.isNotEmpty(listenerName)) {
+                builder.authentication(AuthenticationFactory.token(listenerName));
+            }
             pulsarClient = builder.build();
         } catch (PulsarClientException e) {
             throw new RuntimeException(e);

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/producer/CanalPulsarMQProducer.java

@@ -73,6 +73,11 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
                 // 角色权限认证的token
                 builder.authentication(AuthenticationFactory.token(pulsarMQProducerConfig.getRoleToken()));
             }
+            if (StringUtils.isNotEmpty(pulsarMQProducerConfig.getListenerName())) {
+                // listener name
+                builder.listenerName(pulsarMQProducerConfig.getListenerName());
+            }
+
             client = builder.build();
         } catch (PulsarClientException e) {
             throw new RuntimeException(e);
@@ -127,6 +132,21 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
         if (!StringUtils.isEmpty(adminServerUrl)) {
             tmpProperties.setAdminServerUrl(adminServerUrl);
         }
+        String listenerName = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_LISTENER_NAME);
+        if (!StringUtils.isEmpty(listenerName)) {
+            tmpProperties.setListenerName(listenerName);
+        }
+
+        String enableChunkingStr = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_ENABLE_CHUNKING);
+        if (!StringUtils.isEmpty(enableChunkingStr)) {
+            tmpProperties.setEnableChunking(Boolean.parseBoolean(enableChunkingStr));
+        }
+
+        String compressionType = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_COMPRESSION_TYPE);
+        if (!StringUtils.isEmpty(compressionType)) {
+            tmpProperties.setCompressionType(compressionType);
+        }
+
         if (logger.isDebugEnabled()) {
             logger.debug("Load pulsar properties ==> {}", JSON.toJSON(this.mqProperties));
         }
@@ -398,11 +418,34 @@ public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQ
                     }
 
                     // 创建指定topic的生产者
-                    producer = client.newProducer()
+                    ProducerBuilder producerBuilder = client.newProducer();
-                        .topic(fullTopic)
+                    if (pulsarMQProperties.getEnableChunking()) {
+                        producerBuilder.enableChunking(true);
+                        producerBuilder.enableBatching(false);
+                    }
+
+                    if (!StringUtils.isEmpty(pulsarMQProperties.getCompressionType())) {
+                        switch (pulsarMQProperties.getCompressionType().toLowerCase()) {
+                            case "lz4":
+                                producerBuilder.compressionType(CompressionType.LZ4);
+                                break;
+                            case "zlib":
+                                producerBuilder.compressionType(CompressionType.ZLIB);
+                                break;
+                            case "zstd":
+                                producerBuilder.compressionType(CompressionType.ZSTD);
+                                break;
+                            case "snappy":
+                                producerBuilder.compressionType(CompressionType.SNAPPY);
+                                break;
+                        }
+                    }
+
+                    producer = producerBuilder.topic(fullTopic)
                         // 指定路由器
                         .messageRouter(new MessageRouterImpl(topic))
                         .create();
+
                     // 放入缓存
                     PRODUCERS.put(topic, producer);
                 }

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java

@@ -9,6 +9,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import com.alibaba.otter.canal.common.utils.AddressUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -88,7 +89,7 @@ public class CanalRabbitMQConsumer implements CanalMsgConsumer {
         }
         // 解析出端口 modified by 16075140
         if (nameServer != null && nameServer.contains(":")) {
-            String[] serverHostAndPort = nameServer.split(":");
+            String[] serverHostAndPort = AddressUtils.splitIPAndPort(nameServer);
             factory.setHost(serverHostAndPort[0]);
             factory.setPort(Integer.parseInt(serverHostAndPort[1]));
         } else {

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java

@@ -16,6 +16,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.fastjson2.JSONWriter;
 import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
 import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
 import com.alibaba.otter.canal.common.utils.PropertiesUtils;
 import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
@@ -61,7 +62,7 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
                 throw new CanalException("failed to parse host", ex);
             }
         } else if (servers.contains(":")) {
-            String[] serverHostAndPort = servers.split(":");
+            String[] serverHostAndPort = AddressUtils.splitIPAndPort(servers);
             factory.setHost(serverHostAndPort[0]);
             factory.setPort(Integer.parseInt(serverHostAndPort[1]));
         } else {
@@ -81,13 +82,18 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
         try {
             connect = factory.newConnection();
             channel = connect.createChannel();
-            channel.queueDeclare(rabbitMQProperties.getQueue(), true, false, false, null);
+            String queue = rabbitMQProperties.getQueue();
-            channel.exchangeDeclare(rabbitMQProperties
+            String exchange = rabbitMQProperties.getExchange();
-                .getExchange(), rabbitMQProperties.getDeliveryMode(), true, false, false, null);
+            String deliveryMode = rabbitMQProperties.getDeliveryMode();
-            channel.queueBind(rabbitMQProperties.getQueue(),
+            String routingKey = rabbitMQProperties.getRoutingKey();
-                rabbitMQProperties.getExchange(),
+            if (!StringUtils.isEmpty(queue)) {
-                rabbitMQProperties.getRoutingKey());
+                channel.queueDeclare(queue, true, false, false, null);
-
+            }
+            if (!StringUtils.isEmpty(queue) && !StringUtils.isEmpty(exchange) && !StringUtils.isEmpty(deliveryMode)
+                && !StringUtils.isEmpty(routingKey)) {
+                channel.exchangeDeclare(exchange, deliveryMode, true, false, false, null);
+                channel.queueBind(queue, exchange, routingKey);
+            }
         } catch (IOException | TimeoutException ex) {
             throw new CanalException("Start RabbitMQ producer error", ex);
         }

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java

@@ -1,5 +1,29 @@
 package com.alibaba.otter.canal.connector.rocketmq.producer;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.AccessChannel;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.fastjson2.JSONWriter;
 import com.alibaba.otter.canal.common.CanalException;
@@ -16,29 +40,6 @@ import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
 import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
 import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
 import com.alibaba.otter.canal.protocol.FlatMessage;
-import org.apache.commons.lang.StringUtils;
-import org.apache.rocketmq.acl.common.AclClientRPCHook;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.client.AccessChannel;
-import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
-import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
-import org.apache.rocketmq.client.producer.DefaultMQProducer;
-import org.apache.rocketmq.client.producer.SendResult;
-import org.apache.rocketmq.common.message.Message;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.remoting.RPCHook;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * RocketMQ Producer SPI 实现
@@ -46,15 +47,18 @@ import java.util.stream.Collectors;
  * @author rewerma 2020-01-27
  * @version 1.0.0
  */
-@SPI("rocketmq") public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
+@SPI("rocketmq")
+public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
 
     private static final Logger  logger               = LoggerFactory.getLogger(CanalRocketMQProducer.class);
 
     private DefaultMQProducer    defaultMQProducer;
     private static final String  CLOUD_ACCESS_CHANNEL = "cloud";
+    private static final String  NAMESPACE_SEPARATOR  = "%";
     protected ThreadPoolExecutor sendPartitionExecutor;
 
-    @Override public void init(Properties properties) {
+    @Override
+    public void init(Properties properties) {
         RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
         this.mqProperties = rocketMQProperties;
         super.init(properties);
@@ -143,15 +147,14 @@ import java.util.stream.Collectors;
         }
     }
 
-    @Override public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message,
+    @Override
-                               Callback callback) {
+    public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
         ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
         try {
             if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
                 // 动态topic
-                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
+                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
-                        destination.getTopic(),
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
-                        destination.getDynamicTopic());
 
                 for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
                     String topicName = entry.getKey().replace('.', '_');
@@ -263,7 +266,9 @@ import java.util.stream.Collectors;
                             List<Message> messages = flatMessagePart.stream()
                                 .map(flatMessage -> new Message(topicName,
                                     ((RocketMQProducerConfig) this.mqProperties).getTag(),
-                                            JSON.toJSONBytes(flatMessage, JSONWriter.Feature.WriteNulls, JSONWriter.Feature.LargeObject)))
+                                    JSON.toJSONBytes(flatMessage,
+                                        JSONWriter.Feature.WriteNulls,
+                                        JSONWriter.Feature.LargeObject)))
                                 .collect(Collectors.toList());
                             // 批量发送
                             sendMessage(messages, index);
@@ -278,7 +283,7 @@ import java.util.stream.Collectors;
                 List<Message> messages = flatMessages.stream()
                     .map(flatMessage -> new Message(topicName,
                         ((RocketMQProducerConfig) this.mqProperties).getTag(),
-                                JSON.toJSONBytes(flatMessage, JSONWriter.Feature.WriteNulls,JSONWriter.Feature.LargeObject)))
+                        JSON.toJSONBytes(flatMessage, JSONWriter.Feature.WriteNulls, JSONWriter.Feature.LargeObject)))
                     .collect(Collectors.toList());
                 // 批量发送
                 sendMessage(messages, partition);
@@ -304,14 +309,19 @@ import java.util.stream.Collectors;
         }
     }
 
-    @SuppressWarnings("deprecation") private void sendMessage(List<Message> messages, int partition) {
+    @SuppressWarnings("deprecation")
+    private void sendMessage(List<Message> messages, int partition) {
         if (messages.isEmpty()) {
             return;
         }
 
         // 获取一下messageQueue
         DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
-        TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
+        String topic = messages.get(0).getTopic();
+        if (StringUtils.isNotBlank(this.defaultMQProducer.getNamespace())) {
+            topic = this.defaultMQProducer.getNamespace() + NAMESPACE_SEPARATOR + topic;
+        }
+        TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topic);
         if (topicInfo == null) {
             for (Message message : messages) {
                 sendMessage(message, partition);
@@ -346,7 +356,8 @@ import java.util.stream.Collectors;
         }
     }
 
-    @Override public void stop() {
+    @Override
+    public void stop() {
         logger.info("## Stop RocketMQ producer##");
         this.defaultMQProducer.shutdown();
         if (sendPartitionExecutor != null) {

connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
+import com.alibaba.otter.canal.common.utils.AddressUtils;
 import org.apache.commons.lang.StringUtils;
 
 import com.alibaba.otter.canal.client.CanalConnector;
@@ -47,7 +48,7 @@ public class CanalTCPConsumer implements CanalMsgConsumer {
             batchSize = Integer.parseInt(batchSizePro);
         }
         if (StringUtils.isNotBlank(host)) {
-            String[] ipPort = host.split(":");
+            String[] ipPort =  AddressUtils.splitIPAndPort(host);
             SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
             this.canalConnector = new SimpleCanalConnector(sa, username, password, destination);
         } else {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java

@@ -1,10 +1,8 @@
 package com.taobao.tddl.dbsync.binlog;
 
-import java.nio.charset.Charset;
+import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.*;
 
-import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2;
+import java.nio.charset.Charset;
-import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4;
-import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr;
 
 /**
  * 处理下MySQL json二进制转化为可读的字符串
@@ -34,26 +32,25 @@ public class JsonConversion {
     public static final char JSONB_FALSE_LITERAL     = '\2';
 
     /*
-     * The size of offset or size fields in the small and the large storage
+     * The size of offset or size fields in the small and the large storage format
-     * format for JSON objects and JSON arrays.
+     * for JSON objects and JSON arrays.
      */
     public static final int  SMALL_OFFSET_SIZE       = 2;
     public static final int  LARGE_OFFSET_SIZE       = 4;
 
     /*
-     * The size of key entries for objects when using the small storage format
+     * The size of key entries for objects when using the small storage format or
-     * or the large storage format. In the small format it is 4 bytes (2 bytes
+     * the large storage format. In the small format it is 4 bytes (2 bytes for key
-     * for key length and 2 bytes for key offset). In the large format it is 6
+     * length and 2 bytes for key offset). In the large format it is 6 (2 bytes for
-     * (2 bytes for length, 4 bytes for offset).
+     * length, 4 bytes for offset).
      */
     public static final int  KEY_ENTRY_SIZE_SMALL    = (2 + SMALL_OFFSET_SIZE);
     public static final int  KEY_ENTRY_SIZE_LARGE    = (2 + LARGE_OFFSET_SIZE);
 
     /*
-     * The size of value entries for objects or arrays. When using the small
+     * The size of value entries for objects or arrays. When using the small storage
-     * storage format, the entry size is 3 (1 byte for type, 2 bytes for
+     * format, the entry size is 3 (1 byte for type, 2 bytes for offset). When using
-     * offset). When using the large storage format, it is 5 (1 byte for type, 4
+     * the large storage format, it is 5 (1 byte for type, 4 bytes for offset).
-     * bytes for offset).
      */
     public static final int  VALUE_ENTRY_SIZE_SMALL  = (1 + SMALL_OFFSET_SIZE);
     public static final int  VALUE_ENTRY_SIZE_LARGE  = (1 + LARGE_OFFSET_SIZE);
@@ -160,13 +157,12 @@ public class JsonConversion {
                 if (len < n + str_len) {
                     throw new IllegalArgumentException("illegal json data");
                 }
-                return new Json_Value(Json_enum_type.STRING, buffer.rewind()
+                return new Json_Value(Json_enum_type.STRING,
-                    .forward((int) n)
+                    buffer.rewind().forward((int) n).getFixString((int) str_len, charset));
-                    .getFixString((int) str_len, charset));
             case JSONB_TYPE_OPAQUE:
                 /*
-                 * There should always be at least one byte, which tells the
+                 * There should always be at least one byte, which tells the field type of the
-                 * field type of the opaque value.
+                 * opaque value.
                  */
                 // The type is encoded as a uint8 that maps to an
                 // enum_field_types.
@@ -291,7 +287,7 @@ public class JsonConversion {
                         if (i > 0) {
                             buf.append(", ");
                         }
-                        buf.append('"').append(key(i, charset)).append('"');
+                        buf.append('"').append(escapse(key(i, charset))).append('"');
                         buf.append(": ");
                         element(i, charset).toJsonString(buf, charset);
                     }
@@ -452,7 +448,8 @@ public class JsonConversion {
     }
 
     public static enum Json_enum_type {
-        OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR
+                                       OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE,
+                                       LITERAL_FALSE, OPAQUE, ERROR
     }
 
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java

@@ -819,7 +819,7 @@ public class QueryLogEvent extends LogEvent {
             case Q_MASTER_DATA_WRITTEN_CODE:
                 return "Q_MASTER_DATA_WRITTEN_CODE";
             case Q_INVOKER:
-                return "case Q_INVOKER";
+                return "Q_INVOKER";
             case Q_MICROSECONDS:
                 return "Q_MICROSECONDS";
             case Q_UPDATED_DB_NAMES:

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/JsonConversion_Json_ValueTest.java

@@ -1,5 +1,7 @@
 package com.taobao.tddl.dbsync.binlog;
 
+import java.nio.charset.Charset;
+
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -48,7 +50,31 @@ public class JsonConversion_Json_ValueTest {
         // Act
         thrown.expect(IllegalArgumentException.class);
         JsonConversion.parse_value(type, buffer, len, charsetName);
-
         // Method is not expected to return due to exception thrown
     }
+
+    // test for 5127
+    @Test
+    public void testJsonKeyContainsSpecialCharacter() {
+
+        // {"internal_uri_rewrite": {"(.*)(/[^/]+\\.(mp|MP)4)$": "$1/mp4$2"}}
+        String jsonData = "{\"internal_uri_rewrite\": {\"(.*)(/[^/]+\\\\.(mp|MP)4)$\": \"$1/mp4$2\"}}";
+        byte[] data = new byte[] { 1, 0, 74, 0, 11, 0, 20, 0, 0, 31, 0, 105, 110, 116, 101, 114, 110, 97, 108, 95, 117,
+                                   114, 105, 95, 114, 101, 119, 114, 105, 116, 101, 1, 0, 43, 0, 11, 0, 23, 0, 12, 34,
+                                   0, 40, 46, 42, 41, 40, 47, 91, 94, 47, 93, 43, 92, 46, 40, 109, 112, 124, 77, 80, 41,
+                                   52, 41, 36, 8, 36, 49, 47, 109, 112, 52, 36, 50 };
+        final LogBuffer buffer = new LogBuffer(data, 0, 74);
+
+        Charset charset = Charset.forName("UTF-8");
+        Json_Value jsonValue = JsonConversion.parse_value(0, buffer, 74, charset);
+        StringBuilder builder = new StringBuilder();
+        jsonValue.toJsonString(builder, charset);
+
+        Assert.assertEquals(builder.toString(), jsonData);
+
+        Assert.assertEquals(jsonValue.key(0, charset), "internal_uri_rewrite");
+
+        Json_Value element = jsonValue.element(0, charset);
+        Assert.assertEquals(element.key(0, charset), "(.*)(/[^/]+\\.(mp|MP)4)$");
+    }
 }

deployer/src/main/bin/startup.sh

@@ -81,14 +81,14 @@ esac
 
 JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk  -F '.' '{print $1}'`
 str=`file -L $JAVA | grep 64-bit`
-JAVA_OPTS="$JAVA_OPTS -Xss1m -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
+JAVA_OPTS="$JAVA_OPTS -Xss1m -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
 
 if [ $JavaVersion -ge 11 ] ; then
   #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
   JAVA_OPTS="$JAVA_OPTS"
 else
   #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime"
-  JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
+  JAVA_OPTS="$JAVA_OPTS -XX:+AggressiveOpts -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
 fi
 
 if [ -n "$str" ]; then

deployer/src/main/resources/example/instance.properties

@@ -5,6 +5,11 @@
 # enable gtid use true/false
 canal.instance.gtidon=false
 
+# rds oss binlog
+canal.instance.rds.accesskey=
+canal.instance.rds.secretkey=
+canal.instance.rds.instanceId=
+
 # position info
 canal.instance.master.address=127.0.0.1:3306
 canal.instance.master.journal.name=
@@ -12,10 +17,15 @@ canal.instance.master.position=
 canal.instance.master.timestamp=
 canal.instance.master.gtid=
 
-# rds oss binlog
+# ssl
-canal.instance.rds.accesskey=
+#canal.instance.master.sslMode=DISABLED
-canal.instance.rds.secretkey=
+#canal.instance.master.tlsVersions=
-canal.instance.rds.instanceId=
+#canal.instance.master.trustCertificateKeyStoreType=
+#canal.instance.master.trustCertificateKeyStoreUrl=
+#canal.instance.master.trustCertificateKeyStorePassword=
+#canal.instance.master.clientCertificateKeyStoreType=
+#canal.instance.master.clientCertificateKeyStoreUrl=
+#canal.instance.master.clientCertificateKeyStorePassword=
 
 # table meta tsdb info
 canal.instance.tsdb.enable=true

deployer/src/main/resources/logback.xml

@@ -23,7 +23,7 @@
 						<!-- or whenever the file size reaches 100MB -->
 						<maxFileSize>512MB</maxFileSize>
 					</timeBasedFileNamingAndTriggeringPolicy>
-					<maxHistory>60</maxHistory>
+					<maxHistory>14</maxHistory>
 				</rollingPolicy>
 				<encoder>
 					<pattern>
@@ -50,7 +50,7 @@
 						<!-- or whenever the file size reaches 100MB -->
 						<maxFileSize>32MB</maxFileSize>
 					</timeBasedFileNamingAndTriggeringPolicy>
-					<maxHistory>60</maxHistory>
+					<maxHistory>14</maxHistory>
 				</rollingPolicy>
 				<encoder>
 					<pattern>
@@ -71,7 +71,7 @@
 				<!-- or whenever the file size reaches 100MB -->
 				<maxFileSize>512MB</maxFileSize>
 			</timeBasedFileNamingAndTriggeringPolicy>
-			<maxHistory>60</maxHistory>
+			<maxHistory>14</maxHistory>
 		</rollingPolicy>
 		<encoder charset="UTF-8">
 			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n</pattern>

deployer/src/main/resources/spring/default-instance.xml

@@ -111,7 +111,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 
 		<!-- 解析位点记录 -->
@@ -140,6 +139,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.master.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.master.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.master.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.master.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.master.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.master.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.master.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.master.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 		<property name="standbyInfo">
@@ -150,6 +161,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.standby.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.standby.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.standby.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.standby.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.standby.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.standby.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.standby.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.standby.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 

deployer/src/main/resources/spring/file-instance.xml

@@ -97,7 +97,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 
 		<!-- 解析位点记录 -->
@@ -126,6 +125,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.master.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.master.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.master.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.master.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.master.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.master.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.master.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.master.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 		<property name="standbyInfo">
@@ -136,6 +147,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.standby.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.standby.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.standby.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.standby.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.standby.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.standby.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.standby.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.standby.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 

deployer/src/main/resources/spring/group-instance.xml

@@ -103,7 +103,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 
 		<!-- 解析位点记录 -->
@@ -123,6 +122,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.master.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.master.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.master.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.master.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.master.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.master.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.master.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.master.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 		<property name="standbyInfo">
@@ -133,6 +144,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.standby.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.standby.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.standby.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.standby.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.standby.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.standby.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.standby.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.standby.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 
@@ -211,7 +234,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 
 		<!-- 解析位点记录 -->
@@ -231,6 +253,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.master.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.master.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.master.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.master.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.master.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.master.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.master.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.master.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 		<property name="standbyInfo">
@@ -241,6 +275,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.standby.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.standby.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.standby.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.standby.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.standby.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.standby.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.standby.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.standby.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 

deployer/src/main/resources/spring/memory-instance.xml

@@ -94,7 +94,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 
 		<!-- 解析位点记录 -->
@@ -114,6 +113,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.master.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.master.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.master.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.master.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.master.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.master.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.master.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.master.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 		<property name="standbyInfo">
@@ -124,6 +135,18 @@
 				<property name="pwdPublicKey" value="${canal.instance.pwdPublicKey:retl}" />
 				<property name="enableDruid" value="${canal.instance.enableDruid:false}" />
 				<property name="defaultDatabaseName" value="${canal.instance.defaultDatabaseName:}" />
+				<property name="sslInfo">
+					<bean class="com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo">
+						<property name="sslMode" value="${canal.instance.standby.sslMode:DISABLED}"/>
+						<property name="tlsVersions" value="${canal.instance.standby.tlsVersions:}"/>
+						<property name="trustCertificateKeyStoreType" value="${canal.instance.standby.trustCertificateKeyStoreType:}"/>
+						<property name="trustCertificateKeyStoreUrl" value="${canal.instance.standby.trustCertificateKeyStoreUrl:}"/>
+						<property name="trustCertificateKeyStorePassword" value="${canal.instance.standby.trustCertificateKeyStorePassword:}"/>
+						<property name="clientCertificateKeyStoreType" value="${canal.instance.standby.clientCertificateKeyStoreType:}"/>
+						<property name="clientCertificateKeyStoreUrl" value="${canal.instance.standby.clientCertificateKeyStoreUrl:}"/>
+						<property name="clientCertificateKeyStorePassword" value="${canal.instance.standby.clientCertificateKeyStorePassword:}"/>
+					</bean>
+				</property>
 			</bean>
 		</property>
 

docker/image/alidata/bin/main.sh

@@ -12,8 +12,9 @@ for e in $(ls /alidata/init/*) ; do
 done
 
 echo "==> INIT DEFAULT"
-systemctl start sshd
+# disable systemctl
-systemctl start crond
+# systemctl start sshd
+# systemctl start crond
 
 #echo "check hostname -i: `hostname -i`"
 #hti_num=`hostname -i|awk '{print NF}'`

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java

@@ -1,5 +1,7 @@
 package com.alibaba.otter.canal.parse.driver.mysql;
 
+import static com.alibaba.otter.canal.parse.driver.mysql.packets.Capability.CLIENT_SSL;
+
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.security.DigestException;
@@ -10,12 +12,12 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.client.AuthSwitchResponsePacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.*;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.client.ClientAuthenticationPacket;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QuitCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.*;
 import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
 import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannelPool;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslMode;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
@@ -28,18 +30,17 @@ import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
  */
 public class MysqlConnector {
 
+    public static final int     timeout           = 5 * 1000;                                     // 5s
     private static final Logger logger            = LoggerFactory.getLogger(MysqlConnector.class);
     private InetSocketAddress   address;
     private String              username;
     private String              password;
-
+    private SslInfo             sslInfo;
-    private byte                charsetNumber     = 33;
     private String              defaultSchema;
     private int                 soTimeout         = 30 * 1000;
     private int                 connTimeout       = 5 * 1000;
     private int                 receiveBufferSize = 16 * 1024;
     private int                 sendBufferSize    = 16 * 1024;
-
     private SocketChannel       channel;
     private volatile boolean    dumping           = false;
     // mysql connectionId
@@ -48,8 +49,6 @@ public class MysqlConnector {
     // serverVersion
     private String              serverVersion;
 
-    public static final int     timeout           = 5 * 1000;                                     // 5s
-
     public MysqlConnector(){
     }
 
@@ -62,20 +61,25 @@ public class MysqlConnector {
         this.password = password;
     }
 
-    public MysqlConnector(InetSocketAddress address, String username, String password, byte charsetNumber,
+    public MysqlConnector(InetSocketAddress address, String username, String password, String defaultSchema){
-                          String defaultSchema){
         this(address, username, password);
 
-        this.charsetNumber = charsetNumber;
         this.defaultSchema = defaultSchema;
     }
 
+    public MysqlConnector(InetSocketAddress address, String username, String password, String defaultSchema,
+                          SslInfo sslInfo){
+        this(address, username, password, defaultSchema);
+        this.sslInfo = sslInfo;
+    }
+
     public void connect() throws IOException {
         if (connected.compareAndSet(false, true)) {
             try {
                 channel = SocketChannelPool.open(address);
                 logger.info("connect MysqlConnection to {}...", address);
                 negotiate(channel);
+                printSslStatus();
             } catch (Exception e) {
                 disconnect();
                 throw new IOException("connect " + this.address + " failure", e);
@@ -85,6 +89,28 @@ public class MysqlConnector {
         }
     }
 
+    private void printSslStatus() {
+        try {
+            MysqlQueryExecutor executor = new MysqlQueryExecutor(this);
+            ResultSetPacket result = executor.query("SHOW STATUS LIKE 'Ssl_version'");
+            String sslVersion = "";
+            if (result.getFieldValues() != null && result.getFieldValues().size() >= 2) {
+                sslVersion = result.getFieldValues().get(1);
+            }
+            result = executor.query("SHOW STATUS LIKE 'Ssl_cipher'");
+            String sslCipher = "";
+            if (result.getFieldValues() != null && result.getFieldValues().size() >= 2) {
+                sslCipher = result.getFieldValues().get(1);
+            }
+            logger.info("connect MysqlConnection in sslMode {}, Ssl_version:{}, Ssl_cipher:{}",
+                (sslInfo != null ? sslInfo.getSslMode() : SslMode.DISABLED),
+                sslVersion,
+                sslCipher);
+        } catch (Exception e) {
+            logger.warn("Can't show SSL status, server may not standard MySQL server", e);
+        }
+    }
+
     public void reconnect() throws IOException {
         disconnect();
         connect();
@@ -131,7 +157,6 @@ public class MysqlConnector {
 
     public MysqlConnector fork() {
         MysqlConnector connector = new MysqlConnector();
-        connector.setCharsetNumber(getCharsetNumber());
         connector.setDefaultSchema(getDefaultSchema());
         connector.setAddress(getAddress());
         connector.setPassword(password);
@@ -140,6 +165,7 @@ public class MysqlConnector {
         connector.setSendBufferSize(getSendBufferSize());
         connector.setSoTimeout(getSoTimeout());
         connector.setConnTimeout(connTimeout);
+        connector.setSslInfo(getSslInfo());
         return connector;
     }
 
@@ -170,6 +196,23 @@ public class MysqlConnector {
         }
         HandshakeInitializationPacket handshakePacket = new HandshakeInitializationPacket();
         handshakePacket.fromBytes(body);
+        byte serverCharsetNumber = (handshakePacket.serverCharsetNumber != 0) ? handshakePacket.serverCharsetNumber : 33;
+        SslMode sslMode = sslInfo != null ? sslInfo.getSslMode() : SslMode.DISABLED;
+        if (sslMode != SslMode.DISABLED) {
+            boolean serverSupportSsl = (handshakePacket.serverCapabilities & CLIENT_SSL) > 0;
+            if (!serverSupportSsl) {
+                throw new IOException("MySQL Server does not support SSL: " + address + " serverCapabilities: "
+                                      + handshakePacket.serverCapabilities);
+            }
+            byte[] sslPacket = new SslRequestCommandPacket(serverCharsetNumber).toBytes();
+            HeaderPacket sslHeader = new HeaderPacket();
+            sslHeader.setPacketBodyLength(sslPacket.length);
+            sslHeader.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
+            header.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
+            PacketManager.writePkg(channel, sslHeader.toBytes(), sslPacket);
+            channel = SocketChannelPool.connectSsl(channel, sslInfo);
+            this.channel = channel;
+        }
         if (handshakePacket.protocolVersion != MSC.DEFAULT_PROTOCOL_VERSION) {
             // HandshakeV9
             auth323(channel, (byte) (header.getPacketSequenceNumber() + 1), handshakePacket.seed);
@@ -179,15 +222,27 @@ public class MysqlConnector {
         connectionId = handshakePacket.threadId; // 记录一下connection
         serverVersion = handshakePacket.serverVersion; // 记录serverVersion
         logger.info("handshake initialization packet received, prepare the client authentication packet to send");
-        ClientAuthenticationPacket clientAuth = new ClientAuthenticationPacket();
+        // 某些老协议的 server 默认不返回 auth plugin，需要使用默认的 mysql_native_password
-        clientAuth.setCharsetNumber(charsetNumber);
+        String authPluginName = (handshakePacket.authPluginName != null
+                                 && handshakePacket.authPluginName.length > 0) ? new String(
+                                     handshakePacket.authPluginName) : "mysql_native_password";
+        logger.info("auth plugin: {}", authPluginName);
+        boolean isSha2Password = false;
+        ClientAuthenticationPacket clientAuth;
+        if ("caching_sha2_password".equals(authPluginName)) {
+            clientAuth = new ClientAuthenticationSHA2Packet();
+            isSha2Password = true;
+        } else {
+            clientAuth = new ClientAuthenticationPacket();
+        }
+        clientAuth.setCharsetNumber(serverCharsetNumber);
 
         clientAuth.setUsername(username);
         clientAuth.setPassword(password);
         clientAuth.setServerCapabilities(handshakePacket.serverCapabilities);
         clientAuth.setDatabaseName(defaultSchema);
         clientAuth.setScrumbleBuff(joinAndCreateScrumbleBuff(handshakePacket));
-        clientAuth.setAuthPluginName("mysql_native_password".getBytes());
+        clientAuth.setAuthPluginName(authPluginName.getBytes());
 
         byte[] clientAuthPkgBody = clientAuth.toBytes();
         HeaderPacket h = new HeaderPacket();
@@ -205,6 +260,12 @@ public class MysqlConnector {
         assert body != null;
         byte marker = body[0];
         if (marker == -2 || marker == 1) {
+            if (isSha2Password && body[1] == 3) {
+                // sha2 auth ok
+                logger.info("caching_sha2_password auth success.");
+                header = PacketManager.readHeader(channel, 4);
+                body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
+            } else {
                 byte[] authData = null;
                 String pluginName = null;
                 if (marker == 1) {
@@ -224,7 +285,7 @@ public class MysqlConnector {
                     encryptedPassword = getPassword().getBytes();
                     header = authSwitchAfterAuth(encryptedPassword, header);
                     body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
-            } else if ("mysql_native_password".equals(pluginName)) {
+                } else if (pluginName == null || "mysql_native_password".equals(pluginName)) {
                     try {
                         encryptedPassword = MySQLPasswordEncrypter.scramble411(getPassword().getBytes(), authData);
                     } catch (NoSuchAlgorithmException e) {
@@ -235,7 +296,8 @@ public class MysqlConnector {
                 } else if ("caching_sha2_password".equals(pluginName)) {
                     byte[] scramble = authData;
                     try {
-                    encryptedPassword = MySQLPasswordEncrypter.scrambleCachingSha2(getPassword().getBytes(), scramble);
+                        encryptedPassword = MySQLPasswordEncrypter.scrambleCachingSha2(getPassword().getBytes(),
+                            scramble);
                     } catch (DigestException e) {
                         throw new RuntimeException("can't encrypt password that will be sent to MySQL server.", e);
                     }
@@ -255,7 +317,7 @@ public class MysqlConnector {
                     body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
                 }
             }
-        assert body != null;
+        }
         if (body[0] < 0) {
             if (body[0] == -1) {
                 ErrorPacket err = new ErrorPacket();
@@ -380,14 +442,6 @@ public class MysqlConnector {
         this.username = username;
     }
 
-    public byte getCharsetNumber() {
-        return charsetNumber;
-    }
-
-    public void setCharsetNumber(byte charsetNumber) {
-        this.charsetNumber = charsetNumber;
-    }
-
     public String getDefaultSchema() {
         return defaultSchema;
     }
@@ -428,10 +482,6 @@ public class MysqlConnector {
         this.channel = channel;
     }
 
-    public void setPassword(String password) {
-        this.password = password;
-    }
-
     public long getConnectionId() {
         return connectionId;
     }
@@ -460,8 +510,19 @@ public class MysqlConnector {
         return password;
     }
 
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
     public String getServerVersion() {
         return serverVersion;
     }
 
+    public SslInfo getSslInfo() {
+        return sslInfo;
+    }
+
+    public void setSslInfo(SslInfo sslInfo) {
+        this.sslInfo = sslInfo;
+    }
 }

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationSHA2Packet.java

@@ -0,0 +1,65 @@
+package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.Capability;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter;
+
+import org.apache.commons.lang.StringUtils;
+
+public class ClientAuthenticationSHA2Packet extends ClientAuthenticationPacket {
+
+    @Override
+    public byte[] toBytes() throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        // 1. write client_flags
+        int clientCapability = 0;
+        clientCapability |= Capability.CLIENT_LONG_PASSWORD;
+        clientCapability |= Capability.CLIENT_LONG_FLAG;
+        clientCapability |= Capability.CLIENT_PROTOCOL_41;
+        clientCapability |= Capability.CLIENT_TRANSACTIONS;
+        clientCapability |= Capability.CLIENT_SECURE_CONNECTION;
+        clientCapability |= Capability.CLIENT_MULTI_STATEMENTS;
+        clientCapability |= Capability.CLIENT_PLUGIN_AUTH;
+        clientCapability |= Capability.CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA;
+        if (getDatabaseName() != null) {
+            clientCapability |= Capability.CLIENT_CONNECT_WITH_DB;
+        }
+        ByteHelper.writeUnsignedIntLittleEndian(clientCapability, out);
+
+        // 2. write max_packet_size
+        ByteHelper.writeUnsignedIntLittleEndian(MSC.MAX_PACKET_LENGTH, out);
+        // 3. write charset_number
+        out.write(getCharsetNumber());
+        // 4. write (filler) always 0x00...
+        out.write(new byte[23]);
+        // 5. write (Null-Terminated String) user
+        ByteHelper.writeNullTerminatedString(getUsername(), out);
+        // 6. write (Length Coded Binary) scramble_buff (1 + x bytes)
+        if (StringUtils.isEmpty(getPassword())) {
+            out.write(0x00);
+        } else {
+            try {
+                byte[] encryptedPassword = MySQLPasswordEncrypter.scrambleCachingSha2(getPassword().getBytes(),
+                    getScrumbleBuff());
+                ByteHelper.writeBinaryCodedLengthBytes(encryptedPassword, out);
+            } catch (Exception e) {
+                throw new IOException("can't encrypt password that will be sent to MySQL server.", e);
+            }
+        }
+        // 7 . (Null-Terminated String) databasename (optional)
+        if (getDatabaseName() != null) {
+            ByteHelper.writeNullTerminatedString(getDatabaseName(), out);
+        }
+        // 8 . (Null-Terminated String) auth plugin name (optional)
+        if (getAuthPluginName() != null) {
+            ByteHelper.writeNullTerminated(getAuthPluginName(), out);
+        }
+        // end write
+        return out.toByteArray();
+    }
+
+}

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SslRequestCommandPacket.java

@@ -0,0 +1,41 @@
+package com.alibaba.otter.canal.parse.driver.mysql.packets.client;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.Capability;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.IPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper;
+
+/**
+ * @author 枭博
+ * @date 2024/05/14
+ */
+public class SslRequestCommandPacket implements IPacket {
+
+    private final int serverCharsetNumber;
+
+    public SslRequestCommandPacket(int serverCharsetNumber){
+        this.serverCharsetNumber = serverCharsetNumber;
+    }
+
+    @Override
+    public void fromBytes(byte[] data) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public byte[] toBytes() throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        int clientCapabilities = Capability.CLIENT_LONG_FLAG | Capability.CLIENT_PROTOCOL_41
+                                 | Capability.CLIENT_SECURE_CONNECTION | Capability.CLIENT_PLUGIN_AUTH
+                                 | Capability.CLIENT_SSL;
+        ByteHelper.writeUnsignedIntLittleEndian(clientCapabilities, out);
+        ByteHelper.writeUnsignedIntLittleEndian(0, out);
+        out.write(serverCharsetNumber);
+        for (int i = 0; i < 23; i++) {
+            out.write(0);
+        }
+        return out.toByteArray();
+    }
+}

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java

@@ -10,6 +10,8 @@ import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.nio.channels.ClosedByInterruptException;
 
+import javax.net.ssl.SSLSocket;
+
 /**
  * 使用BIO进行dump
  *
@@ -22,11 +24,13 @@ public class BioSocketChannel implements SocketChannel {
     private Socket        socket;
     private InputStream   input;
     private OutputStream  output;
+    private final boolean ssl;
 
     BioSocketChannel(Socket socket) throws IOException{
         this.socket = socket;
         this.input = new BufferedInputStream(socket.getInputStream(), 16384);
         this.output = socket.getOutputStream();
+        this.ssl = (socket instanceof SSLSocket);
     }
 
     public void write(byte[]... buf) throws IOException {
@@ -88,9 +92,9 @@ public class BioSocketChannel implements SocketChannel {
             }
         }
         if (remain > 0 && accTimeout >= timeout) {
-            throw new SocketTimeoutException("Timeout occurred, failed to read total " + readSize + " bytes in "
+            throw new SocketTimeoutException(
-                                             + timeout + " milliseconds, actual read only " + (readSize - remain)
+                "Timeout occurred, failed to read total " + readSize + " bytes in " + timeout
-                                             + " bytes");
+                                             + " milliseconds, actual read only " + (readSize - remain) + " bytes");
         }
         return data;
     }
@@ -134,6 +138,14 @@ public class BioSocketChannel implements SocketChannel {
         return false;
     }
 
+    public boolean isSsl() {
+        return ssl;
+    }
+
+    public Socket getSocket() {
+        return socket;
+    }
+
     public SocketAddress getRemoteSocketAddress() {
         Socket socket = this.socket;
         if (socket != null) {
@@ -155,16 +167,6 @@ public class BioSocketChannel implements SocketChannel {
     public void close() {
         Socket socket = this.socket;
         if (socket != null) {
-            try {
-                socket.shutdownInput();
-            } catch (IOException e) {
-                // Ignore, could not do anymore
-            }
-            try {
-                socket.shutdownOutput();
-            } catch (IOException e) {
-                // Ignore, could not do anymore
-            }
             try {
                 socket.close();
             } catch (IOException e) {

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java

@@ -1,7 +1,28 @@
 package com.alibaba.otter.canal.parse.driver.mysql.socket;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.net.Socket;
 import java.net.SocketAddress;
+import java.net.URL;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.KeyStore;
+import java.security.cert.*;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import javax.naming.InvalidNameException;
+import javax.naming.ldap.LdapName;
+import javax.naming.ldap.Rdn;
+import javax.net.ssl.*;
+import javax.security.auth.x500.X500Principal;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslMode;
 
 /**
  * @author luoyaogui 实现channel的管理（监听连接、读数据、回收） 2016-12-28
@@ -9,14 +30,295 @@ import java.net.SocketAddress;
  */
 public abstract class BioSocketChannelPool {
 
+    private static final Logger logger = LoggerFactory.getLogger(BioSocketChannelPool.class);
+
     public static BioSocketChannel open(SocketAddress address) throws Exception {
-        Socket socket = new Socket();
+        Socket socket = createSocket(address);
+        return new BioSocketChannel(socket);
+    }
+
+    public static BioSocketChannel openSsl(Socket socket, SslInfo sslInfo) throws Exception {
+        SslMode sslMode = sslInfo.getSslMode();
+        switch (sslMode) {
+            case REQUIRED:
+            case PREFERRED:
+            case VERIFY_CA:
+            case VERIFY_IDENTITY:
+                SSLSocket sslSocket = createSslSocket(socket, sslInfo);
+                return new BioSocketChannel(sslSocket);
+            default:
+                throw new UnsupportedOperationException("Unsupported ssl mode: " + sslMode);
+        }
+    }
+
+    private static Socket createSocket(SocketAddress address) throws IOException {
+        Socket socket;
+        socket = new Socket();
         socket.setSoTimeout(BioSocketChannel.SO_TIMEOUT);
         socket.setTcpNoDelay(true);
         socket.setKeepAlive(true);
         socket.setReuseAddress(true);
         socket.connect(address, BioSocketChannel.DEFAULT_CONNECT_TIMEOUT);
-        return new BioSocketChannel(socket);
+        return socket;
+    }
+
+    /**
+     * from JDBC driver com.mysql.cj.protocol.ExportControlled#performTlsHandshake
+     * com.mysql.cj.protocol.ExportControlled#getSSLContext
+     *
+     * @param socket
+     * @param sslInfo
+     * @return
+     * @throws Exception
+     */
+    private static SSLSocket createSslSocket(Socket socket, SslInfo sslInfo) throws Exception {
+        SslMode sslMode = sslInfo.getSslMode();
+        boolean verifyServerCert = sslMode == SslMode.VERIFY_CA || sslMode == SslMode.VERIFY_IDENTITY;
+
+        String clientCertificateKeyStoreUrl = sslInfo.getClientCertificateKeyStoreUrl();
+        String clientCertificateKeyStoreType = sslInfo.getClientCertificateKeyStoreType() != null ? sslInfo
+            .getClientCertificateKeyStoreType() : "JKS";
+        String clientCertificateKeyStorePassword = sslInfo.getClientCertificateKeyStorePassword();
+        String trustCertificateKeyStoreUrl = sslInfo.getTrustCertificateKeyStoreUrl();
+        String trustCertificateKeyStoreType = sslInfo.getTrustCertificateKeyStoreType() != null ? sslInfo
+            .getTrustCertificateKeyStoreType() : "JKS";
+        String trustCertificateKeyStorePassword = sslInfo.getTrustCertificateKeyStorePassword();
+        boolean fallbackToDefaultTrustStore = true;
+        String hostName = sslMode == SslMode.VERIFY_IDENTITY ? socket.getInetAddress().getHostName() : null;
+
+        SSLContext sslContext = getSSLContext(clientCertificateKeyStoreUrl,
+            clientCertificateKeyStoreType,
+            clientCertificateKeyStorePassword,
+            trustCertificateKeyStoreUrl,
+            trustCertificateKeyStoreType,
+            trustCertificateKeyStorePassword,
+            fallbackToDefaultTrustStore,
+            verifyServerCert,
+            hostName);
+        SSLSocketFactory socketFactory = sslContext.getSocketFactory();
+
+        SSLSocket sslSocket = (SSLSocket) socketFactory
+            .createSocket(socket, socket.getInetAddress().getHostName(), socket.getPort(), true);
+
+        String[] protocolArr = null;
+        if (StringUtils.isNotEmpty(sslInfo.getTlsVersions())) {
+            protocolArr = StringUtils.split(sslInfo.getTlsVersions(), ",");
+        }
+        if (protocolArr == null || protocolArr.length == 0) {
+            protocolArr = new String[] { "TLSv1.2", "TLSv1.3" };
+        }
+        logger.info("SSL protocol: {}", StringUtils.join(protocolArr, ","));
+        sslSocket.setEnabledProtocols(protocolArr);
+        sslSocket.setSoTimeout(BioSocketChannel.SO_TIMEOUT);
+        sslSocket.startHandshake();
+        logger.info("SSL socket handshake success.");
+        return sslSocket;
+    }
+
+    private static SSLContext getSSLContext(String clientCertificateKeyStoreUrl, String clientCertificateKeyStoreType,
+                                            String clientCertificateKeyStorePassword,
+                                            String trustCertificateKeyStoreUrl, String trustCertificateKeyStoreType,
+                                            String trustCertificateKeyStorePassword,
+                                            boolean fallbackToDefaultTrustStore, boolean verifyServerCert,
+                                            String hostName) throws Exception {
+
+        KeyManager[] kms = null;
+        List<TrustManager> tms = new ArrayList<>();
+
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+
+        if (StringUtils.isNotEmpty(clientCertificateKeyStoreUrl)) {
+            InputStream ksIS = null;
+            try {
+                if (StringUtils.isNotEmpty(clientCertificateKeyStoreType)) {
+                    KeyStore clientKeyStore = KeyStore.getInstance(clientCertificateKeyStoreType);
+                    URL ksURL = new URL(clientCertificateKeyStoreUrl);
+                    char[] password = (clientCertificateKeyStorePassword == null) ? new char[0] : clientCertificateKeyStorePassword
+                        .toCharArray();
+                    ksIS = ksURL.openStream();
+                    clientKeyStore.load(ksIS, password);
+                    kmf.init(clientKeyStore, password);
+                    kms = kmf.getKeyManagers();
+                }
+            } finally {
+                if (ksIS != null) {
+                    try {
+                        ksIS.close();
+                    } catch (IOException e) {
+                        // can't close input stream, but keystore can be properly initialized so we
+                        // shouldn't throw this exception
+                    }
+                }
+            }
+        }
+
+        InputStream trustStoreIS = null;
+        try {
+            String trustStoreType = "";
+            char[] trustStorePassword = null;
+            KeyStore trustKeyStore = null;
+
+            if (StringUtils.isNotEmpty(trustCertificateKeyStoreUrl)
+                && StringUtils.isNotEmpty(trustCertificateKeyStoreType)) {
+                trustStoreType = trustCertificateKeyStoreType;
+                trustStorePassword = (trustCertificateKeyStorePassword == null) ? new char[0] : trustCertificateKeyStorePassword
+                    .toCharArray();
+                trustStoreIS = new URL(trustCertificateKeyStoreUrl).openStream();
+
+                trustKeyStore = KeyStore.getInstance(trustStoreType);
+                trustKeyStore.load(trustStoreIS, trustStorePassword);
+            }
+
+            if (trustKeyStore != null || fallbackToDefaultTrustStore) {
+                tmf.init(trustKeyStore);
+                // (trustKeyStore == null) initializes the TrustManagerFactory with the default
+                // truststore.
+
+                // building the customized list of TrustManagers from original one if it's
+                // available
+                TrustManager[] origTms = tmf.getTrustManagers();
+
+                for (TrustManager tm : origTms) {
+                    // wrap X509TrustManager or put original if non-X509 TrustManager
+                    tms.add(tm instanceof X509TrustManager ? new X509TrustManagerWrapper((X509TrustManager) tm,
+                        verifyServerCert,
+                        hostName) : tm);
+                }
+            }
+
+        } finally {
+            if (trustStoreIS != null) {
+                try {
+                    trustStoreIS.close();
+                } catch (IOException e) {
+                    // can't close input stream, but keystore can be properly initialized so we
+                    // shouldn't throw this exception
+                }
+            }
+        }
+
+        // if original TrustManagers are not available then putting one
+        // X509TrustManagerWrapper which take care only
+        // about expiration check
+        if (tms.size() == 0) {
+            tms.add(new X509TrustManagerWrapper(verifyServerCert, hostName));
+        }
+
+        SSLContext sslContext = SSLContext.getInstance("TLS");
+        sslContext.init(kms, tms.toArray(new TrustManager[tms.size()]), null);
+        return sslContext;
+    }
+
+    public static class X509TrustManagerWrapper implements X509TrustManager {
+
+        private X509TrustManager   origTm           = null;
+        private boolean            verifyServerCert = false;
+        private String             hostName         = null;
+        private CertificateFactory certFactory      = null;
+        private PKIXParameters     validatorParams  = null;
+        private CertPathValidator  validator        = null;
+
+        public X509TrustManagerWrapper(X509TrustManager tm, boolean verifyServerCertificate,
+                                       String hostName) throws CertificateException{
+            this.origTm = tm;
+            this.verifyServerCert = verifyServerCertificate;
+            this.hostName = hostName;
+
+            if (verifyServerCertificate) {
+                try {
+                    Set<TrustAnchor> anch = Arrays.stream(tm.getAcceptedIssuers())
+                        .map(c -> new TrustAnchor(c, null))
+                        .collect(Collectors.toSet());
+                    this.validatorParams = new PKIXParameters(anch);
+                    this.validatorParams.setRevocationEnabled(false);
+                    this.validator = CertPathValidator.getInstance("PKIX");
+                    this.certFactory = CertificateFactory.getInstance("X.509");
+                } catch (Exception e) {
+                    throw new CertificateException(e);
+                }
+            }
+
+        }
+
+        public X509TrustManagerWrapper(boolean verifyServerCertificate, String hostName){
+            this.verifyServerCert = verifyServerCertificate;
+            this.hostName = hostName;
+        }
+
+        public X509Certificate[] getAcceptedIssuers() {
+            return this.origTm != null ? this.origTm.getAcceptedIssuers() : new X509Certificate[0];
+        }
+
+        public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+            for (int i = 0; i < chain.length; i++) {
+                chain[i].checkValidity();
+            }
+
+            if (this.validatorParams != null) {
+                X509CertSelector certSelect = new X509CertSelector();
+                certSelect.setSerialNumber(chain[0].getSerialNumber());
+
+                try {
+                    CertPath certPath = this.certFactory.generateCertPath(Arrays.asList(chain));
+                    // Validate against truststore
+                    CertPathValidatorResult result = this.validator.validate(certPath, this.validatorParams);
+                    // Check expiration for the CA used to validate this path
+                    ((PKIXCertPathValidatorResult) result).getTrustAnchor().getTrustedCert().checkValidity();
+
+                } catch (InvalidAlgorithmParameterException e) {
+                    throw new CertificateException(e);
+                } catch (CertPathValidatorException e) {
+                    throw new CertificateException(e);
+                }
+            }
+
+            if (this.verifyServerCert) {
+                if (this.origTm != null) {
+                    this.origTm.checkServerTrusted(chain, authType);
+                } else {
+                    throw new CertificateException(
+                        "Can't verify server certificate because no trust manager is found.");
+                }
+
+                // verify server certificate identity
+                if (this.hostName != null) {
+                    logger.info("verify hostName: {}", this.hostName);
+                    Set<String> expectHostNames = new HashSet<>();
+                    for (X509Certificate certificate : chain) {
+                        String dn = certificate.getSubjectX500Principal().getName(X500Principal.RFC2253);
+                        String cn = null;
+                        try {
+                            LdapName ldapDN = new LdapName(dn);
+                            for (Rdn rdn : ldapDN.getRdns()) {
+                                if (rdn.getType().equalsIgnoreCase("CN")) {
+                                    cn = rdn.getValue().toString();
+                                    break;
+                                }
+                            }
+                        } catch (InvalidNameException e) {
+                            throw new CertificateException(
+                                "Failed to retrieve the Common Name (CN) from the server certificate.");
+                        }
+                        expectHostNames.add(cn);
+                    }
+
+                    if (!expectHostNames.contains(this.hostName)) {
+                        throw new CertificateException(
+                            "Server certificate identity check failed. The certificate Common Name "
+                                                       + expectHostNames.stream()
+                                                           .map(h -> "'" + h + "'")
+                                                           .collect(Collectors.joining(", "))
+                                                       + " does not match with '" + this.hostName + "'.");
+                    }
+
+                }
+            }
+        }
+
+        public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+            this.origTm.checkClientTrusted(chain, authType);
+        }
     }
 
 }

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java

@@ -1,8 +1,14 @@
 package com.alibaba.otter.canal.parse.driver.mysql.socket;
 
+import java.io.IOException;
 import java.net.SocketAddress;
 
 import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslMode;
 
 /**
  * @author agapple 2018年3月12日 下午10:46:22
@@ -10,6 +16,8 @@ import org.apache.commons.lang.StringUtils;
  */
 public abstract class SocketChannelPool {
 
+    private static final Logger logger = LoggerFactory.getLogger(SocketChannelPool.class);
+
     public static SocketChannel open(SocketAddress address) throws Exception {
         String type = chooseSocketChannel();
         if ("netty".equalsIgnoreCase(type)) {
@@ -17,7 +25,32 @@ public abstract class SocketChannelPool {
         } else {
             return BioSocketChannelPool.open(address);
         }
+    }
 
+    public static SocketChannel connectSsl(SocketChannel channel, SslInfo sslInfo) throws IOException {
+        SslMode sslMode = sslInfo.getSslMode();
+        String type = chooseSocketChannel();
+        if ("netty".equalsIgnoreCase(type)) {
+            throw new UnsupportedOperationException("canal socketChannel netty not support ssl mode: " + sslMode);
+        } else {
+            SocketAddress remoteSocketAddress = channel.getRemoteSocketAddress();
+            try {
+                return BioSocketChannelPool.openSsl(((BioSocketChannel) channel).getSocket(), sslInfo);
+            } catch (Exception e) {
+                if (sslMode == SslMode.PREFERRED) {
+                    // still use non ssl channel
+                    logger.info("{} still use non SSL channel due to SSL connect failed.", remoteSocketAddress, e);
+                    return channel;
+                }
+                IOException ioe;
+                if (e instanceof IOException) {
+                    ioe = (IOException) e;
+                } else {
+                    ioe = new IOException(e);
+                }
+                throw ioe;
+            }
+        }
     }
 
     private static String chooseSocketChannel() {

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/ssl/SslInfo.java

@@ -0,0 +1,98 @@
+package com.alibaba.otter.canal.parse.driver.mysql.ssl;
+
+/**
+ * @author 枭博
+ * @date 2024/05/14
+ */
+public class SslInfo {
+
+    private SslMode sslMode = SslMode.DISABLED;
+    private String  tlsVersions;                       // 和 enabledTLSProtocols 同含义，TLSv1.2,TLSv1.3
+    private String  trustCertificateKeyStoreType;      // trustStore 证书类型，支持 JKS (默认) 和 PKCS12
+    private String  trustCertificateKeyStoreUrl;       // trustStore 证书
+    private String  trustCertificateKeyStorePassword;  // trustStore 证书密码
+    private String  clientCertificateKeyStoreType;     // client 证书类型，支持 JKS (默认) 和 PKCS12
+    private String  clientCertificateKeyStoreUrl;      // client 证书
+    private String  clientCertificateKeyStorePassword; // client 证书密码
+
+    public SslInfo(SslMode sslMode, String tlsVersions, String trustCertificateKeyStoreType,
+                   String trustCertificateKeyStoreUrl, String trustCertificateKeyStorePassword,
+                   String clientCertificateKeyStoreType, String clientCertificateKeyStoreUrl,
+                   String clientCertificateKeyStorePassword) {
+        this.sslMode = sslMode;
+        this.tlsVersions = tlsVersions;
+        this.trustCertificateKeyStoreType = trustCertificateKeyStoreType;
+        this.trustCertificateKeyStoreUrl = trustCertificateKeyStoreUrl;
+        this.trustCertificateKeyStorePassword = trustCertificateKeyStorePassword;
+        this.clientCertificateKeyStoreType = clientCertificateKeyStoreType;
+        this.clientCertificateKeyStoreUrl = clientCertificateKeyStoreUrl;
+        this.clientCertificateKeyStorePassword = clientCertificateKeyStorePassword;
+    }
+
+    public SslInfo() {
+    }
+
+    public SslMode getSslMode() {
+        return sslMode;
+    }
+
+    public void setSslMode(SslMode sslMode) {
+        this.sslMode = sslMode;
+    }
+
+    public String getTlsVersions() {
+        return tlsVersions;
+    }
+
+    public void setTlsVersions(String tlsVersions) {
+        this.tlsVersions = tlsVersions;
+    }
+
+    public String getTrustCertificateKeyStoreType() {
+        return trustCertificateKeyStoreType;
+    }
+
+    public void setTrustCertificateKeyStoreType(String trustCertificateKeyStoreType) {
+        this.trustCertificateKeyStoreType = trustCertificateKeyStoreType;
+    }
+
+    public String getTrustCertificateKeyStoreUrl() {
+        return trustCertificateKeyStoreUrl;
+    }
+
+    public void setTrustCertificateKeyStoreUrl(String trustCertificateKeyStoreUrl) {
+        this.trustCertificateKeyStoreUrl = trustCertificateKeyStoreUrl;
+    }
+
+    public String getTrustCertificateKeyStorePassword() {
+        return trustCertificateKeyStorePassword;
+    }
+
+    public void setTrustCertificateKeyStorePassword(String trustCertificateKeyStorePassword) {
+        this.trustCertificateKeyStorePassword = trustCertificateKeyStorePassword;
+    }
+
+    public String getClientCertificateKeyStoreType() {
+        return clientCertificateKeyStoreType;
+    }
+
+    public void setClientCertificateKeyStoreType(String clientCertificateKeyStoreType) {
+        this.clientCertificateKeyStoreType = clientCertificateKeyStoreType;
+    }
+
+    public String getClientCertificateKeyStoreUrl() {
+        return clientCertificateKeyStoreUrl;
+    }
+
+    public void setClientCertificateKeyStoreUrl(String clientCertificateKeyStoreUrl) {
+        this.clientCertificateKeyStoreUrl = clientCertificateKeyStoreUrl;
+    }
+
+    public String getClientCertificateKeyStorePassword() {
+        return clientCertificateKeyStorePassword;
+    }
+
+    public void setClientCertificateKeyStorePassword(String clientCertificateKeyStorePassword) {
+        this.clientCertificateKeyStorePassword = clientCertificateKeyStorePassword;
+    }
+}

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/ssl/SslMode.java

@@ -0,0 +1,26 @@
+package com.alibaba.otter.canal.parse.driver.mysql.ssl;
+
+public enum SslMode {
+
+    /**
+     * 关闭SSL
+     */
+    DISABLED,
+    /**
+     * 尝试SSL传输
+     */
+    PREFERRED,
+    /**
+     * 要求SSL传输，不校验证书
+     */
+    REQUIRED,
+    /**
+     * 要求SSL传输，校验证书，不校验证书里的域名
+     */
+    VERIFY_CA,
+    /**
+     * 要求SSL传输，校验证书，校验证书里的域名
+     */
+    VERIFY_IDENTITY
+
+}

instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java

@@ -276,8 +276,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
             mysqlEventParser.setDestination(destination);
             // 编码参数
             mysqlEventParser.setConnectionCharset(parameters.getConnectionCharset());
-            mysqlEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
+            // 网络相关参数1
-            // 网络相关参数
             mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
             mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
             mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
@@ -291,13 +290,15 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
                 mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
                     parameters.getDbUsername(),
                     parameters.getDbPassword(),
-                    parameters.getDefaultDatabaseName()));
+                    parameters.getDefaultDatabaseName(),
+                    parameters.getSslInfo()));
 
                 if (dbAddresses.size() > 1) {
                     mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1),
                         parameters.getDbUsername(),
                         parameters.getDbPassword(),
-                        parameters.getDefaultDatabaseName()));
+                        parameters.getDefaultDatabaseName(),
+                        parameters.getSslInfo()));
                 }
             }
 
@@ -369,7 +370,6 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
             localBinlogEventParser.setDestination(destination);
             localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
             localBinlogEventParser.setConnectionCharset(parameters.getConnectionCharset());
-            localBinlogEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
             localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
             localBinlogEventParser.setProfilingEnabled(false);
             localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
@@ -381,7 +381,8 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
                 localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
                     parameters.getDbUsername(),
                     parameters.getDbPassword(),
-                    parameters.getDefaultDatabaseName()));
+                    parameters.getDefaultDatabaseName(),
+                    parameters.getSslInfo()));
             }
 
             eventParser = localBinlogEventParser;

instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/PlainCanalInstanceGenerator.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.instance.manager;
 
 import java.util.Properties;
 
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.BeanFactory;
@@ -69,6 +70,9 @@ public class PlainCanalInstanceGenerator implements CanalInstanceGenerator {
     // ================ setter / getter ================
 
     private BeanFactory getBeanFactory(String springXml) {
+        if (!StringUtils.startsWithIgnoreCase(springXml, "classpath:")) {
+            springXml = "classpath:" + springXml;
+        }
         ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
         return applicationContext;
     }

instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java

@@ -9,6 +9,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.builder.ToStringBuilder;
 
 import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslMode;
 
 /**
  * canal运行相关参数
@@ -56,8 +58,6 @@ public class CanalParameter implements Serializable {
     private Integer                  defaultConnectionTimeoutInSeconds  = 30;                        // sotimeout
     private Integer                  receiveBufferSize                  = 64 * 1024;
     private Integer                  sendBufferSize                     = 64 * 1024;
-    // 编码信息
-    private Byte                     connectionCharsetNumber            = (byte) 33;
     private String                   connectionCharset                  = "UTF-8";
 
     // 数据库信息
@@ -66,6 +66,21 @@ public class CanalParameter implements Serializable {
     private String                   dbUsername;                                                     // 数据库用户
     private String                   dbPassword;                                                     // 数据库密码
 
+    private String                   sslMode                            = SslMode.DISABLED.name();
+    private String                   tlsVersions;                                                    // 和
+                                                                                                     // enabledTLSProtocols
+                                                                                                     // 同含义，TLSv1.2,TLSv1.3
+    private String                   trustCertificateKeyStoreType;                                   // trustStore
+                                                                                                     // 证书类型，支持 JKS (默认)
+                                                                                                     // 和 PKCS12
+    private String                   trustCertificateKeyStoreUrl;                                    // trustStore 证书路径
+    private String                   trustCertificateKeyStorePassword;                               // trustStore 证书密码
+    private String                   clientCertificateKeyStoreType;                                  // client 证书类型，支持
+                                                                                                     // JKS (默认) 和
+                                                                                                     // PKCS12
+    private String                   clientCertificateKeyStoreUrl;                                   // client 证书路径
+    private String                   clientCertificateKeyStorePassword;                              // client 证书密码
+
     // binlog链接信息
     private IndexMode                indexMode;
     private List<String>             positions;                                                      // 数据库positions信息
@@ -108,10 +123,39 @@ public class CanalParameter implements Serializable {
     private InetSocketAddress        masterAddress;                                                  // 主库信息
     private String                   masterUsername;                                                 // 帐号
     private String                   masterPassword;                                                 // 密码
+    private String                   masterSslMode                      = SslMode.DISABLED.name();
+    private String                   masterTlsVersions;                                              // 和
+                                                                                                     // enabledTLSProtocols
+                                                                                                     // 同含义，TLSv1.2,TLSv1.3
+    private String                   masterTrustCertificateKeyStoreType;                             // trustStore
+                                                                                                     // 证书类型，支持 JKS (默认)
+                                                                                                     // 和 PKCS12
+    private String                   masterTrustCertificateKeyStoreUrl;                              // trustStore 证书路径
+    private String                   masterTrustCertificateKeyStorePassword;                         // trustStore 证书密码
+    private String                   masterClientCertificateKeyStoreType;                            // client 证书类型，支持
+                                                                                                     // JKS (默认) 和
+                                                                                                     // PKCS12
+    private String                   masterClientCertificateKeyStoreUrl;                             // client 证书路径
+    private String                   masterClientCertificateKeyStorePassword;                        // client 证书密码
 
     private InetSocketAddress        standbyAddress;                                                 // 备库信息
     private String                   standbyUsername;                                                // 帐号
     private String                   standbyPassword;
+    private String                   standbySslMode                     = SslMode.DISABLED.name();
+    private String                   standbyTlsVersions;                                             // 和
+                                                                                                     // enabledTLSProtocols
+                                                                                                     // 同含义，TLSv1.2,TLSv1.3
+    private String                   standbyTrustCertificateKeyStoreType;                            // trustStore
+                                                                                                     // 证书类型，支持 JKS (默认)
+                                                                                                     // 和 PKCS12
+    private String                   standbyTrustCertificateKeyStoreUrl;                             // trustStore 证书路径
+    private String                   standbyTrustCertificateKeyStorePassword;                        // trustStore 证书密码
+    private String                   standbyClientCertificateKeyStoreType;                           // client 证书类型，支持
+                                                                                                     // JKS (默认) 和
+                                                                                                     // PKCS12
+    private String                   standbyClientCertificateKeyStoreUrl;                            // client 证书路径
+    private String                   standbyClientCertificateKeyStorePassword;                       // client 证书密码
+
     private String                   masterLogfileName                  = null;                      // master起始位置
     private Long                     masterLogfileOffest                = null;
     private Long                     masterTimestamp                    = null;
@@ -120,11 +164,11 @@ public class CanalParameter implements Serializable {
     private Long                     standbyTimestamp                   = null;
     private Boolean                  parallel                           = Boolean.FALSE;
 
-    //自定义alarmHandler类全路径
+    // 自定义alarmHandler类全路径
     private String                   alarmHandlerClass                  = null;
-    //自定义alarmHandler插件文件夹路径
+    // 自定义alarmHandler插件文件夹路径
     private String                   alarmHandlerPluginDir              = null;
-    //是否支持多流消费
+    // 是否支持多流消费
     private Boolean                  multiStreamEnable                  = Boolean.FALSE;
 
     public static enum RunMode {
@@ -183,6 +227,7 @@ public class CanalParameter implements Serializable {
     }
 
     public static enum StorageMode {
+
         /** 内存存储模式 */
         MEMORY,
         /** 文件存储模式 */
@@ -205,6 +250,7 @@ public class CanalParameter implements Serializable {
     }
 
     public static enum StorageScavengeMode {
+
         /** 在存储满的时候触发 */
         ON_FULL,
         /** 在每次有ack请求时触发 */
@@ -232,6 +278,7 @@ public class CanalParameter implements Serializable {
     }
 
     public static enum SourcingType {
+
         /** mysql DB */
         MYSQL,
         /** localBinLog */
@@ -259,6 +306,7 @@ public class CanalParameter implements Serializable {
     }
 
     public static enum MetaMode {
+
         /** 内存存储模式 */
         MEMORY,
         /** 文件存储模式 */
@@ -286,6 +334,7 @@ public class CanalParameter implements Serializable {
     }
 
     public static enum IndexMode {
+
         /** 内存存储模式 */
         MEMORY,
         /** 文件存储模式 */
@@ -319,6 +368,7 @@ public class CanalParameter implements Serializable {
     }
 
     public static enum BatchMode {
+
         /** 对象数量 */
         ITEMSIZE,
 
@@ -533,14 +583,6 @@ public class CanalParameter implements Serializable {
         this.sendBufferSize = sendBufferSize;
     }
 
-    public Byte getConnectionCharsetNumber() {
-        return connectionCharsetNumber;
-    }
-
-    public void setConnectionCharsetNumber(Byte connectionCharsetNumber) {
-        this.connectionCharsetNumber = connectionCharsetNumber;
-    }
-
     public String getConnectionCharset() {
         return connectionCharset;
     }
@@ -731,6 +773,134 @@ public class CanalParameter implements Serializable {
         this.dbPassword = dbPassword;
     }
 
+    public SslInfo getSslInfo() {
+        if (dbUsername == null) {
+            if (masterUsername != null) {
+                return new SslInfo(SslMode.valueOf(masterSslMode),
+                    masterTlsVersions,
+                    masterTrustCertificateKeyStoreType,
+                    masterTrustCertificateKeyStoreUrl,
+                    masterTrustCertificateKeyStorePassword,
+                    masterClientCertificateKeyStoreType,
+                    masterClientCertificateKeyStoreUrl,
+                    masterClientCertificateKeyStorePassword);
+            } else {
+                return new SslInfo(SslMode.valueOf(standbySslMode),
+                    standbyTlsVersions,
+                    standbyTrustCertificateKeyStoreType,
+                    standbyTrustCertificateKeyStoreUrl,
+                    standbyTrustCertificateKeyStorePassword,
+                    standbyClientCertificateKeyStoreType,
+                    standbyClientCertificateKeyStoreUrl,
+                    standbyClientCertificateKeyStorePassword);
+            }
+        }
+        return new SslInfo(SslMode.valueOf(sslMode),
+            tlsVersions,
+            trustCertificateKeyStoreType,
+            trustCertificateKeyStoreUrl,
+            trustCertificateKeyStorePassword,
+            clientCertificateKeyStoreType,
+            clientCertificateKeyStoreUrl,
+            clientCertificateKeyStorePassword);
+    }
+
+    public void setSslMode(String sslMode) {
+        this.sslMode = sslMode;
+    }
+
+    public void setTlsVersions(String tlsVersions) {
+        this.tlsVersions = tlsVersions;
+    }
+
+    public void setTrustCertificateKeyStoreType(String trustCertificateKeyStoreType) {
+        this.trustCertificateKeyStoreType = trustCertificateKeyStoreType;
+    }
+
+    public void setTrustCertificateKeyStoreUrl(String trustCertificateKeyStoreUrl) {
+        this.trustCertificateKeyStoreUrl = trustCertificateKeyStoreUrl;
+    }
+
+    public void setTrustCertificateKeyStorePassword(String trustCertificateKeyStorePassword) {
+        this.trustCertificateKeyStorePassword = trustCertificateKeyStorePassword;
+    }
+
+    public void setClientCertificateKeyStoreType(String clientCertificateKeyStoreType) {
+        this.clientCertificateKeyStoreType = clientCertificateKeyStoreType;
+    }
+
+    public void setClientCertificateKeyStoreUrl(String clientCertificateKeyStoreUrl) {
+        this.clientCertificateKeyStoreUrl = clientCertificateKeyStoreUrl;
+    }
+
+    public void setClientCertificateKeyStorePassword(String clientCertificateKeyStorePassword) {
+        this.clientCertificateKeyStorePassword = clientCertificateKeyStorePassword;
+    }
+
+    public void setMasterSslMode(String masterSslMode) {
+        this.masterSslMode = masterSslMode;
+    }
+
+    public void setMasterTlsVersions(String masterTlsVersions) {
+        this.masterTlsVersions = masterTlsVersions;
+    }
+
+    public void setMasterTrustCertificateKeyStoreType(String masterTrustCertificateKeyStoreType) {
+        this.masterTrustCertificateKeyStoreType = masterTrustCertificateKeyStoreType;
+    }
+
+    public void setMasterTrustCertificateKeyStoreUrl(String masterTrustCertificateKeyStoreUrl) {
+        this.masterTrustCertificateKeyStoreUrl = masterTrustCertificateKeyStoreUrl;
+    }
+
+    public void setMasterTrustCertificateKeyStorePassword(String masterTrustCertificateKeyStorePassword) {
+        this.masterTrustCertificateKeyStorePassword = masterTrustCertificateKeyStorePassword;
+    }
+
+    public void setMasterClientCertificateKeyStoreType(String masterClientCertificateKeyStoreType) {
+        this.masterClientCertificateKeyStoreType = masterClientCertificateKeyStoreType;
+    }
+
+    public void setMasterClientCertificateKeyStoreUrl(String masterClientCertificateKeyStoreUrl) {
+        this.masterClientCertificateKeyStoreUrl = masterClientCertificateKeyStoreUrl;
+    }
+
+    public void setMasterClientCertificateKeyStorePassword(String masterClientCertificateKeyStorePassword) {
+        this.masterClientCertificateKeyStorePassword = masterClientCertificateKeyStorePassword;
+    }
+
+    public void setStandbySslMode(String standbySslMode) {
+        this.standbySslMode = standbySslMode;
+    }
+
+    public void setStandbyTlsVersions(String standbyTlsVersions) {
+        this.standbyTlsVersions = standbyTlsVersions;
+    }
+
+    public void setStandbyTrustCertificateKeyStoreType(String standbyTrustCertificateKeyStoreType) {
+        this.standbyTrustCertificateKeyStoreType = standbyTrustCertificateKeyStoreType;
+    }
+
+    public void setStandbyTrustCertificateKeyStoreUrl(String standbyTrustCertificateKeyStoreUrl) {
+        this.standbyTrustCertificateKeyStoreUrl = standbyTrustCertificateKeyStoreUrl;
+    }
+
+    public void setStandbyTrustCertificateKeyStorePassword(String standbyTrustCertificateKeyStorePassword) {
+        this.standbyTrustCertificateKeyStorePassword = standbyTrustCertificateKeyStorePassword;
+    }
+
+    public void setStandbyClientCertificateKeyStoreType(String standbyClientCertificateKeyStoreType) {
+        this.standbyClientCertificateKeyStoreType = standbyClientCertificateKeyStoreType;
+    }
+
+    public void setStandbyClientCertificateKeyStoreUrl(String standbyClientCertificateKeyStoreUrl) {
+        this.standbyClientCertificateKeyStoreUrl = standbyClientCertificateKeyStoreUrl;
+    }
+
+    public void setStandbyClientCertificateKeyStorePassword(String standbyClientCertificateKeyStorePassword) {
+        this.standbyClientCertificateKeyStorePassword = standbyClientCertificateKeyStorePassword;
+    }
+
     public List<String> getPositions() {
         if (positions == null) {
             positions = new ArrayList<>();

instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.instance.spring;
 
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.BeanFactory;
@@ -44,6 +45,9 @@ public class SpringCanalInstanceGenerator implements CanalInstanceGenerator {
     }
 
     private BeanFactory getBeanFactory(String springXml) {
+        if (!StringUtils.startsWithIgnoreCase(springXml, "classpath:")) {
+            springXml = "classpath:" + springXml;
+        }
         ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
         return applicationContext;
     }

instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java

@@ -3,10 +3,11 @@ package com.alibaba.otter.canal.instance.spring.support;
 import java.beans.PropertyEditorSupport;
 import java.net.InetSocketAddress;
 
-import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.PropertyEditorRegistrar;
 import org.springframework.beans.PropertyEditorRegistry;
 
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+
 public class SocketAddressEditor extends PropertyEditorSupport implements PropertyEditorRegistrar {
 
     public void registerCustomEditors(PropertyEditorRegistry registry) {
@@ -14,7 +15,7 @@ public class SocketAddressEditor extends PropertyEditorSupport implements Proper
     }
 
     public void setAsText(String text) throws IllegalArgumentException {
-        String[] addresses = StringUtils.split(text, ":");
+        String[] addresses = AddressUtils.splitIPAndPort(text);
         if (addresses.length > 0) {
             if (addresses.length != 2) {
                 throw new RuntimeException("address[" + text + "] is illegal, eg.127.0.0.1:3306");

instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/DefaultSpringInstanceTest.java

@@ -24,7 +24,7 @@ public class DefaultSpringInstanceTest {
     @Before
     public void start() {
         System.setProperty("canal.instance.destination", "retl");
-        context = new ClassPathXmlApplicationContext(new String[] { "spring/default-instance.xml" });
+        context = new ClassPathXmlApplicationContext(new String[] { "classpath:spring/default-instance.xml" });
     }
 
     @After

instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/GroupSpringInstanceTest.java

@@ -24,7 +24,7 @@ public class GroupSpringInstanceTest {
     @Before
     public void start() {
         System.setProperty("canal.instance.destination", "retl");
-        context = new ClassPathXmlApplicationContext(new String[] { "spring/group-instance.xml" });
+        context = new ClassPathXmlApplicationContext(new String[] { "classpath:spring/group-instance.xml" });
     }
 
     @After

instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/MemorySpringInstanceTest.java

@@ -24,7 +24,7 @@ public class MemorySpringInstanceTest {
     @Before
     public void start() {
         System.setProperty("canal.instance.destination", "retl");
-        context = new ClassPathXmlApplicationContext(new String[] { "spring/memory-instance.xml" });
+        context = new ClassPathXmlApplicationContext(new String[] { "classpath:spring/memory-instance.xml" });
     }
 
     @After

instance/spring/src/test/resources/spring/default-instance.xml

@@ -114,7 +114,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 		
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 	
 		<!-- 解析位点记录 -->

instance/spring/src/test/resources/spring/file-instance.xml

@@ -100,7 +100,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 		
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 
 		<!-- 解析位点记录 -->

instance/spring/src/test/resources/spring/group-instance.xml

@@ -106,7 +106,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 		
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 	
 		<!-- 解析位点记录 -->
@@ -186,7 +185,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 		
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 	
 		<!-- 解析位点记录 -->

instance/spring/src/test/resources/spring/memory-instance.xml

@@ -97,7 +97,6 @@
 		<property name="defaultConnectionTimeoutInSeconds" value="${canal.instance.network.soTimeout:30}" />
 		
 		<!-- 解析编码 -->
-		<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
 		<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
 	
 		<!-- 解析位点记录 -->

parse/pom.xml

@@ -46,8 +46,8 @@
 			<version>${project.version}</version>
 		</dependency>
 		<dependency>
-			<groupId>com.alibaba</groupId>
+			<groupId>com.alibaba.polardbx</groupId>
-			<artifactId>druid</artifactId>
+			<artifactId>polardbx-parser</artifactId>
 		</dependency>
 		<dependency>
 			<groupId>mysql</groupId>

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java

@@ -74,6 +74,22 @@ public class TableMeta {
         throw new RuntimeException("unknow column : " + name);
     }
 
+    /**
+     * 尝试基于列名查找一下Filed信息,找不到时返回为null
+     * 
+     * @param name
+     * @return
+     */
+    public FieldMeta tryGetFieldMetaByName(String name) {
+        for (FieldMeta meta : fields) {
+            if (meta.getColumnName().equalsIgnoreCase(name)) {
+                return meta;
+            }
+        }
+
+        return null;
+    }
+
     public List<FieldMeta> getPrimaryFields() {
         List<FieldMeta> primarys = new ArrayList<>();
         for (FieldMeta meta : fields) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java

@@ -33,7 +33,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
     protected TableMetaTSDB        tableMetaTSDB;
 
     // 编码信息
-    protected byte                 connectionCharsetNumber   = (byte) 33;
     protected Charset              connectionCharset         = Charset.forName("UTF-8");
     protected boolean              filterQueryDcl            = false;
     protected boolean              filterQueryDml            = false;
@@ -205,10 +204,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
 
     // ============================ setter / getter =========================
 
-    public void setConnectionCharsetNumber(byte connectionCharsetNumber) {
-        this.connectionCharsetNumber = connectionCharsetNumber;
-    }
-
     public void setConnectionCharsetStd(Charset connectionCharset) {
         this.connectionCharset = connectionCharset;
     }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java

@@ -116,7 +116,6 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
         MysqlConnection connection = new MysqlConnection(runningInfo.getAddress(),
             runningInfo.getUsername(),
             runningInfo.getPassword(),
-            connectionCharsetNumber,
             runningInfo.getDefaultDatabaseName());
         connection.getConnector().setReceiveBufferSize(64 * 1024);
         connection.getConnector().setSendBufferSize(64 * 1024);

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java

@@ -27,6 +27,7 @@ import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCo
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.SemiAckCommandPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
@@ -73,14 +74,27 @@ public class MysqlConnection implements ErosaConnection {
         connector.setConnTimeout(connTimeout);
     }
 
-    public MysqlConnection(InetSocketAddress address, String username, String password, byte charsetNumber,
+    public MysqlConnection(InetSocketAddress address, String username, String password, String defaultSchema){
-                           String defaultSchema){
         authInfo = new AuthenticationInfo();
         authInfo.setAddress(address);
         authInfo.setUsername(username);
         authInfo.setPassword(password);
         authInfo.setDefaultDatabaseName(defaultSchema);
-        connector = new MysqlConnector(address, username, password, charsetNumber, defaultSchema);
+        connector = new MysqlConnector(address, username, password, defaultSchema);
+        // 将connection里面的参数透传下
+        connector.setSoTimeout(soTimeout);
+        connector.setConnTimeout(connTimeout);
+    }
+
+    public MysqlConnection(InetSocketAddress address, String username, String password, String defaultSchema,
+                           SslInfo sslInfo){
+        authInfo = new AuthenticationInfo();
+        authInfo.setAddress(address);
+        authInfo.setUsername(username);
+        authInfo.setPassword(password);
+        authInfo.setDefaultDatabaseName(defaultSchema);
+        authInfo.setSslInfo(sslInfo);
+        connector = new MysqlConnector(address, username, password, defaultSchema, sslInfo);
         // 将connection里面的参数透传下
         connector.setSoTimeout(soTimeout);
         connector.setConnTimeout(connTimeout);
@@ -589,6 +603,32 @@ public class MysqlConnection implements ErosaConnection {
         }
     }
 
+    /**
+     * MySQL 8.4版本开始部分命令出现变化
+     * https://dev.mysql.com/doc/relnotes/mysql/8.4/en/news-8-4-0.html#mysqld-8-4-0-deprecation-removal
+     * 
+     * @param major
+     * @param minor
+     * @return
+     */
+    public boolean atLeast(int major, int minor) {
+        if (isMariaDB()) {
+            return false;
+        }
+        String version = connector.getServerVersion();
+        if (StringUtils.isNotEmpty(version)) {
+            String[] parts = version.split("\\.");
+            int majorVer = Integer.parseInt(parts[0]);
+            int minorVer = Integer.parseInt(parts[1]);
+            return (majorVer > major) || (majorVer == major && minorVer >= minor);
+        }
+        return false;
+    }
+
+    public boolean atLeastMySQL84() {
+        return atLeast(8, 4);
+    }
+
     private void accumulateReceivedBytes(long x) {
         if (receivedBinlogBytes != null) {
             receivedBinlogBytes.addAndGet(x);
@@ -613,7 +653,7 @@ public class MysqlConnection implements ErosaConnection {
 
         private String value;
 
-        private BinlogFormat(String value){
+        private BinlogFormat(String value) {
             this.value = value;
         }
 
@@ -653,7 +693,7 @@ public class MysqlConnection implements ErosaConnection {
 
         private String value;
 
-        private BinlogImage(String value){
+        private BinlogImage(String value) {
             this.value = value;
         }
 

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java

@@ -305,8 +305,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         MysqlConnection connection = new MysqlConnection(runningInfo.getAddress(),
             runningInfo.getUsername(),
             runningInfo.getPassword(),
-            connectionCharsetNumber,
+            runningInfo.getDefaultDatabaseName(),
-            runningInfo.getDefaultDatabaseName());
+            runningInfo.getSslInfo());
         connection.getConnector().setReceiveBufferSize(receiveBufferSize);
         connection.getConnector().setSendBufferSize(sendBufferSize);
         connection.getConnector().setSoTimeout(defaultConnectionTimeoutInSeconds * 1000);
@@ -668,13 +668,20 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
      * 查询当前的binlog位置
      */
     private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
+        String showSql = "show master status";
         try {
-            String showSql = multiStreamEnable ? "show master status with " + destination : "show master status";
+            if (mysqlConnection.atLeastMySQL84()) {
+                // 8.4新语法
+                showSql = "show binary log status";
+            } else if (multiStreamEnable) {
+                // 兼容polardb-x的多流binlog
+                showSql = "show master status with " + destination;
+            }
             ResultSetPacket packet = mysqlConnection.query(showSql);
             List<String> fields = packet.getFieldValues();
             if (CollectionUtils.isEmpty(fields)) {
                 throw new CanalParseException(
-                        "command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
+                    "command : '" + showSql + "' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
             }
             EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
             if (isGTIDMode() && fields.size() > 4) {
@@ -690,7 +697,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
             }
             return endPosition;
         } catch (IOException e) {
-            throw new CanalParseException("command : 'show master status' has an error!", e);
+            throw new CanalParseException("command : '" + showSql + "' has an error!", e);
         }
     }
 
@@ -721,7 +728,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
     @SuppressWarnings("unused")
     private SlaveEntryPosition findSlavePosition(MysqlConnection mysqlConnection) {
         try {
-            ResultSetPacket packet = mysqlConnection.query("show slave status");
+            String showSql = "show slave stauts";
+            if (mysqlConnection.atLeastMySQL84()) {
+                // 兼容mysql 8.4
+                showSql = "show replica status";
+            }
+            ResultSetPacket packet = mysqlConnection.query(showSql);
             List<FieldPacket> names = packet.getFieldDescriptors();
             List<String> fields = packet.getFieldValues();
             if (CollectionUtils.isEmpty(fields)) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java

@@ -258,7 +258,7 @@ public class TableMetaCache {
             .append('`')
             .append('.')
             .append('`')
-            .append(table)
+            .append(StringUtils.replace(table,"`","``"))
             .append('`')
             .toString();
     }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java

@@ -102,7 +102,8 @@ public class DdlResult {
         DdlResult ddlResult = this;
         StringBuilder sb = new StringBuilder();
         do {
-            sb.append(String.format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];",
+            sb.append(String.format(
+                "DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];",
                 ddlResult.schemaName,
                 ddlResult.tableName,
                 ddlResult.oriSchemaName,

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java

@@ -6,38 +6,16 @@ import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
 
-import com.alibaba.druid.sql.SQLUtils;
-import com.alibaba.druid.sql.ast.SQLExpr;
-import com.alibaba.druid.sql.ast.SQLStatement;
-import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
-import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddConstraint;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddIndex;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropConstraint;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropIndex;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropKey;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableRename;
-import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
-import com.alibaba.druid.sql.ast.statement.SQLConstraint;
-import com.alibaba.druid.sql.ast.statement.SQLCreateDatabaseStatement;
-import com.alibaba.druid.sql.ast.statement.SQLCreateIndexStatement;
-import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
-import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement;
-import com.alibaba.druid.sql.ast.statement.SQLDropDatabaseStatement;
-import com.alibaba.druid.sql.ast.statement.SQLDropIndexStatement;
-import com.alibaba.druid.sql.ast.statement.SQLDropTableStatement;
-import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
-import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
-import com.alibaba.druid.sql.ast.statement.SQLTableSource;
-import com.alibaba.druid.sql.ast.statement.SQLTruncateStatement;
-import com.alibaba.druid.sql.ast.statement.SQLUnique;
-import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement;
-import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement;
-import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement.Item;
-import com.alibaba.druid.sql.parser.ParserException;
-import com.alibaba.druid.util.JdbcConstants;
 import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.polardbx.druid.sql.SQLUtils;
+import com.alibaba.polardbx.druid.sql.ast.SQLExpr;
+import com.alibaba.polardbx.druid.sql.ast.SQLStatement;
+import com.alibaba.polardbx.druid.sql.ast.expr.SQLIdentifierExpr;
+import com.alibaba.polardbx.druid.sql.ast.expr.SQLPropertyExpr;
+import com.alibaba.polardbx.druid.sql.ast.statement.*;
+import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement;
+import com.alibaba.polardbx.druid.sql.parser.ParserException;
+import com.alibaba.polardbx.druid.util.JdbcConstants;
 
 /**
  * @author agapple 2017年7月27日 下午4:05:34
@@ -141,7 +119,7 @@ public class DruidDdlParser {
                 }
             } else if (statement instanceof MySqlRenameTableStatement) {
                 MySqlRenameTableStatement rename = (MySqlRenameTableStatement) statement;
-                for (Item item : rename.getItems()) {
+                for (MySqlRenameTableStatement.Item item : rename.getItems()) {
                     DdlResult ddlResult = new DdlResult();
                     processName(ddlResult, schmeaName, item.getName(), true);
                     processName(ddlResult, schmeaName, item.getTo(), false);

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java

@@ -51,7 +51,6 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
             rdsLocalBinlogEventParser.setDestination(destination);
             rdsLocalBinlogEventParser.setAlarmHandler(this.getAlarmHandler());
             rdsLocalBinlogEventParser.setConnectionCharsetStd(this.connectionCharset);
-            rdsLocalBinlogEventParser.setConnectionCharsetNumber(this.connectionCharsetNumber);
             rdsLocalBinlogEventParser.setEnableTsdb(this.enableTsdb);
             rdsLocalBinlogEventParser.setEventBlackFilter(this.eventBlackFilter);
             rdsLocalBinlogEventParser.setFilterQueryDcl(this.filterQueryDcl);

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java

@@ -195,7 +195,7 @@ public abstract class AbstractRequest<T> {
     private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
         SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, (TrustStrategy) (arg0, arg1) -> true).build();
         SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
-            new String[] { "TLSv1" },
+            new String[] { "TLSv1.2" },
             null,
             SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
         Registry registry = RegistryBuilder.create()

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java

@@ -13,8 +13,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Pattern;
-
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
+import com.alibaba.polardbx.druid.sql.repository.Schema;
 import org.apache.commons.beanutils.BeanUtils;
 import org.apache.commons.lang.ObjectUtils;
 import org.apache.commons.lang.StringUtils;
@@ -22,10 +22,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
 
-import com.alibaba.druid.sql.repository.Schema;
 import com.alibaba.fastjson2.JSON;
 import com.alibaba.fastjson2.JSONObject;
 import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.TableMeta;
@@ -207,6 +207,11 @@ public class DatabaseTableMeta implements TableMetaTSDB {
             }
 
             for (String schema : schemas) {
+                // 如果schema命中黑名单，直接跳过
+                // 解决部分数据库可以看到database，但实际表级别无权限的情况
+                if (blackFilter != null && blackFilter.filter(schema + ".%")) {
+                    continue;
+                }
                 // filter views
                 packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
                 columnSize = packet.getFieldDescriptors().size();
@@ -228,7 +233,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
 
                 StringBuilder sql = new StringBuilder();
                 for (String table : tables) {
-                    sql.append("show create table `" + schema + "`.`" + table + "`;");
+                    sql.append("show create table `" + schema + "`.`" + StringUtils.replace(table,"`","``") + "`;");
                 }
 
                 List<ResultSetPacket> packets = connection.queryMulti(sql.toString());
@@ -247,30 +252,47 @@ public class DatabaseTableMeta implements TableMetaTSDB {
     }
 
     private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) {
-        Map<String, String> content = new HashMap<>();
+        // Map<String, String> content = new HashMap<>();
-        content.put("destination", destination);
+        // content.put("destination", destination);
-        content.put("binlogFile", position.getJournalName());
+        // content.put("binlogFile", position.getJournalName());
-        content.put("binlogOffest", String.valueOf(position.getPosition()));
+        // content.put("binlogOffest", String.valueOf(position.getPosition()));
-        content.put("binlogMasterId", String.valueOf(position.getServerId()));
+        // content.put("binlogMasterId", String.valueOf(position.getServerId()));
-        content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
+        // content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
-        content.put("useSchema", schema);
+        // content.put("useSchema", schema);
-        if (content.isEmpty()) {
+        //
-            throw new RuntimeException("apply failed caused by content is empty in applyHistoryToDB");
+        // if (content.isEmpty()) {
-        }
+        // throw new RuntimeException("apply failed caused by content is empty in
-        // 待补充
+        // applyHistoryToDB");
+        // }
+        // // 待补充
+        // List<DdlResult> ddlResults = DruidDdlParser.parse(ddl, schema);
+        // if (ddlResults.size() > 0) {
+        // DdlResult ddlResult = ddlResults.get(0);
+        // content.put("sqlSchema", ddlResult.getSchemaName());
+        // content.put("sqlTable", ddlResult.getTableName());
+        // content.put("sqlType", ddlResult.getType().name());
+        // content.put("sqlText", ddl);
+        // content.put("extra", extra);
+        // }
+        // BeanUtils.populate(metaDO, content);
+
+        MetaHistoryDO metaDO = new MetaHistoryDO();
+        metaDO.setDestination(destination);
+        metaDO.setBinlogFile(position.getJournalName());
+        metaDO.setBinlogOffest(position.getPosition());
+        metaDO.setBinlogMasterId(String.valueOf(position.getServerId()));
+        metaDO.setBinlogTimestamp(position.getTimestamp());
+        metaDO.setUseSchema(schema);
         List<DdlResult> ddlResults = DruidDdlParser.parse(ddl, schema);
         if (ddlResults.size() > 0) {
             DdlResult ddlResult = ddlResults.get(0);
-            content.put("sqlSchema", ddlResult.getSchemaName());
+            metaDO.setSqlSchema(ddlResult.getSchemaName());
-            content.put("sqlTable", ddlResult.getTableName());
+            metaDO.setSqlTable(ddlResult.getTableName());
-            content.put("sqlType", ddlResult.getType().name());
+            metaDO.setSqlType(ddlResult.getType().name());
-            content.put("sqlText", ddl);
+            metaDO.setSqlText(ddl);
-            content.put("extra", extra);
+            metaDO.setExtra(extra);
         }
-
-        MetaHistoryDO metaDO = new MetaHistoryDO();
         try {
-            BeanUtils.populate(metaDO, content);
             // 会建立唯一约束,解决:
             // 1. 重复的binlog file+offest
             // 2. 重复的masterId+timestamp
@@ -328,20 +350,27 @@ public class DatabaseTableMeta implements TableMetaTSDB {
         }
 
         if (compareAll) {
-            Map<String, String> content = new HashMap<>();
+            // Map<String, String> content = new HashMap<>();
-            content.put("destination", destination);
+            // content.put("destination", destination);
-            content.put("binlogFile", position.getJournalName());
+            // content.put("binlogFile", position.getJournalName());
-            content.put("binlogOffest", String.valueOf(position.getPosition()));
+            // content.put("binlogOffest", String.valueOf(position.getPosition()));
-            content.put("binlogMasterId", String.valueOf(position.getServerId()));
+            // content.put("binlogMasterId", String.valueOf(position.getServerId()));
-            content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
+            // content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
-            content.put("data", JSON.toJSONString(schemaDdls));
+            // content.put("data", JSON.toJSONString(schemaDdls));
-            if (content.isEmpty()) {
+            // if (content.isEmpty()) {
-                throw new RuntimeException("apply failed caused by content is empty in applySnapshotToDB");
+            // throw new RuntimeException("apply failed caused by content is empty in
-            }
+            // applySnapshotToDB");
+            // }
+            // BeanUtils.populate(snapshotDO, content);
 
             MetaSnapshotDO snapshotDO = new MetaSnapshotDO();
+            snapshotDO.setDestination(destination);
+            snapshotDO.setBinlogFile(position.getJournalName());
+            snapshotDO.setBinlogOffest(position.getPosition());
+            snapshotDO.setBinlogMasterId(String.valueOf(position.getServerId()));
+            snapshotDO.setBinlogTimestamp(position.getTimestamp());
+            snapshotDO.setData(JSON.toJSONString(schemaDdls));
             try {
-                BeanUtils.populate(snapshotDO, content);
                 metaSnapshotDAO.insert(snapshotDO);
             } catch (Throwable e) {
                 if (isUkDuplicateException(e)) {
@@ -485,7 +514,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
 
     private String getFullName(String schema, String table) {
         StringBuilder builder = new StringBuilder();
-        return builder.append(structureSchema(schema)).append('.').append('`').append(table).append('`').toString();
+        return builder.append(structureSchema(schema)).append('.').append('`').append(StringUtils.replace(table,"`","``")).append('`').toString();
     }
 
     public static boolean compareTableMeta(TableMeta source, TableMeta target) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java

@@ -1,48 +1,33 @@
 package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
 
-import java.util.Arrays;
+import java.util.*;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.alibaba.druid.DbType;
-import com.alibaba.druid.sql.SQLUtils;
-import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
-import com.alibaba.druid.sql.visitor.VisitorFeature;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.druid.sql.ast.SQLDataType;
-import com.alibaba.druid.sql.ast.SQLDataTypeImpl;
-import com.alibaba.druid.sql.ast.SQLExpr;
-import com.alibaba.druid.sql.ast.SQLStatement;
-import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
-import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
-import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
-import com.alibaba.druid.sql.ast.expr.SQLNullExpr;
-import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
-import com.alibaba.druid.sql.ast.statement.SQLColumnConstraint;
-import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
-import com.alibaba.druid.sql.ast.statement.SQLColumnPrimaryKey;
-import com.alibaba.druid.sql.ast.statement.SQLColumnUniqueKey;
-import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement;
-import com.alibaba.druid.sql.ast.statement.SQLNotNullConstraint;
-import com.alibaba.druid.sql.ast.statement.SQLNullConstraint;
-import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
-import com.alibaba.druid.sql.ast.statement.SQLTableElement;
-import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
-import com.alibaba.druid.sql.dialect.mysql.ast.MySqlUnique;
-import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
-import com.alibaba.druid.sql.repository.Schema;
-import com.alibaba.druid.sql.repository.SchemaObject;
-import com.alibaba.druid.sql.repository.SchemaRepository;
-import com.alibaba.druid.util.JdbcConstants;
 import com.alibaba.otter.canal.parse.inbound.TableMeta;
 import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
 import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser;
 import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.polardbx.druid.DbType;
+import com.alibaba.polardbx.druid.sql.SQLUtils;
+import com.alibaba.polardbx.druid.sql.ast.SQLDataType;
+import com.alibaba.polardbx.druid.sql.ast.SQLDataTypeImpl;
+import com.alibaba.polardbx.druid.sql.ast.SQLExpr;
+import com.alibaba.polardbx.druid.sql.ast.SQLStatement;
+import com.alibaba.polardbx.druid.sql.ast.expr.*;
+import com.alibaba.polardbx.druid.sql.ast.statement.*;
+import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.MySqlPrimaryKey;
+import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.MySqlUnique;
+import com.alibaba.polardbx.druid.sql.dialect.mysql.ast.expr.MySqlOrderingExpr;
+import com.alibaba.polardbx.druid.sql.repository.Schema;
+import com.alibaba.polardbx.druid.sql.repository.SchemaObject;
+import com.alibaba.polardbx.druid.sql.repository.SchemaRepository;
+import com.alibaba.polardbx.druid.sql.visitor.SQLASTOutputVisitor;
+import com.alibaba.polardbx.druid.sql.visitor.VisitorFeature;
+import com.alibaba.polardbx.druid.util.JdbcConstants;
 
 /**
  * 基于DDL维护的内存表结构
@@ -54,9 +39,10 @@ public class MemoryTableMeta implements TableMetaTSDB {
 
     private Logger                       logger     = LoggerFactory.getLogger(MemoryTableMeta.class);
     private Map<List<String>, TableMeta> tableMetas = new ConcurrentHashMap<>();
-    private SchemaRepository             repository = new SchemaRepository(JdbcConstants.MYSQL);
+    private SchemaRepository             repository;
 
     public MemoryTableMeta(){
+        repository = new SchemaRepository(JdbcConstants.MYSQL);
     }
 
     @Override
@@ -271,13 +257,52 @@ public class MemoryTableMeta implements TableMetaTSDB {
         } else if (element instanceof MySqlUnique) {
             MySqlUnique column = (MySqlUnique) element;
             List<SQLSelectOrderByItem> uks = column.getColumns();
+            // https://github.com/alibaba/canal/issues/5094
+            // 处理一下函数索引
+            List<String> columnNames = new ArrayList<String>();
             for (SQLSelectOrderByItem uk : uks) {
-                String name = getSqlName(uk.getExpr());
+                SQLExpr sqlName = uk.getExpr();
-                FieldMeta field = tableMeta.getFieldMetaByName(name);
+                columnNames.addAll(getIndexColumnNames(sqlName));
+            }
+            // uniqe打标
+            for (String name : columnNames) {
+                FieldMeta field = tableMeta.tryGetFieldMetaByName(name);
+                if (field != null) {
                     field.setUnique(true);
                 }
             }
         }
+    }
+
+    private List<String> getIndexColumnNames(SQLExpr expr) {
+        if (expr instanceof SQLMethodInvokeExpr) {
+            // 需要递归处理下函数索引, 尽可能收集一下列名
+            // 常见的case:
+            // 1. upper(col)
+            // 2. left(upper(col) , 10)
+            // 3. col(10)
+            List<SQLExpr> indexExpres = ((SQLMethodInvokeExpr) expr).getArguments();
+            List<String> columnNames = new ArrayList<String>();
+            // 加上当前列
+            columnNames.add(getSqlName(expr));
+            // 处理函数索引列
+            for (SQLExpr exArgs : indexExpres) {
+                if (exArgs instanceof SQLMethodInvokeExpr) {
+                    // 加上当前列
+                    columnNames.add(getSqlName(exArgs));
+                    columnNames.addAll(getIndexColumnNames(exArgs));
+                } else {
+                    String columnName = getSqlName(exArgs);
+                    columnNames.add(columnName);
+                }
+            }
+
+            return columnNames;
+        } else {
+            String columnName = getSqlName(expr);
+            return Arrays.asList(columnName);
+        }
+    }
 
     private String getSqlName(SQLExpr sqlName) {
         if (sqlName == null) {

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java

@@ -30,6 +30,9 @@ public class TableMetaTSDBBuilder {
             if (applicationContext == null) {
                 synchronized (contexts) {
                     if (applicationContext == null) {
+                        if (!StringUtils.startsWithIgnoreCase(springXml, "classpath:")) {
+                            springXml = "classpath:" + springXml;
+                        }
                         applicationContext = new ClassPathXmlApplicationContext(springXml);
                         contexts.put(destination, applicationContext);
                     }

parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java

@@ -2,10 +2,12 @@ package com.alibaba.otter.canal.parse.support;
 
 import java.net.InetSocketAddress;
 
-import com.alibaba.druid.filter.config.ConfigTools;
 import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.commons.lang.builder.ToStringStyle;
 
+import com.alibaba.otter.canal.common.utils.CommonUtils;
+import com.alibaba.otter.canal.parse.driver.mysql.ssl.SslInfo;
+
 /**
  * 数据库认证信息
  *
@@ -14,18 +16,17 @@ import org.apache.commons.lang.builder.ToStringStyle;
  */
 public class AuthenticationInfo {
 
-
-
     private InetSocketAddress address;            // 主库信息
     private String            username;           // 帐号
     private String            password;           // 密码
     private String            defaultDatabaseName;// 默认链接的数据库
-    private String            pwdPublicKey;       //公钥
+    private String            pwdPublicKey;       // 公钥
-    private boolean           enableDruid;        //是否使用druid加密解密数据库密码
+    private boolean           enableDruid;        // 是否使用druid加密解密数据库密码
+    private SslInfo           sslInfo;
 
-    public void initPwd() throws Exception{
+    public void initPwd() throws Exception {
         if (enableDruid) {
-            this.password = ConfigTools.decrypt(pwdPublicKey, password);
+            this.password = CommonUtils.decryptDruidPassword(pwdPublicKey, password);
         }
     }
 
@@ -37,13 +38,19 @@ public class AuthenticationInfo {
         this(address, username, password, "");
     }
 
-    public AuthenticationInfo(InetSocketAddress address, String username, String password, String defaultDatabaseName){
+    private AuthenticationInfo(InetSocketAddress address, String username, String password, String defaultDatabaseName){
         this.address = address;
         this.username = username;
         this.password = password;
         this.defaultDatabaseName = defaultDatabaseName;
     }
 
+    public AuthenticationInfo(InetSocketAddress address, String username, String password, String defaultDatabaseName,
+                              SslInfo sslInfo){
+        this(address, username, password, defaultDatabaseName);
+        this.sslInfo = sslInfo;
+    }
+
     public InetSocketAddress getAddress() {
         return address;
     }
@@ -92,6 +99,14 @@ public class AuthenticationInfo {
         this.enableDruid = enableDruid;
     }
 
+    public SslInfo getSslInfo() {
+        return sslInfo;
+    }
+
+    public void setSslInfo(SslInfo sslInfo) {
+        this.sslInfo = sslInfo;
+    }
+
     @Override
     public String toString() {
         return ToStringBuilder.reflectionToString(this, ToStringStyle.DEFAULT_STYLE);

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java

@@ -3,14 +3,14 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb;
 import org.junit.Assert;
 import org.junit.Test;
 
-import com.alibaba.druid.DbType;
+import com.alibaba.polardbx.druid.DbType;
-import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.polardbx.druid.sql.SQLUtils;
-import com.alibaba.druid.sql.repository.Schema;
+import com.alibaba.polardbx.druid.sql.repository.Schema;
-import com.alibaba.druid.sql.repository.SchemaObject;
+import com.alibaba.polardbx.druid.sql.repository.SchemaObject;
-import com.alibaba.druid.sql.repository.SchemaRepository;
+import com.alibaba.polardbx.druid.sql.repository.SchemaRepository;
-import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
+import com.alibaba.polardbx.druid.sql.visitor.SQLASTOutputVisitor;
-import com.alibaba.druid.sql.visitor.VisitorFeature;
+import com.alibaba.polardbx.druid.sql.visitor.VisitorFeature;
-import com.alibaba.druid.util.JdbcConstants;
+import com.alibaba.polardbx.druid.util.JdbcConstants;
 
 public class FastsqlSchemaTest {
 
@@ -265,4 +265,79 @@ public class FastsqlSchemaTest {
         System.out.println(data.toString());
     }
 
+
+    @Test
+    public void test_escapse_sql2() {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        repository.setDefaultSchema("test");
+        String sql = "CREATE TABLE test1 (\n" + "id int(11) NOT NULL AUTO_INCREMENT,\n"
+                     + "uid int(11) NOT NULL DEFAULT '0' COMMENT '用户ID',\n"
+                     + "`from` tinyint(3) NOT NULL DEFAULT '0' COMMENT '来源 1QQ 2微信 3微博',\n"
+                     + "access_code varchar(20) NOT NULL DEFAULT '' COMMENT '临时code',\n"
+                     + "access_token varchar(200) NOT NULL DEFAULT '' COMMENT '授权token',\n"
+                     + "expires_in int(11) NOT NULL DEFAULT '0' COMMENT '过期时间',\n"
+                     + "refresh_token varchar(200) NOT NULL DEFAULT '' COMMENT '刷新token',\n"
+                     + "openid varchar(255) NOT NULL DEFAULT '' COMMENT '授权用户唯一标识',\n"
+                     + "scope varchar(60) NOT NULL COMMENT '授权范围',\n"
+                     + "status tinyint(3) NOT NULL DEFAULT '1' COMMENT '1 ',\n"
+                     + "json_info text NOT NULL COMMENT '个人信息',\n"
+                     + "create_time timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' COMMENT '创建时间',\n"
+                     + "unionid varchar(128) NOT NULL DEFAULT '' COMMENT 'UnionID',\n"
+                     + "session_key varchar(128) NOT NULL DEFAULT '' COMMENT '小程序session_key',\n"
+                     + "PRIMARY KEY (id),\n" + "KEY uid (uid),\n"
+                     + "KEY idx_openid_from_status (openid, `from`, status),\n" + "KEY idx_access_token (access_token),\n"
+                     + "KEY idx_uid_from_status (uid, `from`, status),\n"
+                     + "KEY idx_unionid_from_status (unionid, `from`, status)\n"
+                     + ") ENGINE = InnoDB AUTO_INCREMENT = 7831300 CHARSET = utf8 COMMENT '用户第三方登录表'";
+        repository.console(sql);
+        SchemaObject table = repository.findTable("test1");
+        Assert.assertTrue(table != null);
+
+        // 应用到新的schema
+        Schema schema = repository.findSchema("test");
+        StringBuilder data = new StringBuilder(4 * 1024);
+        for (String tableIn : schema.showTables()) {
+            SchemaObject schemaObject = schema.findTable(tableIn);
+
+            SQLASTOutputVisitor visitor = SQLUtils.createOutputVisitor(data, DbType.mysql);
+            visitor.config(VisitorFeature.OutputNameQuote, true);
+
+            schemaObject.getStatement().accept(visitor);
+            data.append("; \n");
+        }
+
+        repository.setDefaultSchema("test_new");
+        repository.console(data.toString());
+        table = repository.findTable("test1");
+        Assert.assertTrue(table != null);
+
+        // 打印新的schema的内容
+        schema = repository.findSchema("test_new");
+        data = new StringBuilder(4 * 1024);
+        for (String tableIn : schema.showTables()) {
+            SchemaObject schemaObject = schema.findTable(tableIn);
+
+            SQLASTOutputVisitor visitor = SQLUtils.createOutputVisitor(data, DbType.mysql);
+            visitor.config(VisitorFeature.OutputNameQuote, true);
+
+            schemaObject.getStatement().accept(visitor);
+            data.append("; \n");
+        }
+
+        System.out.println(data.toString());
+    }
+
+
+    @Test
+    public void test_function_index () throws Throwable {
+        SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
+        repository.setDefaultSchema("test");
+        String sql = "CREATE TABLE test1 (\n" + "    id INT AUTO_INCREMENT PRIMARY KEY,\n"
+                     + "    owner_id INT NOT NULL,\n" + "    code VARCHAR(100) NOT NULL,\n"
+                     + "    UNIQUE KEY uk_owner_id_upper_code (owner_id, (upper(code)))\n" + ");";
+        repository.console(sql);
+        SchemaObject table = repository.findTable("test1");
+        Assert.assertTrue(table != null);
+    }
+
 }

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_DDL_Test.java

@@ -14,7 +14,7 @@ import org.junit.runner.RunWith;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
-import com.alibaba.druid.sql.repository.Schema;
+import com.alibaba.polardbx.druid.sql.repository.Schema;
 import com.alibaba.otter.canal.parse.inbound.TableMeta;
 
 /**
@@ -109,4 +109,26 @@ public class MemoryTableMeta_DDL_Test {
             System.out.println(sourceMeta.toString());
         }
     }
+
+    @Test
+    public void test_function_index () throws Throwable {
+        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
+        URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
+        File dummyFile = new File(url.getFile());
+        File create = new File(dummyFile.getParent() + "/ddl", "ddl_create_function_index.sql");
+        String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
+        memoryTableMeta.apply(null, "test", sql, null);
+
+        List<String> tableNames = new ArrayList<>();
+        for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
+            tableNames.addAll(schema.showTables());
+        }
+
+        for (String table : tableNames) {
+            TableMeta sourceMeta = memoryTableMeta.find("test", table);
+            TableMeta.FieldMeta field = sourceMeta.getFieldMetaByName("code");
+            System.out.println(sourceMeta.toString());
+            Assert.assertTrue(field.isUnique());
+        }
+    }
 }

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta_Random_DDL_Test.java

@@ -14,7 +14,7 @@ import org.junit.runner.RunWith;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
-import com.alibaba.druid.sql.repository.Schema;
+import com.alibaba.polardbx.druid.sql.repository.Schema;
 import com.alibaba.otter.canal.parse.inbound.TableMeta;
 
 /**

parse/src/test/resources/ddl/ddl_create_function_index.sql

@@ -0,0 +1 @@
+CREATE TABLE function_index_test (

pom.xml

@@ -99,13 +99,13 @@
         <java_target_version>1.8</java_target_version>
         <file_encoding>UTF-8</file_encoding>
         <javadoc_skip>true</javadoc_skip>
-        <spring_version>5.3.9</spring_version>
+        <spring_version>5.3.26</spring_version>
         <log4j_version>2.17.0</log4j_version>
-        <rocketmq_version>4.8.0</rocketmq_version>
+        <rocketmq_version>4.9.8</rocketmq_version>
-        <rabbitmq_version>5.5.0</rabbitmq_version>
+        <rabbitmq_version>5.18.0</rabbitmq_version>
         <mq_amqp_client>1.0.3</mq_amqp_client>
         <kafka_version>2.4.0</kafka_version>
-        <pulsar_version>2.8.1</pulsar_version>
+        <pulsar_version>2.11.4</pulsar_version>
         <mysql_driver_version>5.1.48</mysql_driver_version>
         <maven-jacoco-plugin.version>0.8.3</maven-jacoco-plugin.version>
         <maven-surefire.version>2.22.1</maven-surefire.version>
@@ -389,6 +389,11 @@
                 <artifactId>log4j-to-slf4j</artifactId>
                 <version>${log4j_version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.alibaba.polardbx</groupId>
+                <artifactId>polardbx-parser</artifactId>
+                <version>5.4.19</version>
+            </dependency>
 
             <!-- test dependency -->
             <dependency>

server/src/test/java/com/alibaba/otter/canal/server/CanalServerTest.java

@@ -253,7 +253,6 @@ public class CanalServerTest {
 
         parameter.setDefaultConnectionTimeoutInSeconds(30);
         parameter.setConnectionCharset("UTF-8");
-        parameter.setConnectionCharsetNumber((byte) 33);
         parameter.setReceiveBufferSize(8 * 1024);
         parameter.setSendBufferSize(8 * 1024);
 

server/src/test/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded_FileModeTest.java

@@ -40,7 +40,6 @@ public class CanalServerWithEmbedded_FileModeTest extends BaseCanalServerWithEmb
 
         parameter.setDefaultConnectionTimeoutInSeconds(30);
         parameter.setConnectionCharset("UTF-8");
-        parameter.setConnectionCharsetNumber((byte) 33);
         parameter.setReceiveBufferSize(8 * 1024);
         parameter.setSendBufferSize(8 * 1024);
 

server/src/test/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded_StandaloneTest.java

@@ -43,7 +43,6 @@ public class CanalServerWithEmbedded_StandaloneTest extends BaseCanalServerWithE
 
         parameter.setDefaultConnectionTimeoutInSeconds(30);
         parameter.setConnectionCharset("UTF-8");
-        parameter.setConnectionCharsetNumber((byte) 33);
         parameter.setReceiveBufferSize(8 * 1024);
         parameter.setSendBufferSize(8 * 1024);
 

server/src/test/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded_StandbyTest.java

@@ -55,7 +55,6 @@ public class CanalServerWithEmbedded_StandbyTest extends BaseCanalServerWithEmbe
 
         parameter.setDefaultConnectionTimeoutInSeconds(30);
         parameter.setConnectionCharset("UTF-8");
-        parameter.setConnectionCharsetNumber((byte) 33);
         parameter.setReceiveBufferSize(8 * 1024);
         parameter.setSendBufferSize(8 * 1024);