optimize gtid

恩baseProperties这个变量是共享的，不能修改这个变量的属性值。要不后续的配置都会改变。关联issue为4355

README.md

@@ -4,6 +4,8 @@
 ![license](https://img.shields.io/github/license/alibaba/canal.svg)
 [![average time to resolve an issue](http://isitmaintained.com/badge/resolution/alibaba/canal.svg)](http://isitmaintained.com/project/alibaba/canal "average time to resolve an issue")
 [![percentage of issues still open](http://isitmaintained.com/badge/open/alibaba/canal.svg)](http://isitmaintained.com/project/alibaba/canal "percentage of issues still open")
+[![Leaderboard](https://img.shields.io/badge/Canal-%E6%9F%A5%E7%9C%8B%E8%B4%A1%E7%8C%AE%E6%8E%92%E8%A1%8C%E6%A6%9C-orange)](https://opensource.alibaba.com/contribution_leaderboard/details?projectValue=canal)
+
 
 ## 简介
 
@@ -101,3 +103,5 @@ canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递
 
 ## 问题反馈
 - 报告 issue: [github issues](https://github.com/alibaba/canal/issues)
+
+本项目的Issues会被同步沉淀至[阿里云开发者社区](https://developer.aliyun.com/ask)

admin/admin-ui/src/views/login/index.vue

@@ -66,8 +66,8 @@ export default {
       }
     }
     const validatePassword = (rule, value, callback) => {
-      if (value.length < 6) {
-        callback(new Error('The password can not be less than 6 digits'))
+      if (value.length < 5) {
+        callback(new Error('The password can not be less than 5 digits'))
       } else {
         callback()
       }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java

@@ -38,12 +38,12 @@ public class SimpleAdminConnectors {
     }
 
     private static String getDesensitizationPassword(String password) {
-        String defaultPassword = "******";
+        String defaultPassword = "*****";
 
-        if (Strings.isNullOrEmpty(password) || password.length() < 6) {
+        if (Strings.isNullOrEmpty(password) || password.length() < 5) {
             return defaultPassword;
         } else {
-            return String.format("%s******", password.substring(0, 3));
+            return String.format("%s*****", password.substring(0, 3));
         }
     }
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/UserServiceImpl.java

@@ -21,7 +21,7 @@ public class UserServiceImpl implements UserService {
 
     private static byte[] seeds = "canal is best!".getBytes();
 
-    private static final Integer PASSWORD_LENGTH = 6;
+    private static final Integer PASSWORD_LENGTH = 5;
 
     public User find4Login(String username, String password) {
         if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) {

admin/admin-web/src/main/resources/canal-template.properties

@@ -25,7 +25,7 @@ canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, rocketMQ, rabbitMQ
+# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -96,7 +96,7 @@ canal.instance.tsdb.snapshot.expire = 360
 #################################################
 ######### 		destinations		#############
 #################################################
-canal.destinations = 
+canal.destinations =
 # conf root dir
 canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
@@ -148,8 +148,13 @@ kafka.max.in.flight.requests.per.connection = 1
 kafka.retries = 0
 
 kafka.kerberos.enable = false
-kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
-kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
+kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
+kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf
+
+# sasl demo
+# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
+# kafka.sasl.mechanism = SCRAM-SHA-512
+# kafka.security.protocol = SASL_PLAINTEXT
 
 ##################################################
 ######### 		    RocketMQ	     #############
@@ -161,7 +166,7 @@ rocketmq.namespace =
 rocketmq.namesrv.addr = 127.0.0.1:9876
 rocketmq.retry.times.when.send.failed = 0
 rocketmq.vip.channel.enabled = false
-rocketmq.tag = 
+rocketmq.tag =
 
 ##################################################
 ######### 		    RabbitMQ	     #############
@@ -171,4 +176,12 @@ rabbitmq.virtual.host =
 rabbitmq.exchange =
 rabbitmq.username =
 rabbitmq.password =
-rabbitmq.deliveryMode =
+rabbitmq.deliveryMode =
+
+
+##################################################
+######### 		      Pulsar         #############
+##################################################
+pulsarmq.serverUrl =
+pulsarmq.roleToken =
+pulsarmq.topicTenantPrefix =

admin/pom.xml

@@ -1,5 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>com.alibaba.otter</groupId>
+        <artifactId>canal</artifactId>
+        <version>1.1.7-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.admin</artifactId>
@@ -18,7 +24,6 @@
         <java_source_version>1.8</java_source_version>
         <java_target_version>1.8</java_target_version>
         <file_encoding>UTF-8</file_encoding>
-        <log4j_version>2.17.0</log4j_version>
     </properties>
 
     <dependencyManagement>
@@ -26,87 +31,25 @@
             <dependency>
                 <groupId>org.springframework.cloud</groupId>
                 <artifactId>spring-cloud-context</artifactId>
-                <version>2.0.0.RELEASE</version>
+                <version>3.0.6</version>
             </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-core</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-aop</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-context</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-jdbc</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-orm</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-test</artifactId>
-                <version>5.0.5.RELEASE</version>
-                <scope>test</scope>
-            </dependency>
-
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>
-                <version>2.0.1.RELEASE</version>
+                <version>2.5.4</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
-            <dependency>
-                <groupId>mysql</groupId>
-                <artifactId>mysql-connector-java</artifactId>
-                <version>5.1.48</version>
-            </dependency>
             <dependency>
                 <groupId>com.github.ben-manes.caffeine</groupId>
                 <artifactId>caffeine</artifactId>
                 <version>2.6.2</version>
             </dependency>
-            <dependency>
-                <groupId>commons-beanutils</groupId>
-                <artifactId>commons-beanutils</artifactId>
-                <version>1.8.2</version>
-            </dependency>
-            <dependency>
-                <groupId>commons-lang</groupId>
-                <artifactId>commons-lang</artifactId>
-                <version>2.6</version>
-            </dependency>
             <dependency>
                 <groupId>io.ebean</groupId>
                 <artifactId>ebean</artifactId>
                 <version>11.41.1</version>
             </dependency>
-            <dependency>
-                <groupId>org.apache.logging.log4j</groupId>
-                <artifactId>log4j-core</artifactId>
-                <version>${log4j_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.logging.log4j</groupId>
-                <artifactId>log4j-api</artifactId>
-                <version>${log4j_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.logging.log4j</groupId>
-                <artifactId>log4j-to-slf4j</artifactId>
-                <version>${log4j_version}</version>
-            </dependency>
         </dependencies>
     </dependencyManagement>
 

client-adapter/common/pom.xml

@@ -60,5 +60,4 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
-
 </project>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/ProxyOuterAdapter.java

@@ -0,0 +1,87 @@
+package com.alibaba.otter.canal.client.adapter;
+
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class ProxyOuterAdapter implements OuterAdapter {
+
+    private OuterAdapter outerAdapter;
+
+    public ProxyOuterAdapter(OuterAdapter outerAdapter) {
+        this.outerAdapter = outerAdapter;
+    }
+
+    private ClassLoader changeCL() {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(outerAdapter.getClass().getClassLoader());
+        return cl;
+    }
+
+    private void revertCL(ClassLoader cl) {
+        Thread.currentThread().setContextClassLoader(cl);
+    }
+
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        ClassLoader cl = changeCL();
+        try {
+            outerAdapter.init(configuration, envProperties);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void sync(List<Dml> dmls) {
+        ClassLoader cl = changeCL();
+        try {
+            outerAdapter.sync(dmls);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void destroy() {
+        ClassLoader cl = changeCL();
+        try {
+            outerAdapter.destroy();
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        ClassLoader cl = changeCL();
+        try {
+            return OuterAdapter.super.etl(task, params);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public Map<String, Object> count(String task) {
+        ClassLoader cl = changeCL();
+        try {
+            return OuterAdapter.super.count(task);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public String getDestination(String task) {
+        ClassLoader cl = changeCL();
+        try {
+            return OuterAdapter.super.getDestination(task);
+        } finally {
+            revertCL(cl);
+        }
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java

@@ -73,6 +73,9 @@ public class YamlUtils {
             PropertySource<?> propertySource = propertySources.get(0);
 
             Properties properties = new Properties();
+            if (baseProperties != null) {
+                properties.putAll(baseProperties);
+            }
 
             properties.putAll((Map<?, ?>) propertySource.getSource());
 
@@ -82,12 +85,7 @@ public class YamlUtils {
                     entry.setValue(((OriginTrackedValue) value).getValue());
                 }
             }
-
-            if (baseProperties != null) {
-                baseProperties.putAll(properties);
-                properties = baseProperties;
-            }
-
+            
             ConfigurationPropertySource sources = new MapConfigurationPropertySource(properties);
             Binder binder = new Binder(sources);
             return binder.bind(prefix, Bindable.of(clazz)).get();

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/Common.java

@@ -5,6 +5,7 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 
 import javax.sql.DataSource;
 
@@ -26,7 +27,7 @@ public class Common {
         outerAdapterConfig.setProperties(properties);
 
         ES6xAdapter esAdapter = new ES6xAdapter();
-        esAdapter.init(outerAdapterConfig, null);
+        esAdapter.init(outerAdapterConfig, new Properties());
         return esAdapter;
     }
 

client-adapter/es8x/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.7-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.es8x</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter es v8x module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.escore</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>co.elastic.clients</groupId>
+            <artifactId>elasticsearch-java</artifactId>
+            <version>8.6.2</version>
+        </dependency>
+        <!--<dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-client</artifactId>
+            <version>8.6.2</version>
+        </dependency>-->
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <version>7.17.9</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/classes/es8" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/es8" erroronmissingdir="true">
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/ES8xAdapter.java

@@ -0,0 +1,117 @@
+package com.alibaba.otter.canal.client.adapter.es8x;
+
+import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es8x.etl.ESEtlService;
+import com.alibaba.otter.canal.client.adapter.es8x.support.ES8xTemplate;
+import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import org.elasticsearch.action.search.SearchResponse;
+
+import javax.sql.DataSource;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * ES 8.x 外部适配器
+ *
+ * @author ymz 2013-02-23
+ * @version 1.0.0
+ */
+@SPI("es8")
+public class ES8xAdapter extends ESAdapter {
+
+    private ESConnection esConnection;
+
+    public ESConnection getEsConnection() {
+        return esConnection;
+    }
+
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        try {
+            Map<String, String> properties = configuration.getProperties();
+
+            String[] hostArray = configuration.getHosts().split(",");
+            esConnection = new ESConnection(hostArray, properties);
+
+            this.esTemplate = new ES8xTemplate(esConnection);
+
+            envProperties.put("es.version", "es8");
+            super.init(configuration, envProperties);
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public Map<String, Object> count(String task) {
+        ESSyncConfig config = esSyncConfig.get(task);
+        ESSyncConfig.ESMapping mapping = config.getEsMapping();
+        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index()).size(0).getResponse();
+
+        long rowCount = response.getHits().getTotalHits().value;
+        Map<String, Object> res = new LinkedHashMap<>();
+        res.put("esIndex", mapping.get_index());
+        res.put("count", rowCount);
+        return res;
+    }
+
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        ESSyncConfig config = esSyncConfig.get(task);
+        if (config != null) {
+            DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+            ESEtlService esEtlService = new ESEtlService(esConnection, config);
+            if (dataSource != null) {
+                return esEtlService.importData(params);
+            } else {
+                etlResult.setSucceeded(false);
+                etlResult.setErrorMessage("DataSource not found");
+                return etlResult;
+            }
+        } else {
+            StringBuilder resultMsg = new StringBuilder();
+            boolean resSuccess = true;
+            for (ESSyncConfig configTmp : esSyncConfig.values()) {
+                // 取所有的destination为task的配置
+                if (configTmp.getDestination().equals(task)) {
+                    ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
+                    EtlResult etlRes = esEtlService.importData(params);
+                    if (!etlRes.getSucceeded()) {
+                        resSuccess = false;
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
+                    } else {
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
+                    }
+                }
+            }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSuccess);
+                if (resSuccess) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
+        }
+        etlResult.setSucceeded(false);
+        etlResult.setErrorMessage("Task not found");
+        return etlResult;
+    }
+
+    @Override
+    public void destroy() {
+        super.destroy();
+        if (esConnection != null) {
+            esConnection.close();
+        }
+    }
+}

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/etl/ESEtlService.java

@@ -0,0 +1,202 @@
+package com.alibaba.otter.canal.client.adapter.es8x.etl;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+
+import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
+import com.alibaba.otter.canal.client.adapter.es8x.support.ES8xTemplate;
+import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection;
+import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
+import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+
+/**
+ * ES ETL Service
+ *
+ * @author rewerma 2018-11-01
+ * @version 1.0.0
+ */
+public class ESEtlService extends AbstractEtlService {
+
+    private ESConnection esConnection;
+    private ESTemplate   esTemplate;
+    private ESSyncConfig config;
+
+    public ESEtlService(ESConnection esConnection, ESSyncConfig config){
+        super("ES", config);
+        this.esConnection = esConnection;
+        this.esTemplate = new ES8xTemplate(esConnection);
+        this.config = config;
+    }
+
+    public EtlResult importData(List<String> params) {
+        ESMapping mapping = config.getEsMapping();
+        logger.info("start etl to import data to index: {}", mapping.get_index());
+        String sql = mapping.getSql();
+        return importData(sql, params);
+    }
+
+    protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
+                                       AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount,
+                                       List<String> errMsg) {
+        try {
+            ESMapping mapping = (ESMapping) adapterMapping;
+            Util.sqlRS(ds, sql, values, rs -> {
+                int count = 0;
+                try {
+                    ESBulkRequest esBulkRequest = this.esConnection.new ES8xBulkRequest();
+
+                    long batchBegin = System.currentTimeMillis();
+                    while (rs.next()) {
+                        Map<String, Object> esFieldData = new LinkedHashMap<>();
+                        Object idVal = null;
+                        for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
+
+                            String fieldName = fieldItem.getFieldName();
+                            if (mapping.getSkips().contains(fieldName)) {
+                                continue;
+                            }
+
+                            // 如果是主键字段则不插入
+                            if (fieldItem.getFieldName().equals(mapping.get_id())) {
+                                idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                            } else {
+                                Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                                esFieldData.put(Util.cleanColumn(fieldName), val);
+                            }
+
+                        }
+
+                        if (!mapping.getRelations().isEmpty()) {
+                            mapping.getRelations().forEach((relationField, relationMapping) -> {
+                                Map<String, Object> relations = new HashMap<>();
+                                relations.put("name", relationMapping.getName());
+                                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
+                                    FieldItem parentFieldItem = mapping.getSchemaItem()
+                                        .getSelectFields()
+                                        .get(relationMapping.getParent());
+                                    Object parentVal;
+                                    try {
+                                        parentVal = esTemplate.getValFromRS(mapping,
+                                            rs,
+                                            parentFieldItem.getFieldName(),
+                                            parentFieldItem.getFieldName());
+                                    } catch (SQLException e) {
+                                        throw new RuntimeException(e);
+                                    }
+                                    if (parentVal != null) {
+                                        relations.put("parent", parentVal.toString());
+                                        esFieldData.put("$parent_routing", parentVal.toString());
+
+                                    }
+                                }
+                                esFieldData.put(Util.cleanColumn(relationField), relations);
+                            });
+                        }
+
+                        if (idVal != null) {
+                            String parentVal = (String) esFieldData.remove("$parent_routing");
+                            if (mapping.isUpsert()) {
+                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(
+                                    mapping.get_index(),
+                                    idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+
+                                if (StringUtils.isNotEmpty(parentVal)) {
+                                    esUpdateRequest.setRouting(parentVal);
+                                }
+
+                                esBulkRequest.add(esUpdateRequest);
+                            } else {
+                                ESIndexRequest esIndexRequest = this.esConnection.new ES8xIndexRequest(
+                                    mapping.get_index(),
+                                    idVal.toString()).setSource(esFieldData);
+                                if (StringUtils.isNotEmpty(parentVal)) {
+                                    esIndexRequest.setRouting(parentVal);
+                                }
+                                esBulkRequest.add(esIndexRequest);
+                            }
+                        } else {
+                            idVal = esFieldData.get(mapping.getPk());
+                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
+                                .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
+                                .size(10000);
+                            SearchResponse response = esSearchRequest.getResponse();
+                            for (SearchHit hit : response.getHits()) {
+                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(
+                                    mapping.get_index(),
+                                    hit.getId()).setDoc(esFieldData);
+                                esBulkRequest.add(esUpdateRequest);
+                            }
+                        }
+
+                        if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
+                            && esBulkRequest.numberOfActions() > 0) {
+                            long esBatchBegin = System.currentTimeMillis();
+                            ESBulkResponse rp = esBulkRequest.bulk();
+                            if (rp.hasFailures()) {
+                                rp.processFailBulkResponse("全量数据 etl 异常 ");
+                            }
+
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                                    (System.currentTimeMillis() - batchBegin),
+                                    (System.currentTimeMillis() - esBatchBegin),
+                                    esBulkRequest.numberOfActions(),
+                                    mapping.get_index());
+                            }
+                            batchBegin = System.currentTimeMillis();
+                            esBulkRequest.resetBulk();
+                        }
+                        count++;
+                        impCount.incrementAndGet();
+                    }
+
+                    if (esBulkRequest.numberOfActions() > 0) {
+                        long esBatchBegin = System.currentTimeMillis();
+                        ESBulkResponse rp = esBulkRequest.bulk();
+                        if (rp.hasFailures()) {
+                            rp.processFailBulkResponse("全量数据 etl 异常 ");
+                        }
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                                (System.currentTimeMillis() - batchBegin),
+                                (System.currentTimeMillis() - esBatchBegin),
+                                esBulkRequest.numberOfActions(),
+                                mapping.get_index());
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
+                    throw new RuntimeException(e);
+                }
+                return count;
+            });
+
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+    }
+}

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ES8xTemplate.java

@@ -0,0 +1,466 @@
+package com.alibaba.otter.canal.client.adapter.es8x.support;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
+import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+
+public class ES8xTemplate implements ESTemplate {
+
+    private static final Logger                               logger         = LoggerFactory
+        .getLogger(ESTemplate.class);
+
+    private static final int                                  MAX_BATCH_SIZE = 1000;
+
+    private ESConnection                                      esConnection;
+
+    private ESBulkRequest                                     esBulkRequest;
+
+    // es 字段类型本地缓存
+    private static ConcurrentMap<String, Map<String, String>> esFieldTypes   = new ConcurrentHashMap<>();
+
+    public ES8xTemplate(ESConnection esConnection){
+        this.esConnection = esConnection;
+        this.esBulkRequest = this.esConnection.new ES8xBulkRequest();
+    }
+
+    public ESBulkRequest getBulk() {
+        return esBulkRequest;
+    }
+
+    public void resetBulkRequestBuilder() {
+        this.esBulkRequest.resetBulk();
+    }
+
+    @Override
+    public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
+        if (mapping.get_id() != null) {
+            String parentVal = (String) esFieldData.remove("$parent_routing");
+            if (mapping.isUpsert()) {
+                ESUpdateRequest updateRequest = esConnection.new ES8xUpdateRequest(mapping.get_index(),
+                    pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+                if (StringUtils.isNotEmpty(parentVal)) {
+                    updateRequest.setRouting(parentVal);
+                }
+                getBulk().add(updateRequest);
+            } else {
+                ESIndexRequest indexRequest = esConnection.new ES8xIndexRequest(mapping.get_index(), pkVal.toString())
+                    .setSource(esFieldData);
+                if (StringUtils.isNotEmpty(parentVal)) {
+                    indexRequest.setRouting(parentVal);
+                }
+                getBulk().add(indexRequest);
+            }
+            commitBulk();
+        } else {
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
+                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+                .size(10000);
+            SearchResponse response = esSearchRequest.getResponse();
+
+            for (SearchHit hit : response.getHits()) {
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.get_index(),
+                    hit.getId()).setDoc(esFieldData);
+                getBulk().add(esUpdateRequest);
+                commitBulk();
+            }
+        }
+    }
+
+    @Override
+    public void update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
+        Map<String, Object> esFieldDataTmp = new LinkedHashMap<>(esFieldData.size());
+        esFieldData.forEach((k, v) -> esFieldDataTmp.put(Util.cleanColumn(k), v));
+        append4Update(mapping, pkVal, esFieldDataTmp);
+        commitBulk();
+    }
+
+    @Override
+    public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
+        if (paramsTmp.isEmpty()) {
+            return;
+        }
+        ESMapping mapping = config.getEsMapping();
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
+
+        // 查询sql批量更新
+        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
+        List<Object> values = new ArrayList<>();
+        paramsTmp.forEach((fieldName, value) -> {
+            sql.append("_v.").append(fieldName).append("=? AND ");
+            values.add(value);
+        });
+        // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
+        int len = sql.length();
+        sql.delete(len - 4, len);
+        Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
+            int count = 0;
+            try {
+                while (rs.next()) {
+                    Object idVal = getIdValFromRS(mapping, rs);
+                    append4Update(mapping, idVal, esFieldData);
+                    commitBulk();
+                    count++;
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+            return count;
+        });
+        if (logger.isTraceEnabled()) {
+            logger.trace("Update ES by query affected {} records", syncCount);
+        }
+    }
+
+    @Override
+    public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
+        if (mapping.get_id() != null) {
+            ESDeleteRequest esDeleteRequest = this.esConnection.new ES8xDeleteRequest(mapping.get_index(),
+                pkVal.toString());
+            getBulk().add(esDeleteRequest);
+            commitBulk();
+        } else {
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
+                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+                .size(10000);
+            SearchResponse response = esSearchRequest.getResponse();
+            for (SearchHit hit : response.getHits()) {
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.get_index(),
+                    hit.getId()).setDoc(esFieldData);
+                getBulk().add(esUpdateRequest);
+                commitBulk();
+            }
+        }
+    }
+
+    @Override
+    public void commit() {
+        if (getBulk().numberOfActions() > 0) {
+            ESBulkResponse response = getBulk().bulk();
+            if (response.hasFailures()) {
+                response.processFailBulkResponse("ES sync commit error ");
+            }
+            resetBulkRequestBuilder();
+        }
+    }
+
+    @Override
+    public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName,
+                               String columnName) throws SQLException {
+        fieldName = Util.cleanColumn(fieldName);
+        columnName = Util.cleanColumn(columnName);
+        String esType = getEsType(mapping, fieldName);
+
+        Object value = resultSet.getObject(columnName);
+        if (value instanceof Boolean) {
+            if (!"boolean".equals(esType)) {
+                value = resultSet.getByte(columnName);
+            }
+        }
+
+        // 如果是对象类型
+        if (mapping.getObjFields().containsKey(fieldName)) {
+            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
+        } else {
+            return ESSyncUtil.typeConvert(value, esType);
+        }
+    }
+
+    @Override
+    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,
+                                  Map<String, Object> esFieldData) throws SQLException {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = value;
+            }
+
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
+                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
+            }
+        }
+
+        // 添加父子文档关联信息
+        putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
+
+        return resultIdVal;
+    }
+
+    @Override
+    public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = value;
+                break;
+            }
+        }
+        return resultIdVal;
+    }
+
+    @Override
+    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
+                                  Map<String, Object> esFieldData) throws SQLException {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
+            }
+
+            for (ColumnItem columnItem : fieldItem.getColumnItems()) {
+                if (dmlOld.containsKey(columnItem.getColumnName())
+                    && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                    esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
+                        getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
+                    break;
+                }
+            }
+        }
+
+        // 添加父子文档关联信息
+        putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
+
+        return resultIdVal;
+    }
+
+    @Override
+    public Object getValFromData(ESMapping mapping, Map<String, Object> dmlData, String fieldName, String columnName) {
+        String esType = getEsType(mapping, fieldName);
+        Object value = dmlData.get(columnName);
+        if (value instanceof Byte) {
+            if ("boolean".equals(esType)) {
+                value = ((Byte) value).intValue() != 0;
+            }
+        }
+
+        // 如果是对象类型
+        if (mapping.getObjFields().containsKey(fieldName)) {
+            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
+        } else {
+            return ESSyncUtil.typeConvert(value, esType);
+        }
+    }
+
+    @Override
+    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
+                                       Map<String, Object> esFieldData) {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
+            Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = value;
+            }
+
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
+                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
+            }
+        }
+
+        // 添加父子文档关联信息
+        putRelationData(mapping, schemaItem, dmlData, esFieldData);
+        return resultIdVal;
+    }
+
+    @Override
+    public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
+                                       Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
+        SchemaItem schemaItem = mapping.getSchemaItem();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
+        Object resultIdVal = null;
+        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
+            ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
+            if (!columnItem.getOwner().equals(owner)) {
+                continue;
+            }
+            String columnName = columnItem.getColumnName();
+
+            if (fieldItem.getFieldName().equals(idFieldName)) {
+                resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
+            }
+
+            if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
+                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
+                    getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
+            }
+        }
+
+        // 添加父子文档关联信息
+        putRelationData(mapping, schemaItem, dmlOld, esFieldData);
+        return resultIdVal;
+    }
+
+    /**
+     * 如果大于批量数则提交批次
+     */
+    private void commitBulk() {
+        if (getBulk().numberOfActions() >= MAX_BATCH_SIZE) {
+            commit();
+        }
+    }
+
+    private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
+        if (mapping.get_id() != null) {
+            String parentVal = (String) esFieldData.remove("$parent_routing");
+            if (mapping.isUpsert()) {
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.get_index(),
+                    pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+                if (StringUtils.isNotEmpty(parentVal)) {
+                    esUpdateRequest.setRouting(parentVal);
+                }
+                getBulk().add(esUpdateRequest);
+            } else {
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.get_index(),
+                    pkVal.toString()).setDoc(esFieldData);
+                if (StringUtils.isNotEmpty(parentVal)) {
+                    esUpdateRequest.setRouting(parentVal);
+                }
+                getBulk().add(esUpdateRequest);
+            }
+        } else {
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
+                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
+                .size(10000);
+            SearchResponse response = esSearchRequest.getResponse();
+            for (SearchHit hit : response.getHits()) {
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.get_index(),
+                    hit.getId()).setDoc(esFieldData);
+                getBulk().add(esUpdateRequest);
+            }
+        }
+    }
+
+    /**
+     * 获取es mapping中的属性类型
+     *
+     * @param mapping mapping配置
+     * @param fieldName 属性名
+     * @return 类型
+     */
+    @SuppressWarnings("unchecked")
+    private String getEsType(ESMapping mapping, String fieldName) {
+        String key = mapping.get_index() + "-" + mapping.get_type();
+        Map<String, String> fieldType = esFieldTypes.get(key);
+        if (fieldType != null) {
+            return fieldType.get(fieldName);
+        } else {
+            MappingMetadata mappingMetaData = esConnection.getMapping(mapping.get_index());
+
+            if (mappingMetaData == null) {
+                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
+            }
+
+            fieldType = new LinkedHashMap<>();
+
+            Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
+            Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
+            for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
+                Map<String, Object> value = (Map<String, Object>) entry.getValue();
+                if (value.containsKey("properties")) {
+                    fieldType.put(entry.getKey(), "object");
+                } else {
+                    fieldType.put(entry.getKey(), (String) value.get("type"));
+                }
+            }
+            esFieldTypes.put(key, fieldType);
+
+            return fieldType.get(fieldName);
+        }
+    }
+
+    private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet,
+                                       Map<String, Object> esFieldData) {
+        // 添加父子文档关联信息
+        if (!mapping.getRelations().isEmpty()) {
+            mapping.getRelations().forEach((relationField, relationMapping) -> {
+                Map<String, Object> relations = new HashMap<>();
+                relations.put("name", relationMapping.getName());
+                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
+                    FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
+                    Object parentVal;
+                    try {
+                        parentVal = getValFromRS(mapping,
+                            resultSet,
+                            parentFieldItem.getFieldName(),
+                            parentFieldItem.getFieldName());
+                    } catch (SQLException e) {
+                        throw new RuntimeException(e);
+                    }
+                    if (parentVal != null) {
+                        relations.put("parent", parentVal.toString());
+                        esFieldData.put("$parent_routing", parentVal.toString());
+
+                    }
+                }
+                esFieldData.put(relationField, relations);
+            });
+        }
+    }
+
+    private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map<String, Object> dmlData,
+                                 Map<String, Object> esFieldData) {
+        // 添加父子文档关联信息
+        if (!mapping.getRelations().isEmpty()) {
+            mapping.getRelations().forEach((relationField, relationMapping) -> {
+                Map<String, Object> relations = new HashMap<>();
+                relations.put("name", relationMapping.getName());
+                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
+                    FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
+                    String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName();
+                    Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName);
+                    if (parentVal != null) {
+                        relations.put("parent", parentVal.toString());
+                        esFieldData.put("$parent_routing", parentVal.toString());
+
+                    }
+                }
+                esFieldData.put(relationField, relations);
+            });
+        }
+    }
+}

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ESConnection.java

@@ -0,0 +1,399 @@
+package com.alibaba.otter.canal.client.adapter.es8x.support;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteRequestBuilder;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateRequestBuilder;
+import org.elasticsearch.client.*;
+import org.elasticsearch.client.indices.GetMappingsRequest;
+import org.elasticsearch.client.indices.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
+
+/**
+ * ES 连接器, 只支持 Rest 方式
+ *
+ * @author ymz 2023-03-02
+ * @version 1.0.0
+ */
+public class ESConnection {
+
+    private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
+
+    private RestHighLevelClient restHighLevelClient;
+
+    public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException{
+        HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
+        RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
+        String nameAndPwd = properties.get("security.auth");
+        if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
+            String[] nameAndPwdArr = nameAndPwd.split(":");
+            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
+            restClientBuilder.setHttpClientConfigCallback(
+                httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+        }
+        restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build()).setApiCompatibilityMode(true)
+            .build();
+    }
+
+    public void close() {
+        try {
+            restHighLevelClient.close();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public MappingMetadata getMapping(String index) {
+        MappingMetadata mappingMetaData = null;
+
+        Map<String, MappingMetadata> mappings;
+        try {
+            GetMappingsRequest request = new GetMappingsRequest();
+            request.indices(index);
+            GetMappingsResponse response = restHighLevelClient.indices().getMapping(request, RequestOptions.DEFAULT);
+
+            mappings = response.mappings();
+        } catch (NullPointerException e) {
+            throw new IllegalArgumentException("Not found the mapping info of index: " + index);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            return null;
+        }
+        mappingMetaData = mappings.get(index);
+
+        return mappingMetaData;
+    }
+
+    public class ES8xIndexRequest implements ESBulkRequest.ESIndexRequest {
+
+        private IndexRequestBuilder indexRequestBuilder;
+
+        private IndexRequest        indexRequest;
+
+        public ES8xIndexRequest(String index, String id){
+            indexRequest = new IndexRequest(index);
+            indexRequest.id(id);
+
+        }
+
+        public ES8xIndexRequest setSource(Map<String, ?> source) {
+
+            indexRequest.source(source);
+
+            return this;
+        }
+
+        public ES8xIndexRequest setRouting(String routing) {
+
+            indexRequest.routing(routing);
+
+            return this;
+        }
+
+        public IndexRequestBuilder getIndexRequestBuilder() {
+            return indexRequestBuilder;
+        }
+
+        public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) {
+            this.indexRequestBuilder = indexRequestBuilder;
+        }
+
+        public IndexRequest getIndexRequest() {
+            return indexRequest;
+        }
+
+        public void setIndexRequest(IndexRequest indexRequest) {
+            this.indexRequest = indexRequest;
+        }
+    }
+
+    public class ES8xUpdateRequest implements ESBulkRequest.ESUpdateRequest {
+
+        private UpdateRequestBuilder updateRequestBuilder;
+
+        private UpdateRequest        updateRequest;
+
+        public ES8xUpdateRequest(String index, String id){
+
+            updateRequest = new UpdateRequest(index, id);
+        }
+
+        public ES8xUpdateRequest setDoc(Map source) {
+
+            updateRequest.doc(source);
+
+            return this;
+        }
+
+        public ES8xUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) {
+
+            updateRequest.docAsUpsert(shouldUpsertDoc);
+
+            return this;
+        }
+
+        public ES8xUpdateRequest setRouting(String routing) {
+
+            updateRequest.routing(routing);
+
+            return this;
+        }
+
+        public UpdateRequestBuilder getUpdateRequestBuilder() {
+            return updateRequestBuilder;
+        }
+
+        public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) {
+            this.updateRequestBuilder = updateRequestBuilder;
+        }
+
+        public UpdateRequest getUpdateRequest() {
+            return updateRequest;
+        }
+
+        public void setUpdateRequest(UpdateRequest updateRequest) {
+            this.updateRequest = updateRequest;
+        }
+    }
+
+    public class ES8xDeleteRequest implements ESBulkRequest.ESDeleteRequest {
+
+        private DeleteRequestBuilder deleteRequestBuilder;
+
+        private DeleteRequest        deleteRequest;
+
+        public ES8xDeleteRequest(String index, String id){
+
+            deleteRequest = new DeleteRequest(index, id);
+
+        }
+
+        public DeleteRequestBuilder getDeleteRequestBuilder() {
+            return deleteRequestBuilder;
+        }
+
+        public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) {
+            this.deleteRequestBuilder = deleteRequestBuilder;
+        }
+
+        public DeleteRequest getDeleteRequest() {
+            return deleteRequest;
+        }
+
+        public void setDeleteRequest(DeleteRequest deleteRequest) {
+            this.deleteRequest = deleteRequest;
+        }
+    }
+
+    public class ESSearchRequest {
+
+        private SearchRequestBuilder searchRequestBuilder;
+
+        private SearchRequest        searchRequest;
+
+        private SearchSourceBuilder  sourceBuilder;
+
+        public ESSearchRequest(String index){
+
+            searchRequest = new SearchRequest(index);
+            sourceBuilder = new SearchSourceBuilder();
+
+        }
+
+        public ESSearchRequest setQuery(QueryBuilder queryBuilder) {
+
+            sourceBuilder.query(queryBuilder);
+
+            return this;
+        }
+
+        public ESSearchRequest size(int size) {
+
+            sourceBuilder.size(size);
+
+            return this;
+        }
+
+        public SearchResponse getResponse() {
+
+            searchRequest.source(sourceBuilder);
+            try {
+                return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+
+        public SearchRequestBuilder getSearchRequestBuilder() {
+            return searchRequestBuilder;
+        }
+
+        public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) {
+            this.searchRequestBuilder = searchRequestBuilder;
+        }
+
+        public SearchRequest getSearchRequest() {
+            return searchRequest;
+        }
+
+        public void setSearchRequest(SearchRequest searchRequest) {
+            this.searchRequest = searchRequest;
+        }
+    }
+
+    public class ES8xBulkRequest implements ESBulkRequest {
+
+        private BulkRequestBuilder bulkRequestBuilder;
+
+        private BulkRequest        bulkRequest;
+
+        public ES8xBulkRequest(){
+
+            bulkRequest = new BulkRequest();
+
+        }
+
+        public void resetBulk() {
+
+            bulkRequest = new BulkRequest();
+
+        }
+
+        public ES8xBulkRequest add(ESIndexRequest esIndexRequest) {
+            ES8xIndexRequest eir = (ES8xIndexRequest) esIndexRequest;
+
+            bulkRequest.add(eir.indexRequest);
+
+            return this;
+        }
+
+        public ES8xBulkRequest add(ESUpdateRequest esUpdateRequest) {
+            ES8xUpdateRequest eur = (ES8xUpdateRequest) esUpdateRequest;
+
+            bulkRequest.add(eur.updateRequest);
+
+            return this;
+        }
+
+        public ES8xBulkRequest add(ESDeleteRequest esDeleteRequest) {
+            ES8xDeleteRequest edr = (ES8xDeleteRequest) esDeleteRequest;
+
+            bulkRequest.add(edr.deleteRequest);
+
+            return this;
+        }
+
+        public int numberOfActions() {
+            return bulkRequest.numberOfActions();
+        }
+
+        public ESBulkResponse bulk() {
+            try {
+                BulkResponse responses = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
+                return new ES8xBulkResponse(responses);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+
+        public BulkRequestBuilder getBulkRequestBuilder() {
+            return bulkRequestBuilder;
+        }
+
+        public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) {
+            this.bulkRequestBuilder = bulkRequestBuilder;
+        }
+
+        public BulkRequest getBulkRequest() {
+            return bulkRequest;
+        }
+
+        public void setBulkRequest(BulkRequest bulkRequest) {
+            this.bulkRequest = bulkRequest;
+        }
+    }
+
+    public static class ES8xBulkResponse implements ESBulkRequest.ESBulkResponse {
+
+        private BulkResponse bulkResponse;
+
+        public ES8xBulkResponse(BulkResponse bulkResponse){
+            this.bulkResponse = bulkResponse;
+        }
+
+        @Override
+        public boolean hasFailures() {
+            return bulkResponse.hasFailures();
+        }
+
+        @Override
+        public void processFailBulkResponse(String errorMsg) {
+            for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
+                if (!itemResponse.isFailed()) {
+                    continue;
+                }
+
+                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
+                    logger.error(itemResponse.getFailureMessage());
+                } else {
+                    throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
+                }
+            }
+        }
+    }
+
+    public RestHighLevelClient getRestHighLevelClient() {
+        return restHighLevelClient;
+    }
+
+    public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    private HttpHost createHttpHost(String uriStr) {
+        URI uri = URI.create(uriStr);
+        if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
+            return HttpHost.create(uri.toString());
+        }
+        try {
+            return HttpHost.create(new URI(uri
+                .getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
+                    .toString());
+        } catch (URISyntaxException ex) {
+            throw new IllegalStateException(ex);
+        }
+    }
+}

client-adapter/es8x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+es8=com.alibaba.otter.canal.client.adapter.es8x.ES8xAdapter

client-adapter/es8x/src/main/resources/es8/biz_order.yml

@@ -0,0 +1,20 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+esMapping:
+  _index: customer
+  _id: _id
+  relations:
+    customer_order:
+      name: order
+      parent: customer_id
+  sql: "select concat('oid_', t.id) as _id,
+        t.customer_id,
+        t.id as order_id,
+        t.serial_code as order_serial,
+        t.c_time as order_time
+        from biz_order t"
+  skips:
+    - customer_id
+  etlCondition: "where t.c_time>={}"
+  commitBatch: 3000

client-adapter/es8x/src/main/resources/es8/customer.yml

@@ -0,0 +1,46 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+esMapping:
+  _index: customer
+  _id: id
+  relations:
+    customer_order:
+      name: customer
+  sql: "select t.id, t.name, t.email from customer t"
+  etlCondition: "where t.c_time>={}"
+  commitBatch: 3000
+
+
+#{
+#  "mappings":{
+#    "_doc":{
+#      "properties":{
+#        "id": {
+#          "type": "long"
+#        },
+#        "name": {
+#          "type": "text"
+#        },
+#        "email": {
+#          "type": "text"
+#        },
+#        "order_id": {
+#          "type": "long"
+#        },
+#        "order_serial": {
+#          "type": "text"
+#        },
+#        "order_time": {
+#          "type": "date"
+#        },
+#        "customer_order":{
+#          "type":"join",
+#          "relations":{
+#            "customer":"order"
+#          }
+#        }
+#      }
+#    }
+#  }
+#}

client-adapter/es8x/src/main/resources/es8/mytest_user.yml

@@ -0,0 +1,15 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+esMapping:
+  _index: mytest_user
+  _id: _id
+  #  upsert: true
+  #  pk: id
+  sql: "select a.id as _id, a.name, a.role_id, b.role_name,
+        a.c_time from user a
+        left join role b on b.id=a.role_id"
+  #  objFields:
+  #    _labels: array:;
+  etlCondition: "where a.c_time>={}"
+  commitBatch: 3000

client-adapter/es8x/src/test/java/com/alibaba/otter/canal/client/adapter/es8x/test/ESConnectionTest.java

@@ -0,0 +1,44 @@
+package com.alibaba.otter.canal.client.adapter.es8x.test;
+
+import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.springframework.util.Assert;
+
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+
+@Ignore
+public class ESConnectionTest {
+
+    ESConnection esConnection;
+
+    @Before
+    public void init() throws UnknownHostException {
+        String[] hosts = new String[]{"127.0.0.1:9200"};
+        Map<String, String> properties = new HashMap<>();
+        properties.put("cluster.name", "elasticsearch");
+        esConnection = new ESConnection(hosts, properties);
+    }
+
+    @Test
+    public void test01() {
+        MappingMetadata mappingMetaData = esConnection.getMapping("mytest_user");
+
+        Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
+        Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
+        for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
+            Map<String, Object> value = (Map<String, Object>) entry.getValue();
+            if (value.containsKey("properties")) {
+                System.out.println(entry.getKey() + " object");
+            } else {
+                System.out.println(entry.getKey() + " " + value.get("type"));
+                Assert.notNull(entry.getKey(), "null column name");
+                Assert.notNull(value.get("type"), "null column type");
+            }
+        }
+    }
+}

client-adapter/es8x/src/test/java/com/alibaba/otter/canal/client/adapter/es8x/test/TestConstant.java

@@ -0,0 +1,40 @@
+package com.alibaba.otter.canal.client.adapter.es8x.test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+import java.sql.SQLException;
+
+public class TestConstant {
+
+    public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
+    public final static String jdbcUser = "root";
+    public final static String jdbcPassword = "121212";
+
+    public final static String esHosts = "127.0.0.1:9300";
+    public final static String clusterName = "elasticsearch";
+
+    public final static DruidDataSource dataSource;
+
+    static {
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUser);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setPoolPreparedStatements(false);
+        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
+        dataSource.setValidationQuery("select 1");
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+}

client-adapter/hbase/pom.xml

@@ -21,7 +21,6 @@
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-shaded-client</artifactId>
-            <version>1.1.2</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

client-adapter/kudu/pom.xml

@@ -21,18 +21,15 @@
         <dependency>
             <groupId>org.apache.kudu</groupId>
             <artifactId>kudu-client</artifactId>
-            <version>1.6.0</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.12</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
-            <version>5.1.48</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

client-adapter/launcher/pom.xml

@@ -17,7 +17,7 @@
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>
-                <version>2.0.1.RELEASE</version>
+                <version>2.5.4</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -128,6 +128,19 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.es8x</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>com.alibaba.otter</groupId>
             <artifactId>client-adapter.rdb</artifactId>

client-adapter/launcher/src/main/assembly/dev.xml

@@ -43,6 +43,13 @@
                 <include>**/*</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>../es8x/src/main/resources/es8</directory>
+            <outputDirectory>/conf/es8</outputDirectory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+        </fileSet>
         <fileSet>
             <directory>../hbase/src/main/resources/hbase</directory>
             <outputDirectory>/conf/hbase</outputDirectory>

client-adapter/launcher/src/main/assembly/release.xml

@@ -44,6 +44,13 @@
                 <include>**/*</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>../es8x/src/main/resources/es8</directory>
+            <outputDirectory>/conf/es8</outputDirectory>
+            <includes>
+                <include>**/*</include>
+            </includes>
+        </fileSet>
         <fileSet>
             <directory>../hbase/src/main/resources/hbase</directory>
             <outputDirectory>/conf/hbase</outputDirectory>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.adapter.launcher;
 import org.springframework.boot.Banner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 
 /**
  * 启动入口
@@ -10,7 +11,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
  * @author rewerma @ 2018-10-20
  * @version 1.0.0
  */
-@SpringBootApplication
+@SpringBootApplication(exclude= { DataSourceAutoConfiguration.class})
 public class CanalAdapterApplication {
     public static void main(String[] args) {
         // 支持rocketmq client 配置日志路径

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -7,7 +7,6 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,36 +22,37 @@ import com.alibaba.otter.canal.connector.core.config.CanalConstants;
 import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
 import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
+import com.alibaba.otter.canal.connector.core.spi.ProxyCanalMsgConsumer;
 
 /**
  * 适配处理器
- * 
+ *
  * @author rewerma 2020-02-01
  * @version 1.0.0
  */
 public class AdapterProcessor {
 
-    private static final Logger             logger                    = LoggerFactory.getLogger(AdapterProcessor.class);
+    private static final Logger logger = LoggerFactory.getLogger(AdapterProcessor.class);
 
-    private static final String             CONNECTOR_SPI_DIR         = "/plugin";
-    private static final String             CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
+    private static final String CONNECTOR_SPI_DIR = "/plugin";
+    private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
 
-    private CanalMsgConsumer                canalMsgConsumer;
+    private CanalMsgConsumer canalMsgConsumer;
 
-    private String                          canalDestination;                                                           // canal实例
-    private String                          groupId                   = null;                                           // groupId
-    private List<List<OuterAdapter>>        canalOuterAdapters;                                                         // 外部适配器
-    private CanalClientConfig               canalClientConfig;                                                          // 配置
-    private ExecutorService                 groupInnerExecutorService;                                                  // 组内工作线程池
-    private volatile boolean                running                   = false;                                          // 是否运行中
-    private Thread                          thread                    = null;
-    private Thread.UncaughtExceptionHandler handler                   = (t, e) -> logger
+    private String canalDestination;                                                           // canal实例
+    private String groupId = null;                                           // groupId
+    private List<List<OuterAdapter>> canalOuterAdapters;                                                         // 外部适配器
+    private CanalClientConfig canalClientConfig;                                                          // 配置
+    private ExecutorService groupInnerExecutorService;                                                  // 组内工作线程池
+    private volatile boolean running = false;                                          // 是否运行中
+    private Thread thread = null;
+    private Thread.UncaughtExceptionHandler handler = (t, e) -> logger
         .error("parse events has an error", e);
 
-    private SyncSwitch                      syncSwitch;
+    private SyncSwitch syncSwitch;
 
     public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
-                            List<List<OuterAdapter>> canalOuterAdapters){
+        List<List<OuterAdapter>> canalOuterAdapters) {
         this.canalClientConfig = canalClientConfig;
         this.canalDestination = destination;
         this.groupId = groupId;
@@ -63,17 +63,15 @@ public class AdapterProcessor {
 
         // load connector consumer
         ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
-        canalMsgConsumer = loader
-            .getExtension(canalClientConfig.getMode().toLowerCase(),destination ,CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+        canalMsgConsumer = new ProxyCanalMsgConsumer(loader
+            .getExtension(canalClientConfig.getMode().toLowerCase(), destination, CONNECTOR_SPI_DIR,
+                CONNECTOR_STANDBY_SPI_DIR));
 
         Properties properties = canalClientConfig.getConsumerProperties();
         properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
         properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey());
         properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey());
-        ClassLoader cl = Thread.currentThread().getContextClassLoader();
-        Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
         canalMsgConsumer.init(properties, canalDestination, groupId);
-        Thread.currentThread().setContextClassLoader(cl);
     }
 
     public void start() {
@@ -88,6 +86,7 @@ public class AdapterProcessor {
     public void writeOut(final List<CommonMessage> commonMessages) {
         List<Future<Boolean>> futures = new ArrayList<>();
         // 组间适配器并行运行
+        // 当 canalOuterAdapters 初始化失败时，消息将会全部丢失
         canalOuterAdapters.forEach(outerAdapters -> {
             futures.add(groupInnerExecutorService.submit(() -> {
                 try {
@@ -171,7 +170,7 @@ public class AdapterProcessor {
         }
 
         int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+            || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
         if (retry == -1) {
             // 重试次数-1代表异常时一直阻塞重试
             retry = Integer.MAX_VALUE;

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -16,9 +16,11 @@ import org.springframework.core.env.EnumerablePropertySource;
 import org.springframework.core.env.Environment;
 import org.springframework.core.env.PropertySource;
 import org.springframework.core.env.StandardEnvironment;
+import org.springframework.util.CollectionUtils;
 
 import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
 import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.ProxyOuterAdapter;
 import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
 import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
@@ -69,7 +71,13 @@ public class CanalAdapterLoader {
                     loadAdapter(config, canalOuterAdapters);
                 }
                 canalOuterAdapterGroups.add(canalOuterAdapters);
-
+                // canalOuterAdapters 存在初始化失败的情况，导致canalOuterAdapters的数量，可能小于group.getOuterAdapters
+                // 由于group下的 所有OuterAdapter实例都会重复消费同一批消息，因此不允许部分adapter初始化成功，必须全部初始化成功才允许消费
+                if(CollectionUtils.isEmpty(canalOuterAdapters) || canalOuterAdapters.size() != group.getOuterAdapters().size() ){
+                    String msg = String.format("instance=%s,groupId=%s Load OuterAdapters is Empty，pls check rdb.yml",
+                                canalAdapter.getInstance(),group.getGroupId());
+                        throw new RuntimeException(msg);
+                 }
                 AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(
                     canalAdapter.getInstance() + "|" + StringUtils.trimToEmpty(group.getGroupId()),
                     f -> new AdapterProcessor(canalClientConfig,
@@ -77,7 +85,6 @@ public class CanalAdapterLoader {
                         group.getGroupId(),
                         canalOuterAdapterGroups));
                 adapterProcessor.start();
-
                 logger.info("Start adapter for canal-client mq topic: {} succeed",
                     canalAdapter.getInstance() + "-" + group.getGroupId());
             }
@@ -87,11 +94,8 @@ public class CanalAdapterLoader {
     private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
         try {
             OuterAdapter adapter;
-            adapter = loader.getExtension(config.getName(), config.getKey());
+            adapter = new ProxyOuterAdapter(loader.getExtension(config.getName(), config.getKey()));
 
-            ClassLoader cl = Thread.currentThread().getContextClassLoader();
-            // 替换ClassLoader
-            Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
             Environment env = (Environment) SpringContext.getBean(Environment.class);
             Properties evnProperties = null;
             if (env instanceof StandardEnvironment) {
@@ -109,7 +113,7 @@ public class CanalAdapterLoader {
                 }
             }
             adapter.init(config, evnProperties);
-            Thread.currentThread().setContextClassLoader(cl);
+            // rdb文件解析异常时，canalOuterAdapters 无法正常加载
             canalOutConnectors.add(adapter);
             logger.info("Load canal adapter: {} succeed", config.getName());
         } catch (Exception e) {

client-adapter/logger/pom.xml

@@ -20,7 +20,6 @@
         </dependency>
     </dependencies>
 
-
     <build>
         <plugins>
             <plugin>

client-adapter/phoenix/pom.xml

@@ -17,13 +17,12 @@
         <dependency>
             <groupId>org.apache.phoenix</groupId>
             <artifactId>phoenix-core</artifactId>
-            <version>4.14.1-HBase-1.4</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-client</artifactId>
-            <version>1.4.8</version>
         </dependency>
+        <!-- 强制指定protobuf版本 -->
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>
@@ -71,7 +70,6 @@
             <artifactId>fastjson2</artifactId>
             <scope>provided</scope>
         </dependency>
-
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/service/PhoenixEtlService.java

@@ -1,5 +1,17 @@
 package com.alibaba.otter.canal.client.adapter.phoenix.service;
 
+import java.sql.*;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.sql.DataSource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
 import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
 import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping;
@@ -10,16 +22,6 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
 import com.alibaba.otter.canal.client.adapter.support.Util;
 import com.google.common.base.Joiner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.sql.DataSource;
-import java.sql.*;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Phoenix ETL 操作业务类
@@ -41,16 +43,17 @@ public class PhoenixEtlService {
         if (srcDataSource == null) {
             return false;
         }
-        try {
-            return syncSchema(srcDataSource.getConnection(), targetDSConnection, config);
+
+        try (Connection conn = srcDataSource.getConnection()) {
+            return syncSchema(conn, targetDSConnection, config);
         } catch (SQLException e) {
             throw new RuntimeException(e);
         }
     }
 
     private static boolean syncSchema(DataSource srcDS,Connection targetDSConnection, MappingConfig config) {
-        try {
-            return syncSchema(srcDS.getConnection(),targetDSConnection, config);
+        try (Connection conn = srcDS.getConnection()) {
+            return syncSchema(conn, targetDSConnection, config);
         } catch (SQLException e) {
             throw new RuntimeException(e);
         }

client-adapter/pom.xml

@@ -31,6 +31,7 @@
         <module>rdb</module>
         <module>es6x</module>
         <module>es7x</module>
+        <module>es8x</module>
         <module>escore</module>
         <module>kudu</module>
         <module>phoenix</module>
@@ -53,7 +54,7 @@
     <repositories>
         <repository>
             <id>central</id>
-            <url>http://repo1.maven.org/maven2</url>
+            <url>https://repo1.maven.org/maven2</url>
             <releases>
                 <enabled>true</enabled>
             </releases>
@@ -63,7 +64,7 @@
         </repository>
         <repository>
             <id>java.net</id>
-            <url>http://download.java.net/maven/2/</url>
+            <url>https://download.java.net/maven/2/</url>
             <releases>
                 <enabled>true</enabled>
             </releases>
@@ -73,7 +74,7 @@
         </repository>
         <repository>
             <id>aliyun</id>
-            <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+            <url>https://maven.aliyun.com/nexus/content/groups/public/</url>
             <releases>
                 <enabled>true</enabled>
             </releases>
@@ -110,12 +111,12 @@
             <dependency>
                 <groupId>org.springframework.cloud</groupId>
                 <artifactId>spring-cloud-context</artifactId>
-                <version>2.0.0.RELEASE</version>
+                <version>3.0.6</version>
             </dependency>
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot</artifactId>
-                <version>2.0.1.RELEASE</version>
+                <version>2.5.4</version>
             </dependency>
             <dependency>
                 <groupId>com.h2database</groupId>
@@ -126,35 +127,35 @@
             <dependency>
                 <groupId>org.apache.rocketmq</groupId>
                 <artifactId>rocketmq-client</artifactId>
-            <version>4.5.1</version>
+            <version>${rocketmq_version}</version>
             </dependency>
             <!-- 单独引入kafka依赖 -->
             <dependency>
                 <groupId>org.apache.kafka</groupId>
                 <artifactId>kafka-clients</artifactId>
-                <version>1.1.1</version>
+                <version>${kafka_version}</version>
             </dependency>
-
+            <!-- rabbotmq -->
             <dependency>
                 <groupId>com.rabbitmq</groupId>
                 <artifactId>amqp-client</artifactId>
-                <version>5.5.0</version>
+                <version>${rabbitmq_version}</version>
             </dependency>
             <dependency>
                 <groupId>com.alibaba.mq-amqp</groupId>
                 <artifactId>mq-amqp-client</artifactId>
-                <version>1.0.3</version>
+                <version>${mq_amqp_client}</version>
             </dependency>
             <!-- jdbc -->
             <dependency>
                 <groupId>mysql</groupId>
                 <artifactId>mysql-connector-java</artifactId>
-                <version>5.1.48</version>
+                <version>${mysql_driver_version}</version>
             </dependency>
             <dependency>
                 <groupId>org.postgresql</groupId>
                 <artifactId>postgresql</artifactId>
-                <version>42.1.4</version>
+                <version>42.3.3</version>
             </dependency>
             <dependency>
                 <groupId>com.oracle.database.jdbc</groupId>
@@ -166,7 +167,6 @@
                 <artifactId>mssql-jdbc</artifactId>
                 <version>7.0.0.jre8</version>
             </dependency>
-
             <dependency>
                 <groupId>org.apache.hbase</groupId>
                 <artifactId>hbase-shaded-client</artifactId>
@@ -182,6 +182,37 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>org.apache.phoenix</groupId>
+                <artifactId>phoenix-core</artifactId>
+                <version>4.14.1-HBase-1.4</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hbase</groupId>
+                <artifactId>hbase-client</artifactId>
+                <version>1.4.8</version>
+            </dependency>
+            <dependency>
+                <groupId>com.aliyun.openservices</groupId>
+                <artifactId>tablestore</artifactId>
+                <version>5.10.3</version>
+                <classifier>jar-with-dependencies</classifier>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.google.protobuf</groupId>
+                        <artifactId>protobuf-java</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpasyncclient</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.kudu</groupId>
+                <artifactId>kudu-client</artifactId>
+                <version>1.6.0</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.curator</groupId>
                 <artifactId>curator-recipes</artifactId>
@@ -201,8 +232,9 @@
             <dependency>
                 <groupId>org.yaml</groupId>
                 <artifactId>snakeyaml</artifactId>
-                <version>1.19</version>
+                <version>1.29</version>
             </dependency>
+            <!-- test -->
             <dependency>
                 <groupId>org.powermock</groupId>
                 <artifactId>powermock-api-mockito</artifactId>
@@ -233,39 +265,6 @@
                 <version>1.9.0</version>
                 <scope>test</scope>
             </dependency>
-
-            <dependency>
-                <groupId>com.aliyun.openservices</groupId>
-                <artifactId>tablestore</artifactId>
-                <version>5.10.3</version>
-                <classifier>jar-with-dependencies</classifier>
-                <exclusions>
-                    <exclusion>
-                        <groupId>com.google.protobuf</groupId>
-                        <artifactId>protobuf-java</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>org.apache.httpcomponents</groupId>
-                        <artifactId>httpasyncclient</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-
-            <dependency>
-                <groupId>org.apache.logging.log4j</groupId>
-                <artifactId>log4j-core</artifactId>
-                <version>${log4j_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.logging.log4j</groupId>
-                <artifactId>log4j-api</artifactId>
-                <version>${log4j_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.logging.log4j</groupId>
-                <artifactId>log4j-to-slf4j</artifactId>
-                <version>${log4j_version}</version>
-            </dependency>
         </dependencies>
     </dependencyManagement>
 

client-adapter/rdb/pom.xml

@@ -42,7 +42,6 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.12</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java

@@ -1,13 +1,8 @@
 package com.alibaba.otter.canal.client.adapter.rdb;
 
-import com.alibaba.otter.canal.client.adapter.support.FileName2KeyMapping;
 import java.sql.Connection;
 import java.sql.SQLException;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.lang.BooleanUtils;
@@ -27,11 +22,7 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
 import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
 import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
 import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.client.adapter.support.*;
 
 /**
  * RDB适配器实现类
@@ -84,6 +75,7 @@ public class RdbAdapter implements OuterAdapter {
         // 从jdbc url获取db类型
         Map<String, String> properties = configuration.getProperties();
         String dbType = JdbcUtils.getDbType(properties.get("jdbc.url"), null);
+        // 当.yml文件编码格式存在问题，此处rdb yml文件构建 可能会抛出异常
         Map<String, MappingConfig> rdbMappingTmp = ConfigLoader.load(envProperties);
         // 过滤不匹配的key的配置
         rdbMappingTmp.forEach((key, config) -> {

client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java

@@ -58,6 +58,7 @@ public class RdbEtlService extends AbstractEtlService {
             Map<String, String> columnsMap = new LinkedHashMap<>();
             Map<String, Integer> columnType = new LinkedHashMap<>();
             DruidDataSource dataSource = (DruidDataSource) srcDS;
+            String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
 
             Util.sqlRS(targetDS,
                 "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " LIMIT 1 ",
@@ -91,7 +92,7 @@ public class RdbEtlService extends AbstractEtlService {
                         .append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()))
                         .append(" (");
                     columnsMap
-                        .forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(","));
+                        .forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick).append(targetColumnName).append(backtick).append(","));
 
                     int len = insertSql.length();
                     insertSql.delete(len - 1, len).append(") VALUES (");
@@ -101,6 +102,7 @@ public class RdbEtlService extends AbstractEtlService {
                     }
                     len = insertSql.length();
                     insertSql.delete(len - 1, len).append(")");
+                    logger.info("executeSqlImport sql:{}",insertSql.toString());
                     try (Connection connTarget = targetDS.getConnection();
                             PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
                         connTarget.setAutoCommit(false);
@@ -114,7 +116,7 @@ public class RdbEtlService extends AbstractEtlService {
                             Map<String, Object> pkVal = new LinkedHashMap<>();
                             StringBuilder deleteSql = new StringBuilder(
                                 "DELETE FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " WHERE ");
-                            appendCondition(dbMapping, deleteSql, pkVal, rs);
+                            appendCondition(dbMapping, deleteSql, pkVal, rs, backtick);
                             try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
                                 int k = 1;
                                 for (Object val : pkVal.values()) {
@@ -132,7 +134,6 @@ public class RdbEtlService extends AbstractEtlService {
                                 }
 
                                 Integer type = columnType.get(targetClolumnName.toLowerCase());
-
                                 Object value = rs.getObject(srcColumnName);
                                 if (value != null) {
                                     SyncUtil.setPStmt(type, pstmt, value, i);
@@ -180,7 +181,7 @@ public class RdbEtlService extends AbstractEtlService {
      * 拼接目标表主键where条件
      */
     private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
-                                        ResultSet rs) throws SQLException {
+                                        ResultSet rs, String backtick) throws SQLException {
         // 拼接主键
         for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
             String targetColumnName = entry.getKey();
@@ -188,7 +189,7 @@ public class RdbEtlService extends AbstractEtlService {
             if (srcColumnName == null) {
                 srcColumnName = targetColumnName;
             }
-            sql.append(targetColumnName).append("=? AND ");
+            sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND ");
             values.put(targetColumnName, rs.getObject(srcColumnName));
         }
         int len = sql.length();

client-adapter/tablestore/pom.xml

@@ -30,7 +30,6 @@
         <dependency>
             <groupId>com.aliyun.openservices</groupId>
             <artifactId>tablestore</artifactId>
-            <version>5.10.3</version>
             <classifier>jar-with-dependencies</classifier>
             <exclusions>
                 <exclusion>
@@ -43,8 +42,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
-
     </dependencies>
 
     <build>

client/pom.xml

@@ -101,42 +101,44 @@
 		<dependency>
 			<groupId>org.apache.rocketmq</groupId>
 			<artifactId>rocketmq-client</artifactId>
-			<version>4.5.2</version>
+			<version>${rocketmq_version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.rocketmq</groupId>
 			<artifactId>rocketmq-acl</artifactId>
-			<version>4.5.2</version>
+			<version>${rocketmq_version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>com.rabbitmq</groupId>
 			<artifactId>amqp-client</artifactId>
-			<version>5.5.0</version>
+			<version>${rabbitmq_version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>com.alibaba.mq-amqp</groupId>
 			<artifactId>mq-amqp-client</artifactId>
-			<version>1.0.3</version>
+			<version>${mq_amqp_client}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.kafka</groupId>
 			<artifactId>kafka-clients</artifactId>
-			<version>1.1.1</version>
+			<version>${kafka_version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<!-- Pulsar -->
 		<dependency>
 			<groupId>org.apache.pulsar</groupId>
 			<artifactId>pulsar-client</artifactId>
+			<version>${pulsar_version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.pulsar</groupId>
 			<artifactId>pulsar-client-admin</artifactId>
+			<version>${pulsar_version}</version>
 			<scope>provided</scope>
 		</dependency>
 

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.client.kafka;
 
+import com.alibaba.fastjson.JSON;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
@@ -10,7 +11,6 @@ import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.StringDeserializer;
 
-import com.alibaba.fastjson2.JSON;
 import com.alibaba.otter.canal.client.CanalMQConnector;
 import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
 import com.alibaba.otter.canal.protocol.FlatMessage;
@@ -43,7 +43,7 @@ public class KafkaCanalConnector implements CanalMQConnector {
     private Map<Integer, Long>               currentOffsets = new ConcurrentHashMap<>();
 
     public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
-                               boolean flatMessage){
+        boolean flatMessage){
         this.topic = topic;
         this.partition = partition;
         this.flatMessage = flatMessage;
@@ -61,11 +61,11 @@ public class KafkaCanalConnector implements CanalMQConnector {
             batchSize = 100;
         }
         properties.put("max.poll.records", batchSize.toString());
-        properties.put("key.deserializer", StringDeserializer.class.getName());
+        properties.put("key.deserializer", StringDeserializer.class);
         if (!flatMessage) {
-            properties.put("value.deserializer", MessageDeserializer.class.getName());
+            properties.put("value.deserializer", MessageDeserializer.class);
         } else {
-            properties.put("value.deserializer", StringDeserializer.class.getName());
+            properties.put("value.deserializer", StringDeserializer.class);
         }
     }
 
@@ -183,14 +183,13 @@ public class KafkaCanalConnector implements CanalMQConnector {
 
         ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
 
-        currentOffsets.clear();
-        for (TopicPartition topicPartition : records.partitions()) {
-            currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
-        }
-
         if (!records.isEmpty()) {
+            currentOffsets.clear();
             List<Message> messages = new ArrayList<>();
             for (ConsumerRecord<String, Message> record : records) {
+                if (currentOffsets.get(record.partition()) == null) {
+                    currentOffsets.put(record.partition(), record.offset());
+                }
                 messages.add(record.value());
             }
             return messages;
@@ -221,14 +220,13 @@ public class KafkaCanalConnector implements CanalMQConnector {
 
         ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
 
-        currentOffsets.clear();
-        for (TopicPartition topicPartition : records.partitions()) {
-            currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
-        }
-
         if (!records.isEmpty()) {
+            currentOffsets.clear();
             List<FlatMessage> flatMessages = new ArrayList<>();
             for (ConsumerRecord<String, String> record : records) {
+                if (currentOffsets.get(record.partition()) == null) {
+                    currentOffsets.put(record.partition(), record.offset());
+                }
                 String flatMessageJson = record.value();
                 FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
                 flatMessages.add(flatMessage);
@@ -248,12 +246,14 @@ public class KafkaCanalConnector implements CanalMQConnector {
         // 回滚所有分区
         if (kafkaConsumer != null) {
             for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
-                kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
+                kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey()));
+                kafkaConsumer.commitSync();
             }
         }
         if (kafkaConsumer2 != null) {
             for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
-                kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
+                kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey()));
+                kafkaConsumer.commitSync();
             }
         }
     }
@@ -324,3 +324,4 @@ public class KafkaCanalConnector implements CanalMQConnector {
     }
 
 }
+

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java

@@ -38,7 +38,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
     private ServerRunningData          serverData;
     // 当前实际运行的节点状态信息
     private volatile ServerRunningData activeData;
-    private ScheduledExecutorService   delayExector = Executors.newScheduledThreadPool(1);
+    private ScheduledExecutorService   delayExecutor;
     private int                        delayTime    = 5;
     private ServerRunningListener      listener;
 
@@ -73,7 +73,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
                     initRunning();
                 } else {
                     // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
-                    delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
+                    delayExecutor.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
                 }
             }
 
@@ -90,6 +90,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
         try {
             processStart();
             if (zkClient != null) {
+                delayExecutor = Executors.newScheduledThreadPool(1);
                 // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
                 String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
                 zkClient.subscribeDataChanges(path, dataListener);
@@ -122,6 +123,10 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
         if (zkClient != null) {
             String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
             zkClient.unsubscribeDataChanges(path, dataListener);
+            if (delayExecutor != null) {
+                delayExecutor.shutdown();
+                delayExecutor = null;
+            }
 
             releaseRunning(); // 尝试一下release
         } else {

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java

@@ -107,11 +107,11 @@ public class AviaterRegexFilter {
     private List<String> completionPattern(List<String> patterns) {
         List<String> result = new ArrayList<>();
         for (String pattern : patterns) {
-            StringBuffer stringBuffer = new StringBuffer();
-            stringBuffer.append("^");
-            stringBuffer.append(pattern);
-            stringBuffer.append("$");
-            result.add(stringBuffer.toString());
+            StringBuilder stringBuilder = new StringBuilder();
+            stringBuilder.append("^");
+            stringBuilder.append(pattern);
+            stringBuilder.append("$");
+            result.add(stringBuilder.toString());
         }
         return result;
     }

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java

@@ -374,6 +374,7 @@ public class MQMessageUtils {
             flatMessage.setEs(entry.getHeader().getExecuteTime());
             flatMessage.setTs(System.currentTimeMillis());
             flatMessage.setSql(rowChange.getSql());
+            flatMessage.setGtid(entry.getHeader().getGtid());
 
             if (!rowChange.getIsDdl()) {
                 Map<String, Integer> sqlType = new LinkedHashMap<>();
@@ -432,9 +433,7 @@ public class MQMessageUtils {
                             }
                         }
                         // update操作将记录修改前的值
-                        if (!rowOld.isEmpty()) {
-                            old.add(rowOld);
-                        }
+                        old.add(rowOld);
                     }
                 }
                 if (!sqlType.isEmpty()) {
@@ -526,6 +525,7 @@ public class MQMessageUtils {
                             flatMessageTmp.setEs(flatMessage.getEs());
                             flatMessageTmp.setTs(flatMessage.getTs());
                             flatMessageTmp.setPkNames(flatMessage.getPkNames());
+                            flatMessageTmp.setGtid(flatMessage.getGtid());
                         }
                         List<Map<String, String>> data = flatMessageTmp.getData();
                         if (data == null) {

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ProxyCanalMQProducer.java

@@ -0,0 +1,66 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.protocol.Message;
+import java.util.Properties;
+
+public class ProxyCanalMQProducer implements CanalMQProducer {
+
+    private CanalMQProducer canalMQProducer;
+
+    public ProxyCanalMQProducer(CanalMQProducer canalMQProducer) {
+        this.canalMQProducer = canalMQProducer;
+    }
+
+    private ClassLoader changeCL() {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
+        return cl;
+    }
+
+    private void revertCL(ClassLoader cl) {
+        Thread.currentThread().setContextClassLoader(cl);
+    }
+
+    @Override
+    public void init(Properties properties) {
+        ClassLoader cl = changeCL();
+        try {
+            canalMQProducer.init(properties);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public MQProperties getMqProperties() {
+        ClassLoader cl = changeCL();
+        try {
+            return canalMQProducer.getMqProperties();
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void send(MQDestination canalDestination, Message message, Callback callback) {
+        ClassLoader cl = changeCL();
+        try {
+            canalMQProducer.send(canalDestination, message, callback);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void stop() {
+        ClassLoader cl = changeCL();
+        try {
+            canalMQProducer.stop();
+        } finally {
+            revertCL(cl);
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ProxyCanalMsgConsumer.java

@@ -0,0 +1,86 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+public class ProxyCanalMsgConsumer implements CanalMsgConsumer {
+
+    private CanalMsgConsumer canalMsgConsumer;
+
+    public ProxyCanalMsgConsumer(CanalMsgConsumer canalMsgConsumer) {
+        this.canalMsgConsumer = canalMsgConsumer;
+    }
+
+    private ClassLoader changeCL() {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
+        return cl;
+    }
+
+    private void revertCL(ClassLoader cl) {
+        Thread.currentThread().setContextClassLoader(cl);
+    }
+
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        ClassLoader cl = changeCL();
+        try {
+            canalMsgConsumer.init(properties, topic, groupId);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void connect() {
+        ClassLoader cl = changeCL();
+        try {
+            canalMsgConsumer.connect();
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        ClassLoader cl = changeCL();
+        try {
+            return canalMsgConsumer.getMessage(timeout, unit);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void ack() {
+        ClassLoader cl = changeCL();
+        try {
+            canalMsgConsumer.ack();
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void rollback() {
+        ClassLoader cl = changeCL();
+        try {
+            canalMsgConsumer.rollback();
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        ClassLoader cl = changeCL();
+        try {
+            canalMsgConsumer.disconnect();
+        } finally {
+            revertCL(cl);
+        }
+    }
+}

connector/kafka-connector/pom.xml

@@ -28,8 +28,8 @@
         </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>1.1.1</version>
+            <artifactId>kafka-clients</artifactId>
+            <version>${kafka_version}</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -89,4 +89,4 @@
         </plugins>
     </build>
 
-</project>
+</project>

connector/pom.xml

@@ -39,7 +39,7 @@
     <repositories>
         <repository>
             <id>central</id>
-            <url>http://repo1.maven.org/maven2</url>
+            <url>https://repo1.maven.org/maven2</url>
             <releases>
                 <enabled>true</enabled>
             </releases>
@@ -49,7 +49,7 @@
         </repository>
         <repository>
             <id>java.net</id>
-            <url>http://download.java.net/maven/2/</url>
+            <url>https://download.java.net/maven/2/</url>
             <releases>
                 <enabled>true</enabled>
             </releases>
@@ -59,7 +59,7 @@
         </repository>
         <repository>
             <id>aliyun</id>
-            <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+            <url>https://maven.aliyun.com/nexus/content/groups/public/</url>
             <releases>
                 <enabled>true</enabled>
             </releases>

connector/pulsarmq-connector/pom.xml

@@ -35,10 +35,12 @@
         <dependency>
             <groupId>org.apache.pulsar</groupId>
             <artifactId>pulsar-client</artifactId>
+            <version>${pulsar_version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.pulsar</groupId>
             <artifactId>pulsar-client-admin</artifactId>
+            <version>${pulsar_version}</version>
         </dependency>
     </dependencies>
 

connector/rabbitmq-connector/pom.xml

@@ -29,12 +29,12 @@
         <dependency>
             <groupId>com.rabbitmq</groupId>
             <artifactId>amqp-client</artifactId>
-            <version>5.5.0</version>
+            <version>${rabbitmq_version}</version>
         </dependency>
         <dependency>
             <groupId>com.alibaba.mq-amqp</groupId>
             <artifactId>mq-amqp-client</artifactId>
-            <version>1.0.3</version>
+            <version>${mq_amqp_client}</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -94,4 +94,4 @@
         </plugins>
     </build>
 
-</project>
+</project>

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java

@@ -15,6 +15,9 @@ public class RabbitMQConstants {
     public static final String RABBITMQ_VIRTUAL_HOST     = ROOT + "." + "virtual.host";
     public static final String RABBITMQ_USERNAME         = ROOT + "." + "username";
     public static final String RABBITMQ_PASSWORD         = ROOT + "." + "password";
+    public static final String RABBITMQ_QUEUE            = ROOT + "." + "queue";
+    public static final String RABBITMQ_ROUTING_KEY      = ROOT + "." + "routingKey";
+    public static final String RABBITMQ_DELIVERY_MODE    = ROOT + "." + "deliveryMode";
 
     public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId";
 }

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java

@@ -15,6 +15,9 @@ public class RabbitMQProducerConfig extends MQProperties {
     private String exchange;
     private String username;
     private String password;
+    private String queue;
+    private String routingKey;
+    private String deliveryMode;
 
     public String getHost() {
         return host;
@@ -55,4 +58,28 @@ public class RabbitMQProducerConfig extends MQProperties {
     public void setPassword(String password) {
         this.password = password;
     }
+
+    public String getQueue() {
+        return queue;
+    }
+
+    public void setQueue(String queue) {
+        this.queue = queue;
+    }
+
+    public String getRoutingKey() {
+        return routingKey;
+    }
+
+    public void setRoutingKey(String routingKey) {
+        this.routingKey = routingKey;
+    }
+
+    public String getDeliveryMode() {
+        return deliveryMode;
+    }
+
+    public void setDeliveryMode(String deliveryMode) {
+        this.deliveryMode = deliveryMode;
+    }
 }

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java

@@ -9,11 +9,11 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import com.alibaba.otter.canal.common.utils.PropertiesUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson2.JSON;
+import com.alibaba.otter.canal.common.utils.PropertiesUtils;
 import com.alibaba.otter.canal.connector.core.config.CanalConstants;
 import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
@@ -24,13 +24,7 @@ import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
 import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
-import com.rabbitmq.client.AMQP;
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-import com.rabbitmq.client.Consumer;
-import com.rabbitmq.client.DefaultConsumer;
-import com.rabbitmq.client.Envelope;
+import com.rabbitmq.client.*;
 
 /**
  * RabbitMQ consumer SPI 实现
@@ -41,7 +35,8 @@ import com.rabbitmq.client.Envelope;
 @SPI("rabbitmq")
 public class CanalRabbitMQConsumer implements CanalMsgConsumer {
 
-    private static final Logger                                logger              = LoggerFactory.getLogger(CanalRabbitMQConsumer.class);
+    private static final Logger                                logger              = LoggerFactory
+        .getLogger(CanalRabbitMQConsumer.class);
 
     // 链接地址
     private String                                             nameServer;
@@ -91,7 +86,7 @@ public class CanalRabbitMQConsumer implements CanalMsgConsumer {
             factory.setUsername(username);
             factory.setPassword(password);
         }
-        //解析出端口 modified by 16075140
+        // 解析出端口 modified by 16075140
         if (nameServer != null && nameServer.contains(":")) {
             String[] serverHostAndPort = nameServer.split(":");
             factory.setHost(serverHostAndPort[0]);

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java

@@ -26,10 +26,7 @@ import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
 import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
-import com.rabbitmq.client.AlreadyClosedException;
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.*;
 
 /**
  * RabbitMQ Producer SPI 实现
@@ -75,7 +72,13 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
         try {
             connect = factory.newConnection();
             channel = connect.createChannel();
-            // channel.exchangeDeclare(mqProperties.getExchange(), "topic");
+            channel.queueDeclare(rabbitMQProperties.getQueue(), true, false, false, null);
+            channel.exchangeDeclare(rabbitMQProperties
+                .getExchange(), rabbitMQProperties.getDeliveryMode(), true, false, false, null);
+            channel.queueBind(rabbitMQProperties.getQueue(),
+                rabbitMQProperties.getExchange(),
+                rabbitMQProperties.getRoutingKey());
+
         } catch (IOException | TimeoutException ex) {
             throw new CanalException("Start RabbitMQ producer error", ex);
         }
@@ -106,6 +109,18 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
         if (!StringUtils.isEmpty(password)) {
             rabbitMQProperties.setPassword(password);
         }
+        String queue = PropertiesUtils.getProperty(properties, RabbitMQConstants.RABBITMQ_QUEUE);
+        if (!StringUtils.isEmpty(queue)) {
+            rabbitMQProperties.setQueue(queue);
+        }
+        String routingKey = PropertiesUtils.getProperty(properties, RabbitMQConstants.RABBITMQ_ROUTING_KEY);
+        if (!StringUtils.isEmpty(routingKey)) {
+            rabbitMQProperties.setRoutingKey(routingKey);
+        }
+        String deliveryMode = PropertiesUtils.getProperty(properties, RabbitMQConstants.RABBITMQ_DELIVERY_MODE);
+        if (!StringUtils.isEmpty(deliveryMode)) {
+            rabbitMQProperties.setDeliveryMode(deliveryMode);
+        }
     }
 
     @Override
@@ -114,9 +129,8 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
         try {
             if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
                 // 动态topic
-                Map<String, Message> messageMap = MQMessageUtils.messageTopics(message,
-                    destination.getTopic(),
-                    destination.getDynamicTopic());
+                Map<String, Message> messageMap = MQMessageUtils
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
 
                 for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
                     final String topicName = entry.getKey().replace('.', '_');
@@ -165,7 +179,10 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
         // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
         try {
             RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
-            channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message);
+            channel.basicPublish(rabbitMQProperties.getExchange(),
+                queueName,
+                MessageProperties.PERSISTENT_TEXT_PLAIN,
+                message);
         } catch (Throwable e) {
             throw new RuntimeException(e);
         }

connector/rocketmq-connector/pom.xml

@@ -13,10 +13,6 @@
     <packaging>jar</packaging>
     <name>canal connector rocketMQ module for otter ${project.version}</name>
 
-    <properties>
-        <rocketmq_version>4.8.0</rocketmq_version>
-    </properties>
-
     <dependencies>
         <dependency>
             <groupId>com.alibaba.otter</groupId>
@@ -99,4 +95,4 @@
         </plugins>
     </build>
 
-</project>
+</project>

connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java

@@ -1,5 +1,13 @@
 package com.alibaba.otter.canal.connector.tcp.consumer;
 
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+
 import com.alibaba.otter.canal.client.CanalConnector;
 import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
 import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
@@ -12,14 +20,6 @@ import com.alibaba.otter.canal.connector.core.util.MessageUtil;
 import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
 import com.alibaba.otter.canal.protocol.Message;
 
-import org.apache.commons.lang.StringUtils;
-
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
 /**
  * TCP 消费者连接器, 一个destination对应一个SPI实例
  *
@@ -99,7 +99,9 @@ public class CanalTCPConsumer implements CanalMsgConsumer {
 
     @Override
     public void disconnect() {
-        canalConnector.unsubscribe();
+        // tcp模式下，因为是单tcp消费，避免adapter异常断开时直接unsubscribe
+        // unsubscribe发送给canal-server会导致清理cursor位点,如果此时canal-server出现重启,就会丢失binlog数据
+        // canalConnector.unsubscribe();
         canalConnector.disconnect();
     }
 }

dbsync/pom.xml

@@ -33,6 +33,14 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-compress</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.github.luben</groupId>
+			<artifactId>zstd-jni</artifactId>
+		</dependency>
 		<!-- test dependency -->
 		<dependency>
 			<groupId>junit</groupId>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java

@@ -21,6 +21,7 @@ public final class CharsetConversion {
         protected final String mysqlCharset;
         protected final String mysqlCollation;
         protected final String javaCharset;
+        protected final Charset charset;
 
         Entry(final int id, String mysqlCharset, // NL
               String mysqlCollation, String javaCharset){
@@ -28,6 +29,7 @@ public final class CharsetConversion {
             this.mysqlCharset = mysqlCharset;
             this.mysqlCollation = mysqlCollation;
             this.javaCharset = javaCharset;
+            this.charset = Charset.isSupported(javaCharset) ? Charset.forName(javaCharset) : null;
         }
     }
 
@@ -390,6 +392,23 @@ public final class CharsetConversion {
         }
     }
 
+    public static Charset getNioCharset(final int id) {
+        Entry entry = getEntry(id);
+
+        if (entry != null) {
+            if (entry.charset != null) {
+                return entry.charset;
+            } else {
+                logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
+                            + entry.mysqlCollation);
+                return null;
+            }
+        } else {
+            logger.warn("Unexpect mysql charset: " + id);
+            return null;
+        }
+    }
+
     public static void main(String[] args) {
         for (int i = 0; i < entries.length; i++) {
             Entry entry = entries[i];

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java

@@ -1,5 +1,7 @@
 package com.taobao.tddl.dbsync.binlog;
 
+import java.nio.charset.Charset;
+
 import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber2;
 import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.appendNumber4;
 import static com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer.usecondsToStr;
@@ -57,23 +59,27 @@ public class JsonConversion {
     public static final int  VALUE_ENTRY_SIZE_LARGE  = (1 + LARGE_OFFSET_SIZE);
 
     public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
+        return parse_value(type, buffer, len, Charset.forName(charsetName));
+    }
+
+    public static Json_Value parse_value(int type, LogBuffer buffer, long len, Charset charset) {
         buffer = buffer.duplicate(buffer.position(), (int) len);
         switch (type) {
             case JSONB_TYPE_SMALL_OBJECT:
-                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charset);
             case JSONB_TYPE_LARGE_OBJECT:
-                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charset);
             case JSONB_TYPE_SMALL_ARRAY:
-                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charset);
             case JSONB_TYPE_LARGE_ARRAY:
-                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charset);
             default:
-                return parse_scalar(type, buffer, len, charsetName);
+                return parse_scalar(type, buffer, len, charset);
         }
     }
 
     private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
-                                                    String charsetName) {
+                                                    Charset charset) {
         long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
         if (len < 2 * offset_size) {
             throw new IllegalArgumentException("illegal json data");
@@ -100,7 +106,7 @@ public class JsonConversion {
         return large ? buffer.getUint32() : buffer.getUint16();
     }
 
-    private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
+    private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, Charset charset) {
         switch (type) {
             case JSONB_TYPE_LITERAL:
                 /* purecov: inspected */
@@ -156,7 +162,7 @@ public class JsonConversion {
                 }
                 return new Json_Value(Json_enum_type.STRING, buffer.rewind()
                     .forward((int) n)
-                    .getFixString((int) str_len, charsetName));
+                    .getFixString((int) str_len, charset));
             case JSONB_TYPE_OPAQUE:
                 /*
                  * There should always be at least one byte, which tells the
@@ -242,7 +248,7 @@ public class JsonConversion {
             this.m_large = large;
         }
 
-        public String key(int i, String charsetName) {
+        public String key(int i, Charset charset) {
             m_data.rewind();
             int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
             int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
@@ -255,10 +261,10 @@ public class JsonConversion {
             // entry, always two
             // bytes.
             long key_length = m_data.getUint16();
-            return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
+            return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charset);
         }
 
-        public Json_Value element(int i, String charsetName) {
+        public Json_Value element(int i, Charset charset) {
             m_data.rewind();
             int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
             int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
@@ -271,13 +277,13 @@ public class JsonConversion {
             int type = m_data.forward(entry_offset).getUint8();
             if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
                 || (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
-                return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
+                return parse_scalar(type, m_data, value_entry_size - 1, charset);
             }
             int value_offset = (int) read_offset_or_size(m_data, m_large);
-            return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
+            return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charset);
         }
 
-        public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
+        public StringBuilder toJsonString(StringBuilder buf, Charset charset) {
             switch (m_type) {
                 case OBJECT:
                     buf.append("{");
@@ -285,9 +291,9 @@ public class JsonConversion {
                         if (i > 0) {
                             buf.append(", ");
                         }
-                        buf.append('"').append(key(i, charsetName)).append('"');
+                        buf.append('"').append(key(i, charset)).append('"');
                         buf.append(": ");
-                        element(i, charsetName).toJsonString(buf, charsetName);
+                        element(i, charset).toJsonString(buf, charset);
                     }
                     buf.append("}");
                     break;
@@ -297,7 +303,7 @@ public class JsonConversion {
                         if (i > 0) {
                             buf.append(", ");
                         }
-                        element(i, charsetName).toJsonString(buf, charsetName);
+                        element(i, charset).toJsonString(buf, charset);
                     }
                     buf.append("]");
                     break;
@@ -398,7 +404,7 @@ public class JsonConversion {
                         }
                         buf.append('"').append(text).append('"');
                     } else {
-                        text = m_data.getFixString((int) m_length, charsetName);
+                        text = m_data.getFixString((int) m_length, charset);
                         buf.append('"').append(escapse(text)).append('"');
                     }
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java

@@ -1,5 +1,6 @@
 package com.taobao.tddl.dbsync.binlog;
 
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -36,6 +37,11 @@ public class JsonDiffConversion {
 
     public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
                                                 String charsetName) {
+        return print_json_diff(buffer, len, columnName, columnIndex, Charset.forName(charsetName));
+    }
+
+    public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
+                                                Charset charset) {
         int position = buffer.position();
         List<String> operation_names = new ArrayList<>();
         while (buffer.hasRemaining()) {
@@ -111,14 +117,14 @@ public class JsonDiffConversion {
                 Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
                     buffer,
                     value_length - 1,
-                    charsetName);
+                    charset);
                 buffer.forward((int) value_length - 1);
                 // Read value
                 if (jsonValue.m_type == Json_enum_type.ERROR) {
                     throw new IllegalArgumentException("parsing json value");
                 }
                 StringBuilder jsonBuilder = new StringBuilder();
-                jsonValue.toJsonString(jsonBuilder, charsetName);
+                jsonValue.toJsonString(jsonBuilder, charset);
                 builder.append(jsonBuilder);
             }
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java

@@ -26,6 +26,8 @@ public final class LogContext {
 
     private LogEvent                          gtidLogEvent; // save current gtid log event
 
+    private boolean                           iterateDecode = false;
+
     public LogContext(){
         this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
     }
@@ -82,4 +84,12 @@ public final class LogContext {
     public void setGtidLogEvent(LogEvent gtidLogEvent) {
         this.gtidLogEvent = gtidLogEvent;
     }
+
+    public boolean isIterateDecode() {
+        return iterateDecode;
+    }
+
+    public void setIterateDecode(boolean iterateDecode) {
+        this.iterateDecode = iterateDecode;
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -1,18 +1,19 @@
 package com.taobao.tddl.dbsync.binlog;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.BitSet;
+import java.util.List;
 
-import com.taobao.tddl.dbsync.binlog.event.*;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
+import org.apache.commons.compress.utils.Lists;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
-import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
-import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.mariadb.StartEncryptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.*;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.*;
 
 /**
  * Implements a binary-log decoder.
@@ -56,6 +57,8 @@ public final class LogDecoder {
         handleSet.set(flagIndex);
     }
 
+    private LogBuffer compressIterateBuffer;
+
     /**
      * Decoding an event from binary-log buffer.
      *
@@ -64,7 +67,6 @@ public final class LogDecoder {
      */
     public LogEvent decode(LogBuffer buffer, LogContext context) throws IOException {
         final int limit = buffer.limit();
-
         if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
             LogHeader header = new LogHeader(buffer, context.getFormatDescription());
 
@@ -109,6 +111,59 @@ public final class LogDecoder {
         return null;
     }
 
+    /**
+     * * process compress binlog payload
+     * 
+     * @param event
+     * @param context
+     * @return
+     * @throws IOException
+     */
+    public List<LogEvent> processIterateDecode(LogEvent event, LogContext context) throws IOException {
+        List<LogEvent> events = Lists.newArrayList();
+        if (event.getHeader().getType() == LogEvent.TRANSACTION_PAYLOAD_EVENT) {
+            // iterate for compresss payload
+            TransactionPayloadLogEvent compressEvent = ((TransactionPayloadLogEvent) event);
+            LogBuffer iterateBuffer = null;
+            if (compressEvent.isCompressByZstd()) {
+                try (ZstdCompressorInputStream in = new ZstdCompressorInputStream(
+                    new ByteArrayInputStream(compressEvent.getPayload()))) {
+                    byte[] decodeBytes = IOUtils.toByteArray(in);
+                    iterateBuffer = new LogBuffer(decodeBytes, 0, decodeBytes.length);
+                }
+            } else if (compressEvent.isCompressByNone()) {
+                iterateBuffer = new LogBuffer(compressEvent.getPayload(), 0, compressEvent.getPayload().length);
+            } else {
+                throw new IllegalArgumentException("unknow compress type for " + event.getHeader().getLogFileName()
+                                                   + ":" + event.getHeader().getLogPos());
+            }
+
+            try {
+                context.setIterateDecode(true);
+                while (iterateBuffer.hasRemaining()) {// iterate
+                    LogEvent deEvent = decode(iterateBuffer, context);
+                    if (deEvent == null) {
+                        break;
+                    }
+
+                    // compress event logPos = 0
+                    deEvent.getHeader().setLogFileName(event.getHeader().getLogFileName());
+                    deEvent.getHeader().setLogPos(event.getHeader().getLogPos());
+                    // 需要重置payload每个event的eventLen , ack位点更新依赖logPos - eventLen,
+                    // 原因:每个payload都是uncompress的eventLen,无法对应物理binlog的eventLen
+                    // 隐患:memory计算空间大小时会出现放大的情况,影响getBatch的数量
+                    deEvent.getHeader().setEventLen(event.getHeader().getEventLen());
+                    events.add(deEvent);
+                }
+            } finally {
+                context.setIterateDecode(false);
+            }
+        } else {
+            // TODO support mariadb compress binlog
+        }
+        return events;
+    }
+
     /**
      * Deserialize an event from buffer.
      *
@@ -127,8 +182,12 @@ public final class LogDecoder {
         }
 
         if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
-            // remove checksum bytes
-            buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
+            if (context.isIterateDecode()) {
+                // transaction compress payload在主事件已经处理了checksum,遍历解析event忽略checksum处理
+            } else {
+                // remove checksum bytes
+                buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
+            }
         }
         GTIDSet gtidSet = context.getGtidSet();
         LogEvent gtidLogEvent = context.getGtidLogEvent();
@@ -360,16 +419,10 @@ public final class LogDecoder {
                 return event;
             }
             case LogEvent.TRANSACTION_PAYLOAD_EVENT: {
-                if (logger.isWarnEnabled()) {
-                    logger.warn("Skipping unsupported MySQL TRANSACTION_PAYLOAD_EVENT from: " + context.getLogPosition());
-                }
-                break;
-
-                // TransactionPayloadLogEvent event = new TransactionPayloadLogEvent(header,
-                // buffer, descriptionEvent);
-                // /* updating position in context */
-                // logPosition.position = header.getLogPos();
-                // return event;
+                TransactionPayloadLogEvent event = new TransactionPayloadLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
             }
             case LogEvent.VIEW_CHANGE_EVENT: {
                 ViewChangeEvent event = new ViewChangeEvent(header, buffer, descriptionEvent);
@@ -394,6 +447,7 @@ public final class LogDecoder {
                 BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                logPosition.fileName = event.getFilename();
                 return event;
             }
             case LogEvent.GTID_EVENT: {
@@ -435,31 +489,37 @@ public final class LogDecoder {
                 return event;
             }
             case LogEvent.QUERY_COMPRESSED_EVENT: {
-                if (logger.isWarnEnabled()) {
-                    logger.warn("Skipping unsupported MaraiDB QUERY_COMPRESSED_EVENT from: " + context.getLogPosition());
-                }
-                break;
+                QueryCompressedLogEvent event = new QueryCompressedLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
             }
             case LogEvent.WRITE_ROWS_COMPRESSED_EVENT_V1:
             case LogEvent.WRITE_ROWS_COMPRESSED_EVENT: {
-                if (logger.isWarnEnabled()) {
-                    logger.warn("Skipping unsupported MaraiDB WRITE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition());
-                }
-                break;
+                WriteRowsCompressLogEvent event = new WriteRowsCompressLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
+                return event;
             }
             case LogEvent.UPDATE_ROWS_COMPRESSED_EVENT_V1:
             case LogEvent.UPDATE_ROWS_COMPRESSED_EVENT: {
-                if (logger.isWarnEnabled()) {
-                    logger.warn("Skipping unsupported MaraiDB UPDATE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition());
-                }
-                break;
+                UpdateRowsCompressLogEvent event = new UpdateRowsCompressLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
+                return event;
             }
             case LogEvent.DELETE_ROWS_COMPRESSED_EVENT_V1:
             case LogEvent.DELETE_ROWS_COMPRESSED_EVENT: {
-                if (logger.isWarnEnabled()) {
-                    logger.warn("Skipping unsupported MaraiDB DELETE_ROWS_COMPRESSED_EVENT from: " + context.getLogPosition());
-                }
-                break;
+                DeleteRowsCompressLogEvent event = new DeleteRowsCompressLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
+                return event;
             }
             default:
                 /*

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java

@@ -9,9 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteRowsLogEvent extends RowsLogEvent {
+public class DeleteRowsLogEvent extends RowsLogEvent {
 
     public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-        super(header, buffer, descriptionEvent);
+        super(header, buffer, descriptionEvent, false, false);
+    }
+
+    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                              boolean compress){
+        super(header, buffer, descriptionEvent, false, compress);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java

@@ -62,6 +62,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
     public static final int   TRANSACTION_CONTEXT_HEADER_LEN      = 18;
     public static final int   VIEW_CHANGE_HEADER_LEN              = 52;
     public static final int   XA_PREPARE_HEADER_LEN               = 0;
+    public static final int   TRANSACTION_PAYLOAD_HEADER_LEN      = 0;
 
     public static final int   ANNOTATE_ROWS_HEADER_LEN            = 0;
     public static final int   BINLOG_CHECKPOINT_HEADER_LEN        = 4;
@@ -113,14 +114,14 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         long calc = getVersionProduct();
         if (calc >= checksumVersionProduct) {
             /*
-             * the last bytes are the checksum alg desc and value (or value's
-             * room)
+             * the last bytes are the checksum alg desc and value (or value's room)
              */
             numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
         }
 
-        if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= "
-                                                + numberOfEventTypes);
+        if (logger.isInfoEnabled()) {
+            logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= " + numberOfEventTypes);
+        }
     }
 
     /** MySQL 5.0 format descriptions. */
@@ -212,6 +213,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[VIEW_CHANGE_EVENT - 1] = VIEW_CHANGE_HEADER_LEN;
                 postHeaderLen[XA_PREPARE_LOG_EVENT - 1] = XA_PREPARE_HEADER_LEN;
                 postHeaderLen[PARTIAL_UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[TRANSACTION_PAYLOAD_EVENT - 1] = TRANSACTION_PAYLOAD_HEADER_LEN;
 
                 // mariadb 10
                 postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;
@@ -219,6 +221,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[GTID_EVENT - 1] = GTID_HEADER_LEN;
                 postHeaderLen[GTID_LIST_EVENT - 1] = GTID_LIST_HEADER_LEN;
                 postHeaderLen[START_ENCRYPTION_EVENT - 1] = START_ENCRYPTION_HEADER_LEN;
+
+                // mariadb compress
+                postHeaderLen[QUERY_COMPRESSED_EVENT - 1] = QUERY_COMPRESSED_EVENT;
+                postHeaderLen[WRITE_ROWS_COMPRESSED_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[UPDATE_ROWS_COMPRESSED_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[DELETE_ROWS_COMPRESSED_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[WRITE_ROWS_COMPRESSED_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
+                postHeaderLen[UPDATE_ROWS_COMPRESSED_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
+                postHeaderLen[DELETE_ROWS_COMPRESSED_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 break;
 
             case 3: /* 4.0.x x>=2 */

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java

@@ -18,12 +18,13 @@ public class GtidLogEvent extends LogEvent {
     // / Length of SID in event encoding
     public static final int ENCODED_SID_LENGTH          = 16;
     public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2;
+    public static final UUID UUID_ZERO                   = new UUID(0, 0);
 
     private boolean         commitFlag;
     private UUID            sid;
     private long            gno;
-    private Long            lastCommitted;
-    private Long            sequenceNumber;
+    private long            lastCommitted;
+    private long            sequenceNumber;
 
     public GtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
@@ -35,11 +36,13 @@ public class GtidLogEvent extends LogEvent {
         buffer.position(commonHeaderLen);
         commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH
 
-        byte[] bs = buffer.getData(ENCODED_SID_LENGTH);
-        ByteBuffer bb = ByteBuffer.wrap(bs);
-        long high = bb.getLong();
-        long low = bb.getLong();
-        sid = new UUID(high, low);
+        long high = buffer.getBeLong64();
+        long low = buffer.getBeLong64();
+        if (high == 0 && low == 0) {
+            sid = UUID_ZERO;
+        } else {
+            sid = new UUID(high, low);
+        }
 
         gno = buffer.getLong64();
 
@@ -73,11 +76,11 @@ public class GtidLogEvent extends LogEvent {
         return gno;
     }
 
-    public Long getLastCommitted() {
+    public long getLastCommitted() {
         return lastCommitted;
     }
 
-    public Long getSequenceNumber() {
+    public long getSequenceNumber() {
         return sequenceNumber;
     }
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java

@@ -3,6 +3,8 @@ package com.taobao.tddl.dbsync.binlog.event;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
+import java.nio.charset.StandardCharsets;
+
 /**
  * <pre>
  * Replication event to ensure to slave that master is alive.
@@ -35,7 +37,7 @@ public class HeartbeatLogEvent extends LogEvent {
             identLen = FN_REFLEN - 1;
         }
 
-        logIdent = buffer.getFullString(commonHeaderLen, identLen, LogBuffer.ISO_8859_1);
+        logIdent = buffer.getFullString(commonHeaderLen, identLen, StandardCharsets.ISO_8859_1);
     }
 
     public int getIdentLen() {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java

@@ -67,7 +67,7 @@ public final class IncidentLogEvent extends LogEvent {
         incident = incidentNumber;
 
         buffer.position(commonHeaderLen + postHeaderLen);
-        message = buffer.getString();
+        message = buffer.getName();
     }
 
     public final int getIncident() {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java

@@ -1,12 +1,12 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
-import java.util.HashMap;
-import java.util.Map;
-
 /**
  * The Common-Header, documented in the table @ref Table_common_header "below",
  * always has the same form and length within one version of MySQL. Each event
@@ -66,7 +66,7 @@ import java.util.Map;
  */
 public final class LogHeader {
 
-    protected final int type;
+    protected int                 type;
 
     /**
      * The offset in the log where this event originally appeared (it is
@@ -293,6 +293,12 @@ public final class LogHeader {
     public void setLogFileName(String logFileName) {
         this.logFileName = logFileName;
     }
+    public void setLogPos(long logPos) {
+        this.logPos = logPos;
+    }
+    public void setEventLen(int eventLen) {
+        this.eventLen = eventLen;
+    }
 
     private void processCheckSum(LogBuffer buffer) {
         if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
@@ -320,7 +326,7 @@ public final class LogHeader {
         if (gtidSet != null) {
             gtidMap.put(GTID_SET_STRING, gtidSet.toString());
             if (gtidEvent != null && gtidEvent instanceof GtidLogEvent) {
-                GtidLogEvent event = (GtidLogEvent)gtidEvent;
+                GtidLogEvent event = (GtidLogEvent) gtidEvent;
                 gtidMap.put(CURRENT_GTID_STRING, event.getGtidStr());
                 gtidMap.put(CURRENT_GTID_SN, String.valueOf(event.getSequenceNumber()));
                 gtidMap.put(CURRENT_GTID_LAST_COMMIT, String.valueOf(event.getLastCommitted()));

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java

@@ -433,14 +433,18 @@ public class QueryLogEvent extends LogEvent {
     private int             serverCollation           = -1;
     private int             tvSec                     = -1;
     private BigInteger      ddlXid                    = BigInteger.valueOf(-1L);
-    private String          charsetName;
+    private Charset         charset;
 
     private String          timezone;
 
     public QueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
-                                                                                                        throws IOException{
-        super(header);
+                                                                                                         throws IOException{
+        this(header, buffer, descriptionEvent, false);
+    }
 
+    public QueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                         boolean compress) throws IOException{
+        super(header);
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
         /*
@@ -495,15 +499,20 @@ public class QueryLogEvent extends LogEvent {
         unpackVariables(buffer, end);
         buffer.position(end);
         buffer.limit(limit);
-
         /* A 2nd variable part; this is common to all versions */
-        final int queryLen = dataLen - dbLen - 1;
-        dbname = buffer.getFixString(dbLen + 1);
+        dbname = buffer.getFixName(dbLen + 1);
+        int queryLen = dataLen - dbLen - 1;
+        if (compress) {
+            // mariadb compress log event
+            // see https://github.com/alibaba/canal/issues/4388
+            buffer = buffer.uncompressBuf();
+            queryLen = buffer.limit();
+        }
         if (clientCharset >= 0) {
-            charsetName = CharsetConversion.getJavaCharset(clientCharset);
+            charset = CharsetConversion.getNioCharset(clientCharset);
 
-            if ((charsetName != null) && (Charset.isSupported(charsetName))) {
-                query = buffer.getFixString(queryLen, charsetName);
+            if (charset != null) {
+                query = buffer.getFixString(queryLen, charset);
             } else {
                 logger.warn("unsupported character set in query log: " + "\n    ID = " + clientCharset + ", Charset = "
                             + CharsetConversion.getCharset(clientCharset) + ", Collation = "
@@ -623,7 +632,7 @@ public class QueryLogEvent extends LogEvent {
                         // is ulonglong
                         break;
                     case Q_CATALOG_NZ_CODE:
-                        catalog = buffer.getString();
+                        catalog = buffer.getName();
                         break;
                     case Q_AUTO_INCREMENT:
                         autoIncrementIncrement = buffer.getUint16();
@@ -639,7 +648,7 @@ public class QueryLogEvent extends LogEvent {
                         serverCollation = buffer.getUint16();
                         break;
                     case Q_TIME_ZONE_CODE:
-                        timezone = buffer.getString();
+                        timezone = buffer.getName();
                         break;
                     case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
                         final int len = buffer.getUint8();
@@ -663,8 +672,8 @@ public class QueryLogEvent extends LogEvent {
                         buffer.forward(4);
                         break;
                     case Q_INVOKER:
-                        user = buffer.getString();
-                        host = buffer.getString();
+                        user = buffer.getName();
+                        host = buffer.getName();
                         break;
                     case Q_MICROSECONDS:
                         // when.tv_usec= uint3korr(pos);
@@ -685,7 +694,7 @@ public class QueryLogEvent extends LogEvent {
                         String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
                         for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
                             int length = end - buffer.position();
-                            mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
+                            mtsAccessedDbNames[i] = buffer.getFixName(length < NAME_LEN ? length : NAME_LEN);
                         }
                         break;
                     case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
@@ -829,8 +838,8 @@ public class QueryLogEvent extends LogEvent {
         return autoIncrementOffset;
     }
 
-    public final String getCharsetName() {
-        return charsetName;
+    public final Charset getCharset() {
+        return charset;
     }
 
     public final String getTimezone() {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java

@@ -88,9 +88,10 @@ public final class RotateLogEvent extends LogEvent {
 
         final int filenameOffset = headerSize + postHeaderLen;
         int filenameLen = buffer.limit() - filenameOffset;
-        if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1;
+        if (filenameLen > FN_REFLEN - 1) {
+            filenameLen = FN_REFLEN - 1;
+        }
         buffer.position(filenameOffset);
-
         filename = buffer.getFixString(filenameLen);
     }
 

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java

@@ -1,8 +1,11 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
 import java.io.Serializable;
+import java.nio.charset.Charset;
 import java.sql.Timestamp;
 import java.sql.Types;
+import java.time.ZoneId;
+import java.time.zone.ZoneRules;
 import java.util.BitSet;
 
 import org.apache.commons.logging.Log;
@@ -23,7 +26,11 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  */
 public final class RowsLogBuffer {
 
-    protected static final Log logger            = LogFactory.getLog(RowsLogBuffer.class);
+    protected static final Log    logger            = LogFactory.getLog(RowsLogBuffer.class);
+    public static final Long[]    longCache         = new Long[1024 * 128];
+    public static final int       longCacheLimit    = longCache.length + 127;
+    public static final Integer[] integerCache      = new Integer[1024 * 128];
+    public static final int       integerCacheLimit = longCache.length + 127;
 
     public static final long   DATETIMEF_INT_OFS = 0x8000000000L;
     public static final long   TIMEF_INT_OFS     = 0x800000L;
@@ -33,7 +40,7 @@ public final class RowsLogBuffer {
     private final LogBuffer    buffer;
     private final int          columnLen;
     private final int          jsonColumnCount;
-    private final String       charsetName;
+    private final Charset      charset;
 
     private final BitSet       nullBits;
     private int                nullBitIndex;
@@ -47,10 +54,10 @@ public final class RowsLogBuffer {
     private int                length;
     private Serializable       value;
 
-    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName, int jsonColumnCount, boolean partial){
+    public RowsLogBuffer(LogBuffer buffer, final int columnLen, Charset charset, int jsonColumnCount, boolean partial){
         this.buffer = buffer;
         this.columnLen = columnLen;
-        this.charsetName = charsetName;
+        this.charset = charset;
         this.partial = partial;
         this.jsonColumnCount = jsonColumnCount;
         this.nullBits = new BitSet(columnLen);
@@ -309,7 +316,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = unsigned ? Long.valueOf(buffer.getUint32()) :
                 // Integer.valueOf(buffer.getInt32());
-                value = Integer.valueOf(buffer.getInt32());
+                value = valueOf(buffer.getInt32());
                 javaType = Types.INTEGER;
                 length = 4;
                 break;
@@ -318,7 +325,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = Integer.valueOf(unsigned ? buffer.getUint8() :
                 // buffer.getInt8());
-                value = Integer.valueOf(buffer.getInt8());
+                value = valueOf(buffer.getInt8());
                 javaType = Types.TINYINT; // java.sql.Types.INTEGER;
                 length = 1;
                 break;
@@ -327,7 +334,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = Integer.valueOf(unsigned ? buffer.getUint16() :
                 // buffer.getInt16());
-                value = Integer.valueOf((short) buffer.getInt16());
+                value = valueOf((short) buffer.getInt16());
                 javaType = Types.SMALLINT; // java.sql.Types.INTEGER;
                 length = 2;
                 break;
@@ -336,7 +343,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = Integer.valueOf(unsigned ? buffer.getUint24() :
                 // buffer.getInt24());
-                value = Integer.valueOf(buffer.getInt24());
+                value = valueOf(buffer.getInt24());
                 javaType = Types.INTEGER;
                 length = 3;
                 break;
@@ -345,7 +352,7 @@ public final class RowsLogBuffer {
                 // XXX: How to check signed / unsigned?
                 // value = unsigned ? buffer.getUlong64()) :
                 // Long.valueOf(buffer.getLong64());
-                value = Long.valueOf(buffer.getLong64());
+                value = valueOf(buffer.getLong64());
                 javaType = Types.BIGINT; // Types.INTEGER;
                 length = 8;
                 break;
@@ -870,7 +877,7 @@ public final class RowsLogBuffer {
                 }
                 // logger.warn("MYSQL_TYPE_ENUM : This enumeration value is "
                 // + "only used internally and cannot exist in a binlog!");
-                value = Integer.valueOf(int32);
+                value = valueOf(int32);
                 javaType = Types.INTEGER;
                 length = len;
                 break;
@@ -1017,7 +1024,7 @@ public final class RowsLogBuffer {
                     javaType = Types.VARBINARY;
                     value = binary;
                 } else {
-                    value = buffer.getFullString(len, charsetName);
+                    value = buffer.getFullString(len, charset);
                     javaType = Types.VARCHAR;
                 }
 
@@ -1039,7 +1046,7 @@ public final class RowsLogBuffer {
                     javaType = Types.BINARY;
                     value = binary;
                 } else {
-                    value = buffer.getFullString(len, charsetName);
+                    value = buffer.getFullString(len, charset);
                     javaType = Types.CHAR; // Types.VARCHAR;
                 }
                 length = len;
@@ -1074,7 +1081,7 @@ public final class RowsLogBuffer {
                         len,
                         columnName,
                         columnIndex,
-                        charsetName);
+                            charset);
                     value = builder.toString();
                     buffer.position(position + len);
                 } else {
@@ -1088,9 +1095,9 @@ public final class RowsLogBuffer {
                         Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(),
                             buffer,
                             len - 1,
-                            charsetName);
+                                charset);
                         StringBuilder builder = new StringBuilder();
-                        jsonValue.toJsonString(builder, charsetName);
+                        jsonValue.toJsonString(builder, charset);
                         value = builder.toString();
                         buffer.position(position + len);
                     }
@@ -1211,4 +1218,38 @@ public final class RowsLogBuffer {
             builder.append('0').append(digits[d]);
         }
     }
+
+    public static Long valueOf(long value) {
+        if (value >= -128 & value <= 127) {
+            // if (l >= -128 && l <= 127) {
+            return Long.valueOf(value);
+        }
+        if (value > 127 && value < longCacheLimit) {
+            int cacheIndex = (int) value - 127;
+            Long cacheValue = longCache[cacheIndex];
+            if (cacheValue == null) {
+                cacheValue = new Long(value);
+                longCache[cacheIndex] = cacheValue;
+            }
+            return cacheValue;
+        }
+        return new Long(value);
+    }
+
+    public static Integer valueOf(int value) {
+        if (value >= -128 & value <= 127) {
+            // if (l >= -128 && l <= 127) {
+            return Integer.valueOf(value);
+        }
+        if (value > 127 && value < integerCacheLimit) {
+            int cacheIndex = (int) value - 127;
+            Integer cacheValue = integerCache[cacheIndex];
+            if (cacheValue == null) {
+                cacheValue = new Integer(value);
+                integerCache[cacheIndex] = cacheValue;
+            }
+            return cacheValue;
+        }
+        return new Integer(value);
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java

@@ -1,12 +1,13 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
+import java.nio.charset.Charset;
 import java.util.BitSet;
 
-import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogContext;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException;
 
 /**
  * Common base class for all row-containing log events.
@@ -117,9 +118,14 @@ public abstract class RowsLogEvent extends LogEvent {
         this(header, buffer, descriptionEvent, false);
     }
 
-    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, boolean partial){
-        super(header);
+    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                        boolean partial){
+        this(header, buffer, descriptionEvent, false, false);
+    }
 
+    public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, boolean partial,
+                        boolean compress){
+        super(header);
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
         int headerLen = 0;
@@ -166,7 +172,8 @@ public abstract class RowsLogEvent extends LogEvent {
         columns = buffer.getBitmap(columnLen);
 
         if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT
-            || header.type == PARTIAL_UPDATE_ROWS_EVENT) {
+            || header.type == PARTIAL_UPDATE_ROWS_EVENT || header.type == UPDATE_ROWS_COMPRESSED_EVENT
+            || header.type == UPDATE_ROWS_COMPRESSED_EVENT_V1) {
             changeColumns = buffer.getBitmap(columnLen);
         } else {
             changeColumns = columns;
@@ -174,6 +181,18 @@ public abstract class RowsLogEvent extends LogEvent {
 
         // XXX: Don't handle buffer in another thread.
         int dataSize = buffer.limit() - buffer.position();
+        if (compress) {
+            // mariadb compress log event
+            // see https://github.com/alibaba/canal/issues/4388
+            buffer = buffer.uncompressBuf();
+            dataSize = buffer.limit();
+            // rewrite type
+            if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2) {
+                header.type = header.type - WRITE_ROWS_COMPRESSED_EVENT + WRITE_ROWS_EVENT;
+            } else {
+                header.type = header.type - WRITE_ROWS_COMPRESSED_EVENT_V1 + WRITE_ROWS_EVENT_V1;
+            }
+        }
         rowsBuf = buffer.duplicate(dataSize);
     }
 
@@ -219,8 +238,8 @@ public abstract class RowsLogEvent extends LogEvent {
         return changeColumns;
     }
 
-    public final RowsLogBuffer getRowsBuf(String charsetName) {
-        return new RowsLogBuffer(rowsBuf, columnLen, charsetName, jsonColumnCount, partial);
+    public final RowsLogBuffer getRowsBuf(Charset charset) {
+        return new RowsLogBuffer(rowsBuf, columnLen, charset, jsonColumnCount, partial);
     }
 
     public final int getFlags(final int flags) {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java

@@ -2,6 +2,8 @@ package com.taobao.tddl.dbsync.binlog.event;
 
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
+import java.nio.charset.StandardCharsets;
+
 /**
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3
@@ -23,7 +25,7 @@ public class RowsQueryLogEvent extends IgnorableLogEvent {
          */
         int offset = commonHeaderLen + postHeaderLen + 1;
         int len = buffer.limit() - offset;
-        rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
+        rowsQuery = buffer.getFullString(offset, len, StandardCharsets.ISO_8859_1);
     }
 
     public String getRowsQuery() {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java

@@ -36,7 +36,7 @@ public class StartLogEventV3 extends LogEvent {
 
         buffer.position(descriptionEvent.commonHeaderLen);
         binlogVersion = buffer.getUint16(); // ST_BINLOG_VER_OFFSET
-        serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
+        serverVersion = buffer.getFixName(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
     }
 
     public StartLogEventV3(){

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java

@@ -423,10 +423,10 @@ public final class TableMapLogEvent extends LogEvent {
 
         /* Read the variable part of the event */
         buffer.position(commonHeaderLen + postHeaderLen);
-        dbname = buffer.getString();
+        dbname = buffer.getName();
         buffer.forward(1); /* termination null */
         // fixed issue #2714
-        tblname = buffer.getString();
+        tblname = buffer.getName();
         buffer.forward(1); /* termination null */
 
         // Read column information from buffer

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TransactionPayloadLogEvent.java

@@ -5,12 +5,91 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * @author agapple 2022年5月23日 下午7:05:39
- * @version 1.1.6
+ * @version 1.1.7
  * @since mysql 8.0.20
  */
 public class TransactionPayloadLogEvent extends LogEvent {
 
+    public static final short COMPRESSION_TYPE_MIN_LENGTH         = 1;
+    public static final short COMPRESSION_TYPE_MAX_LENGTH         = 9;
+    public static final short PAYLOAD_SIZE_MIN_LENGTH             = 0;
+    public static final short PAYLOAD_SIZE_MAX_LENGTH             = 9;
+    public static final short UNCOMPRESSED_SIZE_MIN_LENGTH        = 0;
+    public static final short UNCOMPRESSED_SIZE_MAX_LENGTH        = 9;
+    public static final int   MAX_DATA_LENGTH                     = COMPRESSION_TYPE_MAX_LENGTH
+                                                                    + PAYLOAD_SIZE_MAX_LENGTH
+                                                                    + UNCOMPRESSED_SIZE_MAX_LENGTH;
+
+    /** Marks the end of the payload header. */
+    public static final int   OTW_PAYLOAD_HEADER_END_MARK         = 0;
+
+    /** The payload field */
+    public static final int   OTW_PAYLOAD_SIZE_FIELD              = 1;
+
+    /** The compression type field */
+    public static final int   OTW_PAYLOAD_COMPRESSION_TYPE_FIELD  = 2;
+
+    /** The uncompressed size field */
+    public static final int   OTW_PAYLOAD_UNCOMPRESSED_SIZE_FIELD = 3;
+
+    /* ZSTD compression. */
+    public final static int   COMPRESS_TYPE_ZSTD                  = 0;
+    /* No compression. */
+    public final static int   COMPRESS_TYPE_NONE                  = 255;
+
+    private long              m_compression_type                  = COMPRESS_TYPE_NONE;
+    private long              m_payload_size;
+    private long              m_uncompressed_size;
+    private byte[]            m_payload;
+
     public TransactionPayloadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
+
+        final int commonHeaderLen = descriptionEvent.getCommonHeaderLen();
+        final int postHeaderLen = descriptionEvent.getPostHeaderLen()[header.getType() - 1];
+
+        int offset = commonHeaderLen;
+        buffer.position(offset);
+        long type = 0, length = 0;
+        while (buffer.hasRemaining()) {
+            type = buffer.getPackedLong(); // type
+            if (type == OTW_PAYLOAD_HEADER_END_MARK) {
+                break;
+            }
+
+            length = buffer.getPackedLong(); // length
+            switch ((int) type) {
+                case OTW_PAYLOAD_SIZE_FIELD:
+                    m_payload_size = buffer.getPackedLong(); // value
+                    break;
+                case OTW_PAYLOAD_COMPRESSION_TYPE_FIELD:
+                    m_compression_type = buffer.getPackedLong(); // value
+                    break;
+                case OTW_PAYLOAD_UNCOMPRESSED_SIZE_FIELD:
+                    m_uncompressed_size = buffer.getPackedLong(); // value
+                    break;
+                default:
+                    buffer.forward((int) length);
+                    break;
+            }
+
+        }
+
+        if (m_uncompressed_size == 0) {
+            m_uncompressed_size = m_payload_size;
+        }
+        m_payload = buffer.getData((int) m_payload_size);
+    }
+
+    public boolean isCompressByZstd() {
+        return m_compression_type == COMPRESS_TYPE_ZSTD;
+    }
+
+    public boolean isCompressByNone() {
+        return m_compression_type == COMPRESS_TYPE_NONE;
+    }
+
+    public byte[] getPayload() {
+        return m_payload;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java

@@ -11,14 +11,19 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class UpdateRowsLogEvent extends RowsLogEvent {
+public class UpdateRowsLogEvent extends RowsLogEvent {
 
     public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-        super(header, buffer, descriptionEvent, false);
+        super(header, buffer, descriptionEvent, false , false);
     }
 
     public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
                               boolean partial){
-        super(header, buffer, descriptionEvent, partial);
+        super(header, buffer, descriptionEvent, partial ,false);
+    }
+
+    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                              boolean partial , boolean compress){
+        super(header, buffer, descriptionEvent, partial , compress);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java

@@ -2,6 +2,7 @@ package com.taobao.tddl.dbsync.binlog.event;
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.nio.charset.Charset;
 
 import com.taobao.tddl.dbsync.binlog.CharsetConversion;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
@@ -100,8 +101,8 @@ public final class UserVarLogEvent extends LogEvent {
                     value = buffer.getDecimal(precision, scale); // bin2decimal
                     break;
                 case STRING_RESULT:
-                    String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
-                    value = buffer.getFixString(valueLen, charsetName);
+                    Charset charset = CharsetConversion.getNioCharset(charsetNumber);
+                    value = buffer.getFixString(valueLen, charset);
                     break;
                 case ROW_RESULT:
                     // this seems to be banned in MySQL altogether

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java

@@ -9,9 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class WriteRowsLogEvent extends RowsLogEvent {
+public class WriteRowsLogEvent extends RowsLogEvent {
 
     public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-        super(header, buffer, descriptionEvent);
+        super(header, buffer, descriptionEvent, false, false);
+    }
+
+    public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent,
+                             boolean compress){
+        super(header, buffer, descriptionEvent, false, compress);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java

@@ -5,6 +5,8 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.LogHeader;
 
+import java.nio.charset.StandardCharsets;
+
 /**
  * mariadb的ANNOTATE_ROWS_EVENT类型
  * 
@@ -23,7 +25,7 @@ public class AnnotateRowsEvent extends IgnorableLogEvent {
 
         int offset = commonHeaderLen + postHeaderLen;
         int len = buffer.limit() - offset;
-        rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
+        rowsQuery = buffer.getFullString(offset, len, StandardCharsets.ISO_8859_1);
     }
 
     public String getRowsQuery() {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java

@@ -13,9 +13,20 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
  */
 public class BinlogCheckPointLogEvent extends IgnorableLogEvent {
 
+    private final String filename;
+
     public BinlogCheckPointLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header, buffer, descriptionEvent);
-        // do nothing , just mariadb binlog checkpoint
+        // mariadb binlog checkpoint
+        final int headerSize = descriptionEvent.getCommonHeaderLen();
+        final int postHeaderLen = descriptionEvent.getPostHeaderLen()[getHeader().getType() - 1];
+
+        buffer.position(headerSize);
+        long binlogFileLen = buffer.getUint32();
+        filename = buffer.getFixString((int) binlogFileLen);
     }
 
+    public String getFilename() {
+        return filename;
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/DeleteRowsCompressLogEvent.java

@@ -0,0 +1 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/QueryCompressedLogEvent.java

@@ -0,0 +1 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/UpdateRowsCompressLogEvent.java

@@ -0,0 +1 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/WriteRowsCompressLogEvent.java

@@ -0,0 +1 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java

@@ -52,7 +52,7 @@ public class BaseLogFetcherTest {
                 event.getHeader().getLogPos() - event.getHeader().getEventLen(),
                 event.getTable().getDbName(),
                 event.getTable().getTableName()));
-            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+            RowsLogBuffer buffer = event.getRowsBuf(charset);
             BitSet columns = event.getColumns();
             BitSet changeColumns = event.getChangeColumns();
             while (buffer.nextOneRow(columns)) {

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java

@@ -4,20 +4,15 @@ import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.Statement;
+import java.util.List;
 
+import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent;
 import org.junit.Assert;
-import org.junit.Ignore;
 import org.junit.Test;
 
-import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
-import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.*;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
-@Ignore
+
 public class DirectLogFetcherTest extends BaseLogFetcherTest {
 
     @Test
@@ -25,52 +20,20 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
         DirectLogFetcher fecther = new DirectLogFetcher();
         try {
             Class.forName("com.mysql.jdbc.Driver");
-            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello");
+            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "123456");
             Statement statement = connection.createStatement();
             statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
             statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
 
-            fecther.open(connection, "mysql-bin.000007", 89797036L, 2);
+            fecther.open(connection, "mysql-bin.000002", 4L, 1);
 
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
             while (fecther.fetch()) {
                 LogEvent event = decoder.decode(fecther, context);
-                int eventType = event.getHeader().getType();
-                switch (eventType) {
-                    case LogEvent.ROTATE_EVENT:
-                        binlogFileName = ((RotateLogEvent) event).getFilename();
-                        break;
-                    case LogEvent.WRITE_ROWS_EVENT_V1:
-                    case LogEvent.WRITE_ROWS_EVENT:
-                        parseRowsEvent((WriteRowsLogEvent) event);
-                        break;
-                    case LogEvent.UPDATE_ROWS_EVENT_V1:
-                    case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
-                    case LogEvent.UPDATE_ROWS_EVENT:
-                        parseRowsEvent((UpdateRowsLogEvent) event);
-                        break;
-                    case LogEvent.DELETE_ROWS_EVENT_V1:
-                    case LogEvent.DELETE_ROWS_EVENT:
-                        parseRowsEvent((DeleteRowsLogEvent) event);
-                        break;
-                    case LogEvent.QUERY_EVENT:
-                        parseQueryEvent((QueryLogEvent) event);
-                        break;
-                    case LogEvent.ROWS_QUERY_LOG_EVENT:
-                        parseRowsQueryEvent((RowsQueryLogEvent) event);
-                        break;
-                    case LogEvent.ANNOTATE_ROWS_EVENT:
-                        parseAnnotateRowsEvent((AnnotateRowsEvent) event);
-                        break;
-                    case LogEvent.XID_EVENT:
-                        parseXidEvent((XidLogEvent) event);
-                        break;
-                    default:
-                        break;
-                }
+                processEvent(event, decoder, context);
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             e.printStackTrace();
             Assert.fail(e.getMessage());
         } finally {
@@ -82,4 +45,49 @@ public class DirectLogFetcherTest extends BaseLogFetcherTest {
         }
 
     }
+
+    public void processEvent(LogEvent event, LogDecoder decoder, LogContext context) throws Throwable {
+        int eventType = event.getHeader().getType();
+        switch (eventType) {
+            case LogEvent.ROTATE_EVENT:
+                binlogFileName = ((RotateLogEvent) event).getFilename();
+                break;
+            case LogEvent.BINLOG_CHECKPOINT_EVENT:
+                binlogFileName = ((BinlogCheckPointLogEvent) event).getFilename();
+                break;
+            case LogEvent.WRITE_ROWS_EVENT_V1:
+            case LogEvent.WRITE_ROWS_EVENT:
+                parseRowsEvent((WriteRowsLogEvent) event);
+                break;
+            case LogEvent.UPDATE_ROWS_EVENT_V1:
+            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
+            case LogEvent.UPDATE_ROWS_EVENT:
+                parseRowsEvent((UpdateRowsLogEvent) event);
+                break;
+            case LogEvent.DELETE_ROWS_EVENT_V1:
+            case LogEvent.DELETE_ROWS_EVENT:
+                parseRowsEvent((DeleteRowsLogEvent) event);
+                break;
+            case LogEvent.QUERY_EVENT:
+                parseQueryEvent((QueryLogEvent) event);
+                break;
+            case LogEvent.ROWS_QUERY_LOG_EVENT:
+                parseRowsQueryEvent((RowsQueryLogEvent) event);
+                break;
+            case LogEvent.ANNOTATE_ROWS_EVENT:
+                parseAnnotateRowsEvent((AnnotateRowsEvent) event);
+                break;
+            case LogEvent.XID_EVENT:
+                parseXidEvent((XidLogEvent) event);
+                break;
+            case LogEvent.TRANSACTION_PAYLOAD_EVENT:
+                List<LogEvent> events = decoder.processIterateDecode(event, context);
+                for (LogEvent deEvent : events) {
+                    processEvent(deEvent, decoder, context);
+                }
+                break;
+            default:
+                break;
+        }
+    }
 }

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java

@@ -3,6 +3,7 @@ package com.taobao.tddl.dbsync.binlog;
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
+import java.util.List;
 
 import org.junit.Assert;
 import org.junit.Before;
@@ -43,43 +44,9 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
             while (fetcher.fetch()) {
                 LogEvent event = null;
                 event = decoder.decode(fetcher, context);
-                if (event != null) {
-                    int eventType = event.getHeader().getType();
-                    switch (eventType) {
-                        case LogEvent.ROTATE_EVENT:
-                            binlogFileName = ((RotateLogEvent) event).getFilename();
-                            break;
-                        case LogEvent.WRITE_ROWS_EVENT_V1:
-                        case LogEvent.WRITE_ROWS_EVENT:
-                            parseRowsEvent((WriteRowsLogEvent) event);
-                            break;
-                        case LogEvent.UPDATE_ROWS_EVENT_V1:
-                        case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
-                        case LogEvent.UPDATE_ROWS_EVENT:
-                            parseRowsEvent((UpdateRowsLogEvent) event);
-                            break;
-                        case LogEvent.DELETE_ROWS_EVENT_V1:
-                        case LogEvent.DELETE_ROWS_EVENT:
-                            parseRowsEvent((DeleteRowsLogEvent) event);
-                            break;
-                        case LogEvent.QUERY_EVENT:
-                            parseQueryEvent((QueryLogEvent) event);
-                            break;
-                        case LogEvent.ROWS_QUERY_LOG_EVENT:
-                            parseRowsQueryEvent((RowsQueryLogEvent) event);
-                            break;
-                        case LogEvent.ANNOTATE_ROWS_EVENT:
-                            parseAnnotateRowsEvent((AnnotateRowsEvent) event);
-                            break;
-                        case LogEvent.XID_EVENT:
-                            parseXidEvent((XidLogEvent) event);
-                            break;
-                        default:
-                            break;
-                    }
-                }
+                processEvent(event, decoder, context);
             }
-        } catch (Exception e) {
+        } catch (Throwable e) {
             Assert.fail(e.getMessage());
         } finally {
             try {
@@ -89,4 +56,46 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
             }
         }
     }
+
+    public void processEvent(LogEvent event, LogDecoder decoder, LogContext context) throws Throwable {
+        int eventType = event.getHeader().getType();
+        switch (eventType) {
+            case LogEvent.ROTATE_EVENT:
+                binlogFileName = ((RotateLogEvent) event).getFilename();
+                break;
+            case LogEvent.WRITE_ROWS_EVENT_V1:
+            case LogEvent.WRITE_ROWS_EVENT:
+                parseRowsEvent((WriteRowsLogEvent) event);
+                break;
+            case LogEvent.UPDATE_ROWS_EVENT_V1:
+            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
+            case LogEvent.UPDATE_ROWS_EVENT:
+                parseRowsEvent((UpdateRowsLogEvent) event);
+                break;
+            case LogEvent.DELETE_ROWS_EVENT_V1:
+            case LogEvent.DELETE_ROWS_EVENT:
+                parseRowsEvent((DeleteRowsLogEvent) event);
+                break;
+            case LogEvent.QUERY_EVENT:
+                parseQueryEvent((QueryLogEvent) event);
+                break;
+            case LogEvent.ROWS_QUERY_LOG_EVENT:
+                parseRowsQueryEvent((RowsQueryLogEvent) event);
+                break;
+            case LogEvent.ANNOTATE_ROWS_EVENT:
+                parseAnnotateRowsEvent((AnnotateRowsEvent) event);
+                break;
+            case LogEvent.XID_EVENT:
+                parseXidEvent((XidLogEvent) event);
+                break;
+            case LogEvent.TRANSACTION_PAYLOAD_EVENT:
+                List<LogEvent> events = decoder.processIterateDecode(event, context);
+                for (LogEvent deEvent : events) {
+                    processEvent(deEvent, decoder, context);
+                }
+                break;
+            default:
+                break;
+        }
+    }
 }

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java

@@ -30,6 +30,7 @@ public class CanalConstants {
     public static final String CANAL_WITHOUT_NETTY                  = ROOT + "." + "withoutNetty";
 
     public static final String CANAL_DESTINATIONS                   = ROOT + "." + "destinations";
+    public static final String CANAL_DESTINATIONS_EXPR              = ROOT + "." + "destinations.expr";
     public static final String CANAL_AUTO_SCAN                      = ROOT + "." + "auto.scan";
     public static final String CANAL_AUTO_SCAN_INTERVAL             = ROOT + "." + "auto.scan.interval";
     public static final String CANAL_CONF_DIR                       = ROOT + "." + "conf.dir";

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java

@@ -1,18 +1,5 @@
 package com.alibaba.otter.canal.deployer;
 
-import java.util.Map;
-import java.util.Properties;
-
-import org.I0Itec.zkclient.IZkStateListener;
-import org.I0Itec.zkclient.exception.ZkNoNodeException;
-import org.I0Itec.zkclient.exception.ZkNodeExistsException;
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
 import com.alibaba.otter.canal.common.utils.AddressUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
@@ -36,6 +23,26 @@ import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
 import com.google.common.base.Function;
 import com.google.common.collect.MapMaker;
 import com.google.common.collect.MigrateMap;
+import org.I0Itec.zkclient.IZkStateListener;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static com.alibaba.otter.canal.deployer.CanalConstants.CANAL_DESTINATIONS;
+import static com.alibaba.otter.canal.deployer.CanalConstants.CANAL_DESTINATIONS_EXPR;
 
 /**
  * canal调度控制器
@@ -58,7 +65,7 @@ public class CanalController {
     private boolean                                  autoScan = true;
     private InstanceAction                           defaultAction;
     private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
-    private CanalServerWithEmbedded                  embededCanalServer;
+    private CanalServerWithEmbedded                  embeddedCanalServer;
     private CanalServerWithNetty                     canalServer;
 
     private CanalInstanceGenerator                   instanceGenerator;
@@ -102,15 +109,15 @@ public class CanalController {
         registerIp = getProperty(properties, CanalConstants.CANAL_REGISTER_IP);
         port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT, "11111"));
         adminPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_ADMIN_PORT, "11110"));
-        embededCanalServer = CanalServerWithEmbedded.instance();
-        embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
+        embeddedCanalServer = CanalServerWithEmbedded.instance();
+        embeddedCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
         int metricsPort = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_METRICS_PULL_PORT, "11112"));
-        embededCanalServer.setMetricsPort(metricsPort);
+        embeddedCanalServer.setMetricsPort(metricsPort);
 
         this.adminUser = getProperty(properties, CanalConstants.CANAL_ADMIN_USER);
         this.adminPasswd = getProperty(properties, CanalConstants.CANAL_ADMIN_PASSWD);
-        embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER));
-        embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD));
+        embeddedCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER));
+        embeddedCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD));
 
         String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
         if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
@@ -149,7 +156,7 @@ public class CanalController {
                 public void processActiveEnter() {
                     try {
                         MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
-                        embededCanalServer.start(destination);
+                        embeddedCanalServer.start(destination);
                         if (canalMQStarter != null) {
                             canalMQStarter.startDestination(destination);
                         }
@@ -164,7 +171,7 @@ public class CanalController {
                         if (canalMQStarter != null) {
                             canalMQStarter.stopDestination(destination);
                         }
-                        embededCanalServer.stop(destination);
+                        embeddedCanalServer.stop(destination);
                     } finally {
                         MDC.remove(CanalConstants.MDC_DESTINATION);
                     }
@@ -232,7 +239,7 @@ public class CanalController {
                         instanceConfigs.put(destination, config);
                     }
 
-                    if (!embededCanalServer.isStart(destination)) {
+                    if (!embeddedCanalServer.isStart(destination)) {
                         // HA机制启动
                         ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
                         if (!config.getLazy() && !runningMonitor.isStart()) {
@@ -247,7 +254,7 @@ public class CanalController {
                     // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息
                     InstanceConfig config = instanceConfigs.remove(destination);
                     if (config != null) {
-                        embededCanalServer.stop(destination);
+                        embeddedCanalServer.stop(destination);
                         ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
                         if (runningMonitor.isStart()) {
                             runningMonitor.stop();
@@ -377,7 +384,7 @@ public class CanalController {
                 instanceGenerator.setSpringXml(config.getSpringXml());
                 return instanceGenerator.generate(destination);
             } else {
-                throw new UnsupportedOperationException("unknow mode :" + config.getMode());
+                throw new UnsupportedOperationException("unknown mode :" + config.getMode());
             }
 
         };
@@ -390,7 +397,7 @@ public class CanalController {
     }
 
     private void initInstanceConfig(Properties properties) {
-        String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+        String destinationStr = getDestinations(properties);
         String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT);
 
         for (String destination : destinations) {
@@ -461,6 +468,40 @@ public class CanalController {
         return StringUtils.trim(value);
     }
 
+    public static String getDestinations(Properties properties) {
+        String expr = getProperty(properties, CANAL_DESTINATIONS_EXPR);
+        if (StringUtils.isNotBlank(expr)) {
+            return parseExpr(expr);
+        } else {
+            return getProperty(properties, CANAL_DESTINATIONS);
+        }
+    }
+
+    private static String parseExpr(String expr) {
+        String prefix = StringUtils.substringBefore(expr, "{");
+        String range = StringUtils.substringAfter(expr, "{");
+        range = StringUtils.substringBefore(range, "}");
+
+        String regex = "(\\d+)-(\\d+)";
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(range);
+        if (matcher.find()) {
+            String head = matcher.group(1);
+            String tail = matcher.group(2);
+            int start = Integer.parseInt(head);
+            int end = Integer.parseInt(tail);
+
+            List<String> list = new ArrayList<>();
+            for (int i = start; i <= end; i++) {
+                String d = prefix + i;
+                list.add(d);
+            }
+            return list.stream().map(Object::toString).collect(Collectors.joining(","));
+        } else {
+            throw new CanalServerException("invalid destinations expr " + expr);
+        }
+    }
+
     public void start() throws Throwable {
         logger.info("## start the canal server[{}({}):{}]", ip, registerIp, port);
         // 创建整个canal的工作节点
@@ -483,14 +524,14 @@ public class CanalController {
                 }
             });
         }
-        // 优先启动embeded服务
-        embededCanalServer.start();
+        // 优先启动embedded服务
+        embeddedCanalServer.start();
         // 尝试启动一下非lazy状态的通道
         for (Map.Entry<String, InstanceConfig> entry : instanceConfigs.entrySet()) {
             final String destination = entry.getKey();
             InstanceConfig config = entry.getValue();
             // 创建destination的工作节点
-            if (!embededCanalServer.isStart(destination)) {
+            if (!embeddedCanalServer.isStart(destination)) {
                 // HA机制启动
                 ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
                 if (!config.getLazy() && !runningMonitor.isStart()) {
@@ -558,6 +599,11 @@ public class CanalController {
         }
 
         ZkClientx.clearClients();
+
+        // 需要释放 CanalServerWithEmbedded 否则主线程退出后，进程无法自动完整退出...
+        if (embeddedCanalServer != null) {
+            embeddedCanalServer.stop();
+        }
     }
 
     private void initCid(String path) {

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.deployer;
 
+import com.alibaba.otter.canal.connector.core.spi.ProxyCanalMQProducer;
 import java.util.Properties;
 
 import com.alibaba.otter.canal.connector.core.config.MQProperties;
@@ -21,21 +22,21 @@ import com.alibaba.otter.canal.server.CanalMQStarter;
  */
 public class CanalStarter {
 
-    private static final Logger logger                    = LoggerFactory.getLogger(CanalStarter.class);
+    private static final Logger logger = LoggerFactory.getLogger(CanalStarter.class);
 
-    private static final String CONNECTOR_SPI_DIR         = "/plugin";
+    private static final String CONNECTOR_SPI_DIR = "/plugin";
     private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";
 
-    private CanalController     controller                = null;
-    private CanalMQProducer     canalMQProducer           = null;
-    private Thread              shutdownThread            = null;
-    private CanalMQStarter      canalMQStarter            = null;
+    private CanalController controller = null;
+    private CanalMQProducer canalMQProducer = null;
+    private Thread shutdownThread = null;
+    private CanalMQStarter canalMQStarter = null;
     private volatile Properties properties;
-    private volatile boolean    running                   = false;
+    private volatile boolean running = false;
 
     private CanalAdminWithNetty canalAdmin;
 
-    public CanalStarter(Properties properties){
+    public CanalStarter(Properties properties) {
         this.properties = properties;
     }
 
@@ -67,10 +68,8 @@ public class CanalStarter {
             canalMQProducer = loader
                 .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
             if (canalMQProducer != null) {
-                ClassLoader cl = Thread.currentThread().getContextClassLoader();
-                Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
+                canalMQProducer =  new ProxyCanalMQProducer(canalMQProducer);
                 canalMQProducer.init(properties);
-                Thread.currentThread().setContextClassLoader(cl);
             }
         }
 
@@ -103,7 +102,7 @@ public class CanalStarter {
 
         if (canalMQProducer != null) {
             canalMQStarter = new CanalMQStarter(canalMQProducer);
-            String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+            String destinations = CanalController.getDestinations(properties);
             canalMQStarter.start(destinations);
             controller.setCanalMQStarter(canalMQStarter);
         }

deployer/src/main/resources/example/instance.properties

@@ -56,4 +56,7 @@ canal.mq.partition=0
 #canal.mq.partitionsNum=3
 #canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
 #canal.mq.partitionHash=test.table:id^name,.*\\..*
+#
+# multi stream for polardbx
+canal.instance.multi.stream.on=false
 #################################################

deployer/src/main/resources/spring/default-instance.xml

@@ -197,6 +197,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
 
 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>
 
 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">

deployer/src/main/resources/spring/file-instance.xml

@@ -183,6 +183,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
 
 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>
 
 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">

deployer/src/main/resources/spring/group-instance.xml

@@ -170,6 +170,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
 
 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>
 
 	<bean id="eventParser2" parent="baseEventParser">
@@ -278,6 +279,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
 
 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>
 
     <bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">

deployer/src/main/resources/spring/memory-instance.xml

@@ -171,6 +171,7 @@
 		<property name="parallelBufferSize" value="${canal.instance.parser.parallelBufferSize:256}" />
 
 		<property name="autoResetLatestPosMode" value="${canal.auto.reset.latest.pos.mode:false}" />
+		<property name="multiStreamEnable" value="${canal.instance.multi.stream.on:false}"/>
 	</bean>
 
 	<bean id="mqConfig" class="com.alibaba.otter.canal.instance.core.CanalMQConfig">

docker/Dockerfile

@@ -1,6 +1,6 @@
 FROM canal/osbase:v2
 
-MAINTAINER agapple (jianghang115@gmail.com)
+LABEL agapple (jianghang115@gmail.com)
 
 # install canal
 COPY image/ /tmp/docker/
@@ -34,4 +34,4 @@ EXPOSE 11110 11111 11112 9100
 WORKDIR /home/admin
 
 ENTRYPOINT [ "/alidata/bin/main.sh" ]
-CMD [ "/home/admin/app.sh" ]
+CMD [ "/home/admin/app.sh" ]

docker/image/app.sh

@@ -95,9 +95,20 @@ function start_canal() {
         fi
 
         destination=`perl -le 'print $ENV{"canal.destinations"}'`
-        if [[ "$destination" =~ ',' ]]; then
-            echo "multi destination:$destination is not support"
-            exit 1;
+        destinationExpr=`perl -le 'print $ENV{"canal.destinations.expr"}'`
+        multistream=`perl -le 'print $ENV{"canal.instance.multi.stream.on"}'`
+
+        if [[ "$destination" =~ ',' ]] || [[ -n "$destinationExpr" ]]; then
+            if [[ "$multistream" = 'true' ]] ; then
+                if [[ -n "$destinationExpr" ]] ; then
+                    splitDestinations '1' $destinationExpr
+                else
+                    splitDestinations '2' $destination
+                fi
+            else
+                echo "multi destination is not support, destinationExpr:$destinationExpr, destinations:$destination"
+                exit 1;
+            fi
         else
             if [ "$destination" != "" ] && [ "$destination" != "example" ] ; then
                 if [ -d /home/admin/canal-server/conf/example ]; then
@@ -113,6 +124,34 @@ function start_canal() {
     fi  
 }
 
+function splitDestinations() {
+    holdExample="false"
+    prefix=''
+    array=()
+
+    if [[  "$1" == '1' ]] ; then
+        echo "split destinations expr "$2
+        prefix=$(echo $2 | sed 's/{.*//')
+        num=$(echo $2 | sed 's/.*{//;s/}//;s/-/ /')
+        array=($(seq $num))
+    else
+        echo "split destinations "$2
+        array=(${2//,/ })
+    fi
+
+    for var in ${array[@]}
+    do
+        cp -r /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$prefix$var
+        chown admin:admin -R /home/admin/canal-server/conf/$prefix$var
+        if [[ "$prefix$var" = 'example' ]] ; then
+            holdExample="true"
+        fi
+    done
+    if [[ "$holdExample" != 'true' ]] ; then
+        rm -rf /home/admin/canal-server/conf/example
+    fi
+}
+
 function stop_canal() {
     echo "stop canal"
     su admin -c 'cd /home/admin/canal-server/bin/ && sh stop.sh 1>>/tmp/start.log 2>&1'

driver/pom.xml

@@ -33,6 +33,10 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.jboss.netty</groupId>
+			<artifactId>netty</artifactId>
+		</dependency>
 		<!-- test dependency -->
 		<dependency>
 			<groupId>junit</groupId>

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java

@@ -13,11 +13,7 @@ import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.AuthSwitchResponsePacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.ClientAuthenticationPacket;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QuitCommandPacket;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.server.AuthSwitchRequestMoreData;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.server.AuthSwitchRequestPacket;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.server.HandshakeInitializationPacket;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.server.Reply323Packet;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.*;
 import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
 import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannelPool;
 import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC;
@@ -26,7 +22,7 @@ import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
 
 /**
  * 基于mysql socket协议的链接实现
- * 
+ *
  * @author jianghang 2013-2-18 下午09:22:30
  * @version 1.0.1
  */
@@ -220,53 +216,46 @@ public class MysqlConnector {
                 packet.fromBytes(body);
                 authData = packet.authData;
                 pluginName = packet.authName;
+                logger.info("auth switch pluginName is {}.", pluginName);
             }
 
-            boolean isSha2Password = false;
             byte[] encryptedPassword = null;
-            if (pluginName != null && "mysql_native_password".equals(pluginName)) {
+            if ("mysql_clear_password".equals(pluginName)) {
+                encryptedPassword = getPassword().getBytes();
+                header = authSwitchAfterAuth(encryptedPassword, header);
+                body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
+            } else if ("mysql_native_password".equals(pluginName)) {
                 try {
                     encryptedPassword = MySQLPasswordEncrypter.scramble411(getPassword().getBytes(), authData);
                 } catch (NoSuchAlgorithmException e) {
                     throw new RuntimeException("can't encrypt password that will be sent to MySQL server.", e);
                 }
-            } else if (pluginName != null && "caching_sha2_password".equals(pluginName)) {
-                isSha2Password = true;
+                header = authSwitchAfterAuth(encryptedPassword, header);
+                body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
+            } else if ("caching_sha2_password".equals(pluginName)) {
+                byte[] scramble = authData;
                 try {
-                    encryptedPassword = MySQLPasswordEncrypter.scrambleCachingSha2(getPassword().getBytes(), authData);
+                    encryptedPassword = MySQLPasswordEncrypter.scrambleCachingSha2(getPassword().getBytes(), scramble);
                 } catch (DigestException e) {
                     throw new RuntimeException("can't encrypt password that will be sent to MySQL server.", e);
                 }
-            }
-            assert encryptedPassword != null;
-            AuthSwitchResponsePacket responsePacket = new AuthSwitchResponsePacket();
-            responsePacket.authData = encryptedPassword;
-            byte[] auth = responsePacket.toBytes();
-
-            h = new HeaderPacket();
-            h.setPacketBodyLength(auth.length);
-            h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
-            PacketManager.writePkg(channel, h.toBytes(), auth);
-            logger.info("auth switch response packet is sent out.");
-
-            header = null;
-            header = PacketManager.readHeader(channel, 4);
-            body = null;
-            body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
-            assert body != null;
-            if (isSha2Password) {
+                header = authSwitchAfterAuth(encryptedPassword, header);
+                body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
+                assert body != null;
                 if (body[0] == 0x01 && body[1] == 0x04) {
-                    // password auth failed
-                    throw new IOException("caching_sha2_password Auth failed");
+                    // fixed issue https://github.com/alibaba/canal/pull/4767, support mysql 8.0.30+
+                    header = cachingSha2PasswordFullAuth(channel, header, getPassword().getBytes(), scramble);
+                    body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
+                } else {
+                    header = PacketManager.readHeader(channel, 4);
+                    body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
                 }
-
-                header = null;
-                header = PacketManager.readHeader(channel, 4);
-                body = null;
+            } else {
+                header = authSwitchAfterAuth(encryptedPassword, header);
                 body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
             }
         }
-
+        assert body != null;
         if (body[0] < 0) {
             if (body[0] == -1) {
                 ErrorPacket err = new ErrorPacket();
@@ -278,6 +267,62 @@ public class MysqlConnector {
         }
     }
 
+    private HeaderPacket cachingSha2PasswordFullAuth(SocketChannel channel, HeaderPacket header, byte[] pass,
+                                                     byte[] seed) throws IOException {
+        AuthSwitchResponsePacket responsePacket = new AuthSwitchResponsePacket();
+        responsePacket.authData = new byte[] { 2 };
+        byte[] auth = responsePacket.toBytes();
+        HeaderPacket h = new HeaderPacket();
+        h.setPacketBodyLength(auth.length);
+        h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
+        PacketManager.writePkg(channel, h.toBytes(), auth);
+        logger.info("caching sha2 password fullAuth request public key packet is sent out.");
+
+        header = PacketManager.readHeader(channel, 4);
+        byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout);
+        AuthSwitchRequestMoreData packet = new AuthSwitchRequestMoreData();
+        packet.fromBytes(body);
+        if (packet.status != 0x01) {
+            throw new IOException("caching_sha2_password get public key failed");
+        }
+        logger.info("caching sha2 password fullAuth get server public key succeed.");
+        byte[] publicKeyBytes = packet.authData;
+
+        byte[] encryptedPassword = null;
+        try {
+            encryptedPassword = MySQLPasswordEncrypter.scrambleRsa(publicKeyBytes, pass, seed);
+        } catch (Exception e) {
+            logger.error("rsa encrypt failed {}", publicKeyBytes);
+            throw new IOException("caching_sha2_password auth failed", e);
+        }
+
+        // send auth
+        responsePacket = new AuthSwitchResponsePacket();
+        responsePacket.authData = encryptedPassword;
+        auth = responsePacket.toBytes();
+        h = new HeaderPacket();
+        h.setPacketBodyLength(auth.length);
+        h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
+        PacketManager.writePkg(channel, h.toBytes(), auth);
+        logger.info("caching sha2 password fullAuth response auth data packet is sent out.");
+        return PacketManager.readHeader(channel, 4);
+    }
+
+    private HeaderPacket authSwitchAfterAuth(byte[] encryptedPassword, HeaderPacket header) throws IOException {
+        assert encryptedPassword != null;
+        AuthSwitchResponsePacket responsePacket = new AuthSwitchResponsePacket();
+        responsePacket.authData = encryptedPassword;
+        byte[] auth = responsePacket.toBytes();
+
+        HeaderPacket h = new HeaderPacket();
+        h.setPacketBodyLength(auth.length);
+        h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1));
+        PacketManager.writePkg(channel, h.toBytes(), auth);
+        logger.info("auth switch response packet is sent out.");
+        header = PacketManager.readHeader(channel, 4);
+        return header;
+    }
+
     private void auth323(SocketChannel channel, byte packetSequenceNumber, byte[] seed) throws IOException {
         // auth 323
         Reply323Packet r323 = new Reply323Packet();