fixed phoenix version

* add tablestore adapter

* add tablestore adapter

* fix bug add add log when etl fail

* fix bug，tinyint(1)以及rowupdatechange

Co-authored-by: agapple <jianghang115@gmail.com>

.travis.yml

@@ -14,7 +14,7 @@ cache:
 install: true
 
 script:
-  - travis_wait 30 ./mvnw clean install -DskipTests=false -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true
+  - travis_wait 30 ./mvnw -T 1C clean install -DskipTests=false -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

admin/admin-ui/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <artifactId>canal.admin</artifactId>
     <groupId>com.alibaba.otter</groupId>
-    <version>1.1.5</version>
+    <version>1.1.6-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

admin/admin-web/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.admin</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>canal.admin-web</artifactId>

admin/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.admin</artifactId>
-    <version>1.1.5</version>
+    <version>1.1.6-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal admin module for otter ${project.version}</name>
     <modules>

client-adapter/common/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -54,6 +54,8 @@ public class CanalClientConfig {
     // canal adapters 配置
     private List<CanalAdapter> canalAdapters;
 
+    private Boolean terminateOnException = false;
+
     public String getCanalServerHost() {
         return canalServerHost;
     }
@@ -222,6 +224,14 @@ public class CanalClientConfig {
         this.namespace = namespace;
     }
 
+    public Boolean getTerminateOnException() {
+        return terminateOnException;
+    }
+
+    public void setTerminateOnException(Boolean terminateOnException) {
+        this.terminateOnException = terminateOnException;
+    }
+
     public static class CanalAdapter {
 
         private String      instance; // 实例名

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/URLClassExtensionLoader.java

@@ -21,7 +21,9 @@ public class URLClassExtensionLoader extends URLClassLoader {
 
         if (name.startsWith("java.") || name.startsWith("org.slf4j.") || name.startsWith("org.apache.logging")
             || name.startsWith("org.apache.zookeeper.") || name.startsWith("org.I0Itec.zkclient.")
-            || name.startsWith("org.apache.commons.logging.")) {
+            || name.startsWith("org.apache.commons.logging.")
+            || name.startsWith("com.alibaba.druid")
+            ) {
             // || name.startsWith("org.apache.hadoop."))
             // {
             c = super.loadClass(name);

client-adapter/es6x/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/es7x/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/escore/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/hbase/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/kudu/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/launcher/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -141,6 +141,19 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.tablestore</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
         <!-- connector plugin -->
         <dependency>
             <groupId>com.alibaba.otter</groupId>

client-adapter/launcher/src/main/assembly/dev.xml

@@ -64,6 +64,13 @@
                 <exclude>META-INF/**</exclude>
             </excludes>
         </fileSet>
+        <fileSet>
+            <directory>../tablestore/src/main/resources/</directory>
+            <outputDirectory>/conf</outputDirectory>
+            <excludes>
+                <exclude>META-INF/**</exclude>
+            </excludes>
+        </fileSet>
         <fileSet>
             <directory>target</directory>
             <outputDirectory>logs</outputDirectory>

client-adapter/launcher/src/main/assembly/release.xml

@@ -65,6 +65,13 @@
                 <exclude>META-INF/**</exclude>
             </excludes>
         </fileSet>
+        <fileSet>
+            <directory>../tablestore/src/main/resources/</directory>
+            <outputDirectory>/conf</outputDirectory>
+            <excludes>
+                <exclude>META-INF/**</exclude>
+            </excludes>
+        </fileSet>
         <fileSet>
             <directory>target</directory>
             <outputDirectory>logs</outputDirectory>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java

@@ -12,8 +12,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
  */
 @SpringBootApplication
 public class CanalAdapterApplication {
-
     public static void main(String[] args) {
+
         SpringApplication application = new SpringApplication(CanalAdapterApplication.class);
         application.setBannerMode(Banner.Mode.OFF);
         application.run(args);

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -7,6 +7,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,7 +64,7 @@ public class AdapterProcessor {
         // load connector consumer
         ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
         canalMsgConsumer = loader
-            .getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+            .getExtension(canalClientConfig.getMode().toLowerCase(),destination ,CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
 
         Properties properties = canalClientConfig.getConsumerProperties();
         properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
@@ -217,8 +218,15 @@ public class AdapterProcessor {
                                 canalMsgConsumer.rollback(); // 处理失败, 回滚数据
                                 logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
                             } else {
-                                canalMsgConsumer.ack();
-                                logger.error(e.getMessage() + " Error sync but ACK!");
+                                if (canalClientConfig.getTerminateOnException()) {
+                                    canalMsgConsumer.rollback();
+                                    logger.error("Retry fail, turn switch off and abort data transfer.");
+                                    syncSwitch.off(canalDestination);
+                                    logger.error("finish turn off switch of destination:" + canalDestination);
+                                } else {
+                                    canalMsgConsumer.ack();
+                                    logger.error(e.getMessage() + " Error sync but ACK!");
+                                }
                             }
                             Thread.sleep(500);
                         }

client-adapter/launcher/src/main/resources/application.yml

@@ -94,4 +94,11 @@ canal.conf:
 #        - name: kudu
 #          key: kudu
 #          properties:
-#            kudu.master.address: 127.0.0.1 # ',' split multi address
+#            kudu.master.address: 127.0.0.1 # ',' split multi address
+#        - name: phoenix
+#            key: phoenix
+#            properties:
+#              jdbc.driverClassName: org.apache.phoenix.jdbc.PhoenixDriver
+#              jdbc.url: jdbc:phoenix:127.0.0.1:2181:/hbase/db
+#              jdbc.username:
+#              jdbc.password:

client-adapter/logger/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/phoenix/pom.xml

@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.6-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.phoenix</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter phoenix module for otter ${project.version}</name>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.phoenix</groupId>
+            <artifactId>phoenix-core</artifactId>
+            <version>4.14.1-HBase-1.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>1.4.8</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>2.5.0</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.12</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <version>2.6</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.4</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>18.0</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <version>1.1.9</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+            <version>1.2.58</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>5.1.48</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+<!--                            <tasks>-->
+<!--                                <copy todir="${project.basedir}/../launcher/target/classes/phoenix" overwrite="true">-->
+<!--                                    <fileset dir="${project.basedir}/target/classes/phoenix" erroronmissingdir="true">-->
+<!--                                        <include name="*.yml"/>-->
+<!--                                    </fileset>-->
+<!--                                </copy>-->
+<!--                            </tasks>-->
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/canal-adapter/conf/phoenix" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/phoenix" erroronmissingdir="true">
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/" erroronmissingdir="true">
+                                        <include name="*with-dependencies.jar"/>
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../launcher/target/classes/phoenix" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/phoenix" erroronmissingdir="true">
+                                        <include name="*.yml" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/PhoenixAdapter.java

@@ -0,0 +1,291 @@
+package com.alibaba.otter.canal.client.adapter.phoenix;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.ConfigLoader;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.phoenix.monitor.PhoenixConfigMonitor;
+import com.alibaba.otter.canal.client.adapter.phoenix.service.PhoenixEtlService;
+import com.alibaba.otter.canal.client.adapter.phoenix.service.PhoenixSyncService;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.*;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 15:01
+ * @Description: Phoenix适配器实现类
+ */
+@SPI("phoenix")
+public class PhoenixAdapter implements OuterAdapter {
+
+    private static Logger logger = LoggerFactory.getLogger(PhoenixAdapter.class);
+
+    private Map<String, MappingConfig> phoenixMapping = new ConcurrentHashMap<>();                // 文件名对应配置
+    private Map<String, Map<String, MappingConfig>> mappingConfigCache = new ConcurrentHashMap<>();                // 库名-表名对应配置
+
+    private static  String DriverClass;
+    private static  String PhoenixUrl;
+    private static  Properties phoenixPro = new Properties();
+
+    private PhoenixSyncService phoenixSyncService;
+
+    private PhoenixConfigMonitor phoenixConfigMonitor;
+
+    private Properties envProperties;
+
+
+
+    public Map<String, MappingConfig> getPhoenixMapping() {
+        return phoenixMapping;
+    }
+
+    public Map<String, Map<String, MappingConfig>> getMappingConfigCache() {
+        return mappingConfigCache;
+    }
+
+    public PhoenixAdapter() {
+        logger.info("PhoenixAdapter create: {} {}", this, Thread.currentThread().getStackTrace());
+    }
+    /**
+     * 初始化方法
+     *
+     * @param configuration 外部适配器配置信息
+     */
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        this.envProperties = envProperties;
+        Map<String, MappingConfig> phoenixMappingTmp = ConfigLoader.load(envProperties);
+        // 过滤不匹配的key的配置
+        phoenixMappingTmp.forEach((key, mappingConfig) -> {
+            if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
+                    || (mappingConfig.getOuterAdapterKey() != null
+                    && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
+                phoenixMapping.put(key, mappingConfig);
+            }
+        });
+
+        if (phoenixMapping.isEmpty()) {
+            throw new RuntimeException("No phoenix adapter found for config key: " + configuration.getKey());
+        } else {
+            logger.info("[{}]phoenix config mapping: {}", this, phoenixMapping.keySet());
+        }
+
+        for (Map.Entry<String, MappingConfig> entry : phoenixMapping.entrySet()) {
+            String configName = entry.getKey();
+            MappingConfig mappingConfig = entry.getValue();
+            String key;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+                        + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
+            } else {
+                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable().toLowerCase();
+            }
+            Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
+                    k1 -> new ConcurrentHashMap<>());
+            configMap.put(configName, mappingConfig);
+        }
+
+
+        Map<String, String> properties = configuration.getProperties();
+
+        DriverClass= properties.get("jdbc.driverClassName");
+        PhoenixUrl=properties.get("jdbc.url");
+
+        try {
+            //phoenix内部本身有连接池，不需要使用Druid初始化
+            phoenixPro.setProperty("hbase.rpc.timeout","600000");
+            phoenixPro.setProperty("hbase.client.scanner.timeout.period","600000");
+            phoenixPro.setProperty("dfs.client.socket-timeout","600000");
+            phoenixPro.setProperty("phoenix.query.keepAliveMs","600000");
+            phoenixPro.setProperty("phoenix.query.timeoutMs","3600000");
+            Class.forName(DriverClass);
+        } catch (ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        String threads = properties.get("threads");
+        phoenixSyncService = new PhoenixSyncService(
+                threads != null ? Integer.valueOf(threads) : null
+        );
+
+        phoenixConfigMonitor = new PhoenixConfigMonitor();
+        phoenixConfigMonitor.init(configuration.getKey(), this, envProperties);
+    }
+
+
+    /**
+     * 获取phoenix连接
+     * @return
+     */
+    public static  Connection  getPhoenixConnection() {
+        try {
+            return DriverManager.getConnection(PhoenixUrl,phoenixPro);
+        } catch (SQLException e) {
+            logger.error("getPhoenixConnection Exception"+e.getMessage());
+        }
+        return  null;
+    }
+
+
+
+    /**
+     * 同步方法
+     *
+     * @param dmls 数据包
+     */
+    @Override
+    public void sync(List<Dml> dmls) {
+        if (dmls == null || dmls.isEmpty()) {
+            return;
+        }
+        try {
+            phoenixSyncService.sync(mappingConfigCache, dmls, envProperties);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * ETL方法
+     *
+     * @param task   任务名, 对应配置名
+     * @param params etl筛选条件
+     * @return ETL结果
+     */
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        MappingConfig config = phoenixMapping.get(task);
+        if (config != null) {
+            DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+            if (srcDataSource != null) {
+                return PhoenixEtlService.importData(srcDataSource, getPhoenixConnection(), config, params);
+            } else {
+                etlResult.setSucceeded(false);
+                etlResult.setErrorMessage("DataSource not found");
+                return etlResult;
+            }
+        } else {
+            StringBuilder resultMsg = new StringBuilder();
+            boolean resSucc = true;
+            // ds不为空说明传入的是destination
+            for (MappingConfig configTmp : phoenixMapping.values()) {
+                // 取所有的destination为task的配置
+                if (configTmp.getDestination().equals(task)) {
+                    DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(configTmp.getDataSourceKey());
+                    if (srcDataSource == null) {
+                        continue;
+                    }
+                    EtlResult etlRes = PhoenixEtlService.importData(srcDataSource,getPhoenixConnection(), configTmp, params);
+                    if (!etlRes.getSucceeded()) {
+                        resSucc = false;
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
+                    } else {
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
+                    }
+                }
+            }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSucc);
+                if (resSucc) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
+        }
+        etlResult.setSucceeded(false);
+        etlResult.setErrorMessage("Task not found");
+        return etlResult;
+    }
+
+    /**
+     * 获取总数方法
+     *
+     * @param task 任务名, 对应配置名
+     * @return 总数
+     */
+    @Override
+    public Map<String, Object> count(String task) {
+        Map<String, Object> res = new LinkedHashMap<>();
+        MappingConfig config = phoenixMapping.get(task);
+        if (config == null) {
+            logger.info("[{}]phoenix config mapping: {}", this, phoenixMapping.keySet());
+            res.put("succeeded", false);
+            res.put("errorMessage", "Task[" + task + "] not found");
+            res.put("tasks", phoenixMapping.keySet());
+            return res;
+        }
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+        String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping);
+        Connection conn = null;
+        try {
+            //conn = dataSource.getConnection();
+            conn = getPhoenixConnection();
+            Util.sqlRS(conn, sql, rs -> {
+                try {
+                    if (rs.next()) {
+                        Long rowCount = rs.getLong("cnt");
+                        res.put("count", rowCount);
+                    }
+                } catch (SQLException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+        res.put("targetTable", SyncUtil.getDbTableName(dbMapping));
+
+        return res;
+    }
+
+    /**
+     * 获取对应canal instance name 或 mq topic
+     *
+     * @param task 任务名, 对应配置名
+     * @return destination
+     */
+    @Override
+    public String getDestination(String task) {
+        MappingConfig config = phoenixMapping.get(task);
+        if (config != null) {
+            return config.getDestination();
+        }
+        return null;
+    }
+
+    /**
+     * 销毁方法
+     */
+    @Override
+    public void destroy() {
+        if (phoenixConfigMonitor != null) {
+            phoenixConfigMonitor.destroy();
+        }
+
+        if (phoenixSyncService != null) {
+            phoenixSyncService.close();
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigLoader.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.config;
+
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Phoenix表映射配置加载器
+ */
+public class ConfigLoader {
+
+    private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
+
+    /**
+     * 加载Phoenix表映射配置
+     *
+     * @return 配置名/配置文件名--对象
+     */
+    public static Map<String, MappingConfig> load(Properties envProperties) {
+        logger.info("## Start loading phoenix mapping config ... ");
+
+        Map<String, MappingConfig> result = new LinkedHashMap<>();
+
+        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("phoenix");
+        configContentMap.forEach((fileName, content) -> {
+            MappingConfig config = YmlConfigBinder
+                    .bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
+            if (config == null) {
+                return;
+            }
+            try {
+                config.validate();
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
+            }
+            result.put(fileName, config);
+        });
+
+        logger.info("## Phoenix mapping config loaded");
+        logger.info("## Phoenix sync threads: " + ConfigurationManager.getInteger("threads"));
+        return result;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/ConfigurationManager.java

@@ -0,0 +1,49 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.config;
+
+import java.io.InputStream;
+import java.util.Properties;
+
+/**
+ * 配置管理组件
+ * @author Administrator
+ *
+ */
+public class ConfigurationManager {
+
+	private static Properties prop = new Properties();
+
+	static {
+		try {
+			InputStream in = ConfigurationManager.class
+					.getClassLoader().getResourceAsStream("phoenix/phoenix_common.properties");
+			prop.load(in);  
+		} catch (Exception e) {
+			e.printStackTrace();  
+		}
+	}
+	
+	/**
+	 * 获取指定key对应的value
+	 *
+	 * @param key 
+	 * @return 返回value是字符串
+	 */
+	public static String getProperty(String key) {
+		return prop.getProperty(key);
+	}
+	
+	/**
+	 * 获取整数类型的配置项
+	 * @param key StringKye
+	 * @return value
+	 */
+	public static Integer getInteger(String key) {
+		String value = getProperty(key);
+		try {
+			return Integer.valueOf(value);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		return 0;
+	}
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/config/MappingConfig.java

@@ -0,0 +1,288 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.config;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.*;
+
+/**
+ * Phoenix表映射配置
+ */
+@SuppressWarnings("unused")
+public class MappingConfig {
+
+    private String dataSourceKey;           // 数据源key
+    private String destination;             // canal实例或MQ的topic
+    private String groupId;                 // groupId
+    private String outerAdapterKey;         // 对应适配器的key
+    private boolean concurrent = false;     // 是否并行同步
+    private DbMapping dbMapping;            // db映射配置
+    private boolean debug = false;          // 调试
+
+    public String getDataSourceKey() {
+        return dataSourceKey;
+    }
+
+    public void setDataSourceKey(String dataSourceKey) {
+        this.dataSourceKey = dataSourceKey;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getOuterAdapterKey() {
+        return outerAdapterKey;
+    }
+
+    public void setOuterAdapterKey(String outerAdapterKey) {
+        this.outerAdapterKey = outerAdapterKey;
+    }
+
+    public boolean getConcurrent() {
+        return concurrent;
+    }
+
+    public void setConcurrent(boolean concurrent) {
+        this.concurrent = concurrent;
+    }
+
+    public DbMapping getDbMapping() {
+        return dbMapping;
+    }
+
+    public void setDbMapping(DbMapping dbMapping) {
+        this.dbMapping = dbMapping;
+    }
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public boolean isDebug() {
+        return debug;
+    }
+
+    public void setDebug(boolean debug) {
+        this.debug = debug;
+    }
+
+    public void validate() {
+        if (dbMapping.database == null || dbMapping.database.isEmpty()) {
+            throw new NullPointerException("dbMapping.database");
+        }
+        if ((dbMapping.table == null || dbMapping.table.isEmpty())) {
+            throw new NullPointerException("dbMapping.table");
+        }
+        if ((dbMapping.targetTable == null || dbMapping.targetTable.isEmpty())) {
+            throw new NullPointerException("dbMapping.targetTable");
+        }
+    }
+
+    public static class DbMapping {
+
+        private String database;                            // 数据库名或schema名
+        private String table;                               // 表名
+        private Map<String, String> targetPk = new LinkedHashMap<>(); // 目标表主键字段
+        private boolean mapAll = true;                      // 映射所有字段
+
+        private boolean alter = true;                       // 是否允许修改表
+        private boolean drop = false;                       // 是否允许删除字段
+        private boolean limit = false;                      // 是否限制字段长度
+        private boolean skipMissing = false;                // 是否跳过丢失的字段
+        private boolean escapeUpper = true;                 // 字段默认大写加双引号
+
+        private String targetDb;                            // 目标库名
+        private String targetTable;                         // 目标表名
+        private Map<String, String> targetColumns;          // 目标表字段映射
+
+        private List<String> excludeColumns;                // 不映射的字段
+
+        private String etlCondition;                        // etl条件sql
+        private int readBatch = 5000;
+        private int commitBatch = 5000;                     // etl等批量提交大小
+        private Map<String, String> allMapColumns;
+
+        public String escape(String name) {
+            if (escapeUpper) {
+                return "\"" + name.toUpperCase() + "\"";
+            } else {
+                return name;
+            }
+        }
+
+        public String getDatabase() {
+            return database;
+        }
+
+        public void setDatabase(String database) {
+            this.database = database;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
+        public void setTable(String table) {
+            this.table = table;
+        }
+
+        public Map<String, String> getTargetPk() {
+            return targetPk;
+        }
+
+        public void setTargetPk(Map<String, String> targetPk) {
+            this.targetPk = targetPk;
+        }
+
+        public Boolean getMapAll() {
+            return mapAll;
+        }
+
+        public void setMapAll(Boolean mapAll) {
+            this.mapAll = mapAll;
+        }
+
+        public boolean isAlter() {
+            return alter;
+        }
+
+        public void setAlter(boolean alter) {
+            this.alter = alter;
+        }
+
+        public boolean isLimit() {
+            return limit;
+        }
+
+        public void setLimit(boolean limit) {
+            this.limit = limit;
+        }
+
+        public boolean isDrop() {
+            return drop;
+        }
+
+        public void setDrop(boolean drop) {
+            this.drop = drop;
+        }
+
+        public boolean isSkipMissing() {
+            return skipMissing;
+        }
+
+        public void setSkipMissing(boolean skipMissing) {
+            this.skipMissing = skipMissing;
+        }
+
+        public boolean isEscapeUpper() {
+            return escapeUpper;
+        }
+
+        public void setEscapeUpper(boolean escapeUpper) {
+            this.escapeUpper = escapeUpper;
+        }
+
+        public String getTargetDb() {
+            return targetDb;
+        }
+
+        public void setTargetDb(String targetDb) {
+            this.targetDb = targetDb;
+        }
+
+        public String getTargetTable() {
+            return targetTable;
+        }
+
+        public void setTargetTable(String targetTable) {
+            this.targetTable = targetTable;
+        }
+
+        public Map<String, String> getTargetColumns() {
+            if (targetColumns != null) {
+                targetColumns.forEach((key, value) -> {
+                    if (StringUtils.isEmpty(value)) {
+                        targetColumns.put(key, key);
+                    }
+                });
+            } else {
+                targetColumns = new HashMap<>();
+            }
+            return targetColumns;
+        }
+
+        public void setTargetColumns(Map<String, String> targetColumns) {
+            this.targetColumns = targetColumns;
+        }
+
+        public void addTargetColumn(String key, String value) {
+            if (targetColumns == null) {
+                targetColumns = new HashMap<>();
+            }
+            targetColumns.put(key, value);
+            if (allMapColumns != null) {
+                allMapColumns.put(key, value);
+            }
+        }
+
+        public void removeTargetColumn(String key) {
+            if (targetColumns != null) {
+                targetColumns.remove(key);
+            }
+            if (allMapColumns != null) {
+                allMapColumns.remove(key);
+            }
+        }
+
+        public List<String> getExcludeColumns() {
+            if (excludeColumns == null) {
+                excludeColumns = new ArrayList<>();
+            }
+            return excludeColumns;
+        }
+
+        public void setExcludeColumns(List<String> excludeColumns) {
+            this.excludeColumns = excludeColumns;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public int getReadBatch() {
+            return readBatch;
+        }
+
+        public void setReadBatch(int readBatch) {
+            this.readBatch = readBatch;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public Map<String, String> getAllMapColumns() {
+            return allMapColumns;
+        }
+
+        public void setAllMapColumns(Map<String, String> allMapColumns) {
+            this.allMapColumns = allMapColumns;
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/monitor/PhoenixConfigMonitor.java

@@ -0,0 +1,167 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.monitor;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * phoenix config monitor
+ */
+public class PhoenixConfigMonitor {
+
+    private static final Logger logger = LoggerFactory.getLogger(PhoenixConfigMonitor.class);
+
+    private static final String adapterName = "phoenix";  //相应组件名字
+
+    private String key;
+
+    private PhoenixAdapter phoenixAdapter;   //相应适配器名实现类
+
+    private Properties envProperties;
+
+    private FileAlterationMonitor fileMonitor;
+
+    public void init(String key, PhoenixAdapter phoenixAdapter, Properties envProperties) {
+        this.key = key;
+        this.phoenixAdapter = phoenixAdapter;
+        this.envProperties = envProperties;
+        File confDir = Util.getConfDirPath(adapterName);
+        try {
+            FileAlterationObserver observer = new FileAlterationObserver(confDir,
+                    FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+            FileListener listener = new FileListener();
+            observer.addListener(listener);
+            fileMonitor = new FileAlterationMonitor(3000, observer);
+            fileMonitor.start();
+
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    public void destroy() {
+        try {
+            fileMonitor.stop();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private class FileListener extends FileAlterationListenerAdaptor {
+
+        @Override
+        public void onFileCreate(File file) {
+            super.onFileCreate(file);
+            try {
+                // 加载新增的配置文件
+                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
+                MappingConfig config = YmlConfigBinder
+                        .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                if (config == null) {
+                    return;
+                }
+                config.validate();
+                if ((key == null && config.getOuterAdapterKey() == null)
+                        || (key != null && key.equals(config.getOuterAdapterKey()))) {
+                    addConfigToCache(file, config);
+
+                    logger.info("Add a new phoenix mapping config: {} to canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileChange(File file) {
+            super.onFileChange(file);
+
+            try {
+                if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
+                    // 加载配置文件
+                    String configContent = MappingConfigsLoader
+                            .loadConfig(adapterName + File.separator + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
+                    MappingConfig config = YmlConfigBinder
+                            .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                    if (config == null) {
+                        return;
+                    }
+                    config.validate();
+                    if ((key == null && config.getOuterAdapterKey() == null)
+                            || (key != null && key.equals(config.getOuterAdapterKey()))) {
+                        if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
+                            deleteConfigFromCache(file);
+                        }
+                        addConfigToCache(file, config);
+                    } else {
+                        // 不能修改outerAdapterKey
+                        throw new RuntimeException("Outer adapter key not allowed modify");
+                    }
+                    logger.info("Change a phoenix mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileDelete(File file) {
+            super.onFileDelete(file);
+
+            try {
+                if (phoenixAdapter.getPhoenixMapping().containsKey(file.getName())) {
+                    deleteConfigFromCache(file);
+
+                    logger.info("Delete a phoenix mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        private void addConfigToCache(File file, MappingConfig mappingConfig) {
+            if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
+                return;
+            }
+            phoenixAdapter.getPhoenixMapping().put(file.getName(), mappingConfig);
+            Map<String, MappingConfig> configMap = phoenixAdapter.getMappingConfigCache()
+                    .computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                                    + mappingConfig.getDbMapping().getDatabase() + "-"
+                                    + mappingConfig.getDbMapping().getTable().toLowerCase(),
+                            k1 -> new HashMap<>());
+            configMap.put(file.getName(), mappingConfig);
+        }
+
+        private void deleteConfigFromCache(File file) {
+            logger.info("deleteConfigFromCache: {}", file.getName());
+            MappingConfig mappingConfig = phoenixAdapter.getPhoenixMapping().remove(file.getName());
+
+            if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
+                return;
+            }
+            for (Map<String, MappingConfig> configMap : phoenixAdapter.getMappingConfigCache().values()) {
+                if (configMap != null) {
+                    configMap.remove(file.getName());
+                }
+            }
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/service/PhoenixEtlService.java

@@ -0,0 +1,429 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.service;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.PhoenixSupportUtil;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.TypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.google.common.base.Joiner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.*;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Phoenix ETL 操作业务类
+ */
+public class PhoenixEtlService {
+
+    private static final Logger logger = LoggerFactory.getLogger(PhoenixEtlService.class);
+
+    private static String[] splitNotEmpty(String s) {
+        if (s != null && s.trim().length() > 0) {
+            return s.trim().split(",");
+        }
+        return new String[]{};
+    }
+
+
+    static boolean syncSchema(Connection targetDSConnection, MappingConfig config) {
+        DataSource srcDataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        if (srcDataSource == null) {
+            return false;
+        }
+        try {
+            return syncSchema(srcDataSource.getConnection(), targetDSConnection, config);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static boolean syncSchema(DataSource srcDS,Connection targetDSConnection, MappingConfig config) {
+        try {
+            return syncSchema(srcDS.getConnection(),targetDSConnection, config);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static boolean syncSchema(Connection srcDS, Connection targetDS, MappingConfig config) {
+        DbMapping dbMapping = config.getDbMapping();
+        if (dbMapping.getMapAll() && dbMapping.isAlter()) { // 检查字段是否缺失
+            Map<String, Integer> targetColumnType = new LinkedHashMap<>();
+            String targetTable = SyncUtil.getDbTableName(dbMapping);
+            try {
+                Util.sqlRS(targetDS, "SELECT * FROM " + targetTable + " LIMIT 1", rs -> {
+                    try {
+                        ResultSetMetaData rsd = rs.getMetaData();
+                        int columnCount = rsd.getColumnCount();
+                        for (int i = 1; i <= columnCount; i++) {
+                            targetColumnType.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                        }
+                    } catch (Exception e) {
+                        logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
+                    }
+                });
+            } catch (RuntimeException e) {
+                if (!e.getCause().getClass().getName().endsWith("TableNotFoundException")) {
+                    throw e;
+                }
+            }
+            StringBuilder missing = new StringBuilder();
+            StringBuilder constraint = new StringBuilder();
+            Util.sqlRS(srcDS, "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '" + dbMapping.getDatabase() + "'  AND TABLE_NAME = '" + dbMapping.getTable() + "'", rs -> {
+                try {
+                    List<String> excludeColumns = config.getDbMapping().getExcludeColumns();
+                    while (rs.next()) {
+                        String name = rs.getString("COLUMN_NAME");
+                        String lower = name.toLowerCase();
+                        String colType = rs.getString("COLUMN_TYPE");
+                        if (targetColumnType.get(lower) == null && !excludeColumns.contains(lower)) {
+                            boolean isPri = rs.getString("COLUMN_KEY").equals("PRI");
+                            String[] args = splitNotEmpty(colType.replaceAll("^\\w+(?:\\(([^)]*)\\))?[\\s\\S]*$", "$1"));
+                            missing.append(dbMapping.escape(name)).append(" ").append(TypeUtil.getPhoenixType(
+                                    rs.getString("DATA_TYPE").toUpperCase(),
+                                    args,
+                                    colType.contains("unsigned"),
+                                    dbMapping.isLimit()
+                            ));
+                            if (isPri) {
+                                if (args.length > 0 && dbMapping.isLimit() || rs.getString("IS_NULLABLE").equals("NO")) {
+                                    missing.append(" NOT NULL");
+                                }
+                                constraint.append(dbMapping.escape(name)).append(',');
+                            }
+                            missing.append(',');
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.error(dbMapping.getDatabase() + "." + dbMapping.getTable() + " schema failed! ==>" + e.getMessage(), e);
+                    throw new RuntimeException(e);
+                }
+            });
+            if (missing.length() > 0) {
+                String sql;
+                if (targetColumnType.isEmpty()) {
+                    if (constraint.length() > 0) {
+                        constraint.deleteCharAt(constraint.length() - 1);
+                        missing.append("CONSTRAINT pk PRIMARY KEY(").append(constraint.toString()).append(")");
+                    } else {
+                        missing.deleteCharAt(missing.length() - 1);
+                    }
+                    sql = "CREATE TABLE " + targetTable + " (" + missing.toString() + ")";
+                } else {
+                    missing.deleteCharAt(missing.length() - 1);
+                    sql = "ALTER TABLE " + targetTable + " ADD " + missing.toString();
+                }
+                logger.info("schema missing: {} {}", targetColumnType, sql);
+                try (PreparedStatement pstmt = targetDS.prepareStatement(sql)) {
+                    pstmt.executeUpdate();
+                } catch (SQLException e) {
+                    logger.error("sync schema error: " + e.getMessage(), e);
+                }
+            } else {
+                logger.debug("schema ok: {}", targetColumnType);
+            }
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * 导入数据
+     */
+    public static EtlResult importData(DataSource srcDS, Connection targetDSConnection, MappingConfig config,
+                                       List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        AtomicLong successCount = new AtomicLong();
+        List<String> errMsg = new ArrayList<>();
+        String hbaseTable = "";
+        try {
+            if (config == null) {
+                logger.error("Config is null!");
+                etlResult.setSucceeded(false);
+                etlResult.setErrorMessage("Config is null!");
+                return etlResult;
+            }
+            boolean debug = params != null && params.get(0).equals("_debug");
+            if (debug) {
+                params = params.subList(1, params.size());
+            }
+            syncSchema(srcDS, targetDSConnection, config);
+            DbMapping dbMapping = config.getDbMapping();
+
+            long start = System.currentTimeMillis();
+
+            // 拼接sql
+            StringBuilder sql = new StringBuilder(
+                    "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable());
+
+            // 拼接条件
+            appendCondition(params, dbMapping, srcDS, sql);
+
+            // 获取总数
+            String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
+            long cnt = (Long) Util.sqlRS(srcDS, countSql, rs -> {
+                Long count = null;
+                try {
+                    if (rs.next()) {
+                        count = ((Number) rs.getObject(1)).longValue();
+                    }
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                }
+                return count == null ? 0 : count;
+            });
+
+            // 当大于1万条记录时开启多线程
+            if (cnt >= 10000) {
+                int threadCount = 3;
+                long perThreadCnt = cnt / threadCount;
+                ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
+                for (int i = 0; i < threadCount; i++) {
+                    long offset = i * perThreadCnt;
+                    Long size = null;
+                    if (i != threadCount - 1) {
+                        size = perThreadCnt;
+                    }
+                    String sqlFinal;
+                    if (size != null) {
+                        sqlFinal = sql + " LIMIT " + offset + "," + size;
+                    } else {
+                        sqlFinal = sql + " LIMIT " + offset + "," + cnt;
+                    }
+                    executor
+                            .execute(() -> executeSqlImport(srcDS, targetDSConnection, sqlFinal, dbMapping, successCount, errMsg, debug));
+                }
+
+                executor.shutdown();
+                //noinspection StatementWithEmptyBody
+                while (!executor.awaitTermination(3, TimeUnit.SECONDS)) ;
+            } else {
+                executeSqlImport(srcDS, targetDSConnection, sql.toString(), dbMapping, successCount, errMsg, debug);
+            }
+
+            logger.info(
+                    dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
+
+            etlResult
+                    .setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
+        }
+
+        if (errMsg.isEmpty()) {
+            etlResult.setSucceeded(true);
+        } else {
+            etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
+        }
+        return etlResult;
+    }
+
+    private static void appendCondition(List<String> params, DbMapping dbMapping, DataSource ds,
+                                        StringBuilder sql) {
+        if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) {
+            AtomicBoolean stExists = new AtomicBoolean(false);
+            // 验证是否有SYS_TIME字段
+            Util.sqlRS(ds, sql.toString(), rs -> {
+                try {
+                    ResultSetMetaData rsmd = rs.getMetaData();
+                    int cnt = rsmd.getColumnCount();
+                    for (int i = 1; i <= cnt; i++) {
+                        String columnName = rsmd.getColumnName(i);
+                        if ("SYS_TIME".equalsIgnoreCase(columnName)) {
+                            stExists.set(true);
+                            break;
+                        }
+                    }
+                } catch (Exception e) {
+                    // ignore
+                }
+                return null;
+            });
+            if (stExists.get()) {
+                sql.append(" WHERE SYS_TIME >= '").append(params.get(0)).append("' ");
+            }
+        } else if (dbMapping.getEtlCondition() != null && params != null && params.size() > 0) {
+            String etlCondition = dbMapping.getEtlCondition();
+            int size = params.size();
+            for (int i = 0; i < size; i++) {
+                etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
+            }
+
+            sql.append(" ").append(etlCondition);
+        }
+    }
+
+    /**
+     * 执行导入
+     */
+    private static boolean executeSqlImport(DataSource srcDS, Connection targetDSConnection, String sql, DbMapping dbMapping,
+                                            AtomicLong successCount, List<String> errMsg, boolean debug) {
+        try {
+            Map<String, String> columnsMap = new LinkedHashMap<>();
+            Map<String, Integer> columnType = new LinkedHashMap<>();
+
+
+            PhoenixSupportUtil.sqlRS(targetDSConnection, "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " LIMIT 1 ", rs -> {
+                try {
+
+                    ResultSetMetaData rsd = rs.getMetaData();
+                    int columnCount = rsd.getColumnCount();
+                    List<String> columns = new ArrayList<>();
+                    List<String> excludeColumns = dbMapping.getExcludeColumns();
+                    for (int i = 1; i <= columnCount; i++) {
+                        String lower = rsd.getColumnName(i).toLowerCase();
+                        if (!excludeColumns.contains(lower)) {
+                            columnType.put(lower, rsd.getColumnType(i));
+                            columns.add(lower);
+                        }
+                    }
+                    columnsMap.putAll(SyncUtil.getColumnsMap(dbMapping, columns));
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            });
+            Util.sqlRS(srcDS, sql, rs -> {
+                int idx = 1;
+
+                try {
+                    boolean completed = false;
+
+                    // if (dbMapping.isMapAll()) {
+                    // columnsMap = dbMapping.getAllColumns();
+                    // } else {
+                    // columnsMap = dbMapping.getTargetColumns();
+                    // }
+
+                    StringBuilder insertSql = new StringBuilder();
+                    insertSql.append("UPSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
+                    columnsMap
+                            .forEach((targetColumnName, srcColumnName) -> insertSql.append(dbMapping.escape(targetColumnName)).append(","));
+
+                    int len = insertSql.length();
+                    insertSql.delete(len - 1, len).append(") VALUES (");
+                    int mapLen = columnsMap.size();
+                    for (int i = 0; i < mapLen; i++) {
+                        insertSql.append("?,");
+                    }
+                    len = insertSql.length();
+                    insertSql.delete(len - 1, len).append(")");
+                    try (
+                         //Connection connTarget = targetDS.getConnection();
+                         Connection connTarget =PhoenixAdapter.getPhoenixConnection();
+                         PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
+                        connTarget.setAutoCommit(false);
+
+                        while (rs.next()) {
+                            completed = false;
+
+                            pstmt.clearParameters();
+
+                            // 删除数据
+                            Map<String, Object> values = new LinkedHashMap<>();
+                            StringBuilder deleteSql = new StringBuilder(
+                                    "DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE ");
+                            appendCondition(dbMapping, deleteSql, values, rs);
+                            try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
+                                int k = 1;
+                                for (Object val : values.values()) {
+                                    pstmt2.setObject(k++, val);
+                                }
+                                pstmt2.execute();
+                            }
+
+                            Map<String, Object> insertValues = new HashMap<>();
+                            int i = 1;
+                            for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+                                String targetClolumnName = entry.getKey();
+                                String srcColumnName = entry.getValue();
+                                if (srcColumnName == null) {
+                                    srcColumnName = targetClolumnName;
+                                }
+
+                                Integer type = columnType.get(targetClolumnName.toLowerCase());
+
+                                try {
+                                    Object value = rs.getObject(srcColumnName);
+                                    insertValues.put(srcColumnName, value);
+                                    if (value != null) {
+                                        SyncUtil.setPStmt(type, pstmt, value, i);
+                                    } else {
+                                        pstmt.setNull(i, type);
+                                    }
+                                } catch (SQLException e) {
+                                    insertValues.put(srcColumnName, null);
+                                    pstmt.setNull(i, type);
+                                }
+                                i++;
+                            }
+                            if (debug) {
+                                logger.info("insert sql: {} {} {}", insertSql, insertValues, pstmt);
+                            }
+
+                            pstmt.execute();
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("Insert into target table, sql: {}", insertSql);
+                            }
+
+                            if (idx % dbMapping.getCommitBatch() == 0) {
+                                connTarget.commit();
+                                completed = true;
+                            }
+                            idx++;
+                            successCount.incrementAndGet();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("successful import count:" + successCount.get());
+                            }
+                        }
+                        if (!completed) {
+                            connTarget.commit();
+                        }
+                    }
+
+                } catch (Exception e) {
+                    logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
+                    errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage());
+                }
+                return idx;
+            });
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+    }
+
+    /**
+     * 拼接目标表主键where条件
+     */
+    private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
+                                        ResultSet rs) throws SQLException {
+        // 拼接主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = targetColumnName;
+            }
+            sql.append(dbMapping.escape(targetColumnName)).append("=? AND ");
+            values.put(targetColumnName, rs.getObject(srcColumnName));
+        }
+        int len = sql.length();
+        sql.delete(len - 4, len);
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/service/PhoenixSyncService.java

@@ -0,0 +1,644 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.service;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLName;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.ast.statement.*;
+import com.alibaba.druid.sql.parser.ParserException;
+import com.alibaba.druid.util.JdbcConstants;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.ConfigurationManager;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.BatchExecutor;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SingleDml;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.phoenix.support.TypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.function.Function;
+
+/**
+ * Phoenix同步操作业务
+ */
+public class PhoenixSyncService {
+
+    private static final Logger logger = LoggerFactory.getLogger(PhoenixSyncService.class);
+
+    // 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
+    private Map<String, Map<String, Integer>> columnsTypeCache;
+
+    //同步线程数
+    //默认开启3个线程同步,此处配置了自定义同步线程数
+    private int threads = ConfigurationManager.getInteger("threads");
+
+    private List<SyncItem>[] dmlsPartition;
+    private BatchExecutor[] batchExecutors;
+    private ExecutorService[] executorThreads;
+
+    public PhoenixSyncService(Integer threads) {
+        this(threads, new ConcurrentHashMap<>());
+    }
+
+
+    @SuppressWarnings("unchecked")
+    private PhoenixSyncService(Integer threads, Map<String, Map<String, Integer>> columnsTypeCache) {
+        this.columnsTypeCache = columnsTypeCache;
+        try {
+            if (threads != null) {
+                this.threads = threads;
+            }
+            this.dmlsPartition = new List[this.threads];
+            this.batchExecutors = new BatchExecutor[this.threads];
+            this.executorThreads = new ExecutorService[this.threads];
+            for (int i = 0; i < this.threads; i++) {
+                dmlsPartition[i] = new ArrayList<>();
+                batchExecutors[i] = new BatchExecutor();
+                //创建单个线程，用来操作一个无界的队列任务，不会使用额外的线程。如果线程崩溃会重新创建一个，直到任务完成。
+                executorThreads[i] = Executors.newSingleThreadExecutor();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 批量同步回调
+     *
+     * @param dmls     批量 DML
+     * @param function 回调方法
+     */
+    private void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
+        try {
+            boolean toExecute = false;
+            for (Dml dml : dmls) {
+                if (!toExecute) {
+                    toExecute = function.apply(dml);
+                } else {
+                    function.apply(dml);
+                }
+            }
+            if (toExecute) {
+                List<Future<Boolean>> futures = new ArrayList<>();
+                for (int i = 0; i < threads; i++) {
+                    int j = i;
+                    if (dmlsPartition[j].isEmpty()) {
+                        // bypass
+                        continue;
+                    }
+
+                    futures.add(executorThreads[i].submit(() -> {
+                        try {
+                            dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
+                                    syncItem.config,
+                                    syncItem.singleDml));
+                            //相对于RDB同步 少了  dmlsPartition[j].clear();
+                            //在 try catch中获取异常后再次执行一次batchExecutors[j].commit()
+                            batchExecutors[j].commit();
+                            return true;
+                        } catch (Throwable e) {
+                            batchExecutors[j].rollback();
+                            if (!e.getClass().getName().endsWith("ColumnNotFoundException")
+                                    && !e.getClass().getName().endsWith("TableNotFoundException")) {
+                                throw new RuntimeException(e);
+                            }
+                            logger.info("table or column not found: " + e.getMessage());
+                            boolean synced = false;
+                            for (SyncItem syncItem : dmlsPartition[j]) {
+                                if (PhoenixEtlService.syncSchema(batchExecutors[j].getConn(), syncItem.config)) {
+                                    synced = true;
+                                }
+                            }
+                            if (!synced) {
+                                throw new RuntimeException(e);
+                            }
+                            dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j],
+                                    syncItem.config,
+                                    syncItem.singleDml));
+                            try {
+                                batchExecutors[j].commit();
+                                return true;
+                            } catch (Throwable e1) {
+                                batchExecutors[j].rollback();
+                                throw new RuntimeException(e1);
+                            }
+                        } finally {
+                            dmlsPartition[j].clear();
+                        }
+                    }));
+                }
+                futures.forEach(future -> {
+                    try {
+                        future.get();
+                    } catch (ExecutionException | InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+            }
+        } finally {
+            for (BatchExecutor batchExecutor : batchExecutors) {
+                if (batchExecutor != null) {
+                    batchExecutor.close();
+                }
+            }
+        }
+    }
+
+    /**
+     * 批量同步 :将批量DML进行解析并放入List<SingleDml> --> dmlsPartition[hash].add(syncItem);
+     * @param mappingConfig 配置集合
+     * @param dmls   批量 DML
+     */
+    public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls, Properties envProperties) {
+        sync(dmls, dml -> {
+            String destination = StringUtils.trimToEmpty(dml.getDestination());
+            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
+            String database = dml.getDatabase();
+            String table = dml.getTable().toLowerCase();
+            Map<String, MappingConfig> configMap;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                //tcp 模式
+                configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
+            } else {
+                //kafka 模式 或者 RocketMQ模式
+                configMap = mappingConfig.get(destination + "_" + database + "-" + table);
+            }
+
+            if (configMap == null) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace("no config map: destination={},groupId={}, database={}, table={}, keys={}", destination, groupId, database, table, mappingConfig.keySet());
+                }
+                return false;
+            }
+            if (configMap.values().isEmpty()) {
+                logger.info("config map empty: destination={},groupId={}, database={}, table={}, keys={}", destination, groupId, database, table, mappingConfig.keySet());
+                return false;
+            }
+            if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
+                // DDL
+                columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
+                List<SQLStatement> stmtList;
+                try {
+                    stmtList = SQLUtils.parseStatements(dml.getSql(), JdbcConstants.MYSQL, false);
+                } catch (ParserException e) {
+                    // 可能存在一些SQL是不支持的，比如存储过程
+                    logger.info("parse sql error: " + dml.getSql(), e);
+                    return false;
+                }
+                for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
+                    try {
+                        alter(batchExecutors[0], entry.getValue(), dml, stmtList, entry.getKey());
+                    } catch (SQLException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                return false;
+            } else {
+                // DML
+                for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
+                    MappingConfig config = entry.getValue();
+                    if (config.isDebug()) {
+                        logger.info("DML: {} {}", entry.getKey(), JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                    }
+                    if (config.getConcurrent()) {
+                        //并行同步
+                        //将一批DML转成SingleDml
+                        List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                        singleDmls.forEach(singleDml -> {
+                            //取主键hash
+                            int hash = pkHash(config.getDbMapping(), singleDml.getData());
+                            SyncItem syncItem = new SyncItem(config, singleDml);
+                            //相同的主键数据的顺序是可以保证的
+                            dmlsPartition[hash].add(syncItem);
+                        });
+                    } else {
+                        //不并行同步
+                        int hash = 0;
+                        List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
+                        singleDmls.forEach(singleDml -> {
+                            SyncItem syncItem = new SyncItem(config, singleDml);
+                            //这里线程默认是3个,如果不并行，则会出现2个线程空跑
+                            dmlsPartition[hash].add(syncItem);
+                        });
+                    }
+                }
+                return true;
+            }
+        });
+    }
+
+    /**
+     * 单条 dml 同步
+     *
+     * @param batchExecutor 批量事务执行器
+     * @param config        对应配置对象
+     * @param dml           DML
+     */
+    private void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) {
+        if (config != null) {
+            try {
+                String type = dml.getType();
+                if (type != null && type.equalsIgnoreCase("INSERT")) {
+                    insert(batchExecutor, config, dml);
+                } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
+                    insert(batchExecutor, config, dml);
+                } else if (type != null && type.equalsIgnoreCase("DELETE")) {
+                    delete(batchExecutor, config, dml);
+                } else if (type != null && type.equalsIgnoreCase("TRUNCATE")) {
+                    truncate(batchExecutor, config);
+                } else if (logger.isInfoEnabled()){
+                    logger.info("SingleDml: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+                }
+            } catch (SQLException e) {
+                logger.error("sync error: " + e.getMessage(), e);
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    private void alter(BatchExecutor batchExecutor, MappingConfig config, Dml dml, List<SQLStatement> stmtList, String configFile) throws SQLException {
+        if (config.isDebug()) {
+            logger.info("DML: {} {}", configFile, JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+        }
+        DbMapping dbMapping = config.getDbMapping();
+        if (!dbMapping.isAlter()) {
+            logger.info("not alterable table: {} {}", dml.getTable(), configFile);
+            return;
+        }
+
+        Map<String, String> columnsMap = dbMapping.getTargetColumns();
+
+        Map<String, String> columnsMap1 = new HashMap<>();
+        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+            columnsMap1.put(entry.getValue(), entry.getKey());
+        }
+
+        String targetTable = SyncUtil.getDbTableName(dbMapping);
+        Map<String, String> defValues = new HashMap<>();
+        for (SQLStatement statement : stmtList) {
+            if (statement instanceof SQLAlterTableStatement) {
+                SQLAlterTableStatement alterTable = (SQLAlterTableStatement) statement;
+                for (SQLAlterTableItem item : alterTable.getItems()) {
+                    if (item instanceof SQLAlterTableDropColumnItem) {
+                        SQLAlterTableDropColumnItem dropColumnItem = (SQLAlterTableDropColumnItem) item;
+                        if (!dbMapping.isDrop()) {
+                            logger.info("drop table column disabled: {} {}", targetTable, dropColumnItem.getColumns());
+                            continue;
+                        }
+                        for (SQLName sqlName : dropColumnItem.getColumns()) {
+                            String name = Util.cleanColumn(sqlName.getSimpleName());
+                            String sql = "ALTER TABLE " + targetTable + " DROP COLUMN IF EXISTS " +
+                                    dbMapping.escape(columnsMap1.getOrDefault(name, name));
+                            try {
+                                logger.info("drop table column: {} {}", sql, batchExecutor.executeUpdate(sql));
+                                dbMapping.removeTargetColumn(name);
+                            } catch (Exception e) {
+                                logger.warn("drop table column error: " + sql, e);
+                            }
+                        }
+                    } else if (item instanceof SQLAlterTableAddColumn) {
+                        SQLAlterTableAddColumn addColumn = (SQLAlterTableAddColumn) item;
+                        if (!dbMapping.getMapAll()) {
+                            logger.info("add table column disabled: {} {}", targetTable, addColumn.getColumns());
+                            continue;
+                        }
+                        for (SQLColumnDefinition definition : addColumn.getColumns()) {
+                            String name = Util.cleanColumn(definition.getNameAsString());
+                            if (dbMapping.getExcludeColumns().contains(name)) {
+                                continue;
+                            }
+                            String sql = "ALTER TABLE " + targetTable +
+                                    " ADD IF NOT EXISTS " +
+                                    dbMapping.escape(name) + " " + TypeUtil.getPhoenixType(definition, dbMapping.isLimit());
+                            try {
+                                logger.info("add table column: {} {}", sql, batchExecutor.executeUpdate(sql));
+                                dbMapping.addTargetColumn(name, name);
+                                if (definition.getDefaultExpr() != null) {
+                                    String defVal = definition.getDefaultExpr().toString();
+                                    if (!defVal.equalsIgnoreCase("NULL") && !defVal.equalsIgnoreCase("NOT NULL") && name.length() > 0) {
+                                        defValues.put(name, defVal);
+                                    }
+                                }
+                            } catch (Exception e) {
+                                logger.error("add table column error: " + sql, e);
+                                throw e;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        if (!defValues.isEmpty()) {
+            StringBuilder defSql = new StringBuilder();
+            defSql.append("UPSERT INTO ").append(targetTable).append("(");
+            Set<Map.Entry<String, String>> pkSet = dbMapping.getTargetPk().entrySet();
+            Set<Map.Entry<String, String>> defSet = defValues.entrySet();
+            for (Map.Entry<String, String> entry : pkSet) {
+                defSql.append(dbMapping.escape(entry.getKey())).append(",");
+            }
+            for (Map.Entry<String, String> entry : defSet) {
+                defSql.append(dbMapping.escape(entry.getKey())).append(",");
+            }
+            defSql.deleteCharAt(defSql.length() - 1).append(") SELECT ");
+            for (Map.Entry<String, String> entry : pkSet) {
+                defSql.append(dbMapping.escape(entry.getKey())).append(",");
+            }
+            for (Map.Entry<String, String> entry : defSet) {
+                defSql.append(entry.getValue()).append(",");
+            }
+            defSql.deleteCharAt(defSql.length() - 1).append(" FROM ").append(targetTable);
+            try {
+                logger.info("set column default value: {} {}", defSql, batchExecutor.executeUpdate(defSql.toString()));
+                batchExecutor.commit();
+            } catch (SQLException e) {
+                logger.error("set column default value error: {}", defSql, e);
+                batchExecutor.rollback();
+                throw e;
+            }
+        }
+    }
+
+    /**
+     * 插入操作
+     *
+     * @param config 配置项
+     * @param dml    DML数据
+     */
+    private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
+        Map<String, Object> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+        Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
+
+        StringBuilder insertSql = new StringBuilder();
+        insertSql.append("UPSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
+
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        int mapLen = columnsMap.size();
+        List<Map<String, ?>> values = new ArrayList<>();
+        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+
+            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                if (dbMapping.isSkipMissing()) {
+                    logger.warn("Target missing field: {}", targetColumnName);
+                    mapLen -= 1;
+                    continue;
+                } else if (dbMapping.getMapAll() && dbMapping.isAlter() && PhoenixEtlService.syncSchema(batchExecutor.getConn(), config)) {
+                    columnsTypeCache.remove(config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable());
+                    ctype = getTargetColumnType(batchExecutor.getConn(), config);
+                    type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+                }
+                if (type == null) {
+                    throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+                }
+            }
+            insertSql.append(dbMapping.escape(targetColumnName)).append(",");
+            Object value = data.get(srcColumnName);
+            BatchExecutor.setValue(values, type, value);
+        }
+
+        int len = insertSql.length();
+        insertSql.delete(len - 1, len).append(") VALUES (");
+        for (int i = 0; i < mapLen; i++) {
+            insertSql.append("?,");
+        }
+        len = insertSql.length();
+        insertSql.delete(len - 1, len).append(")");
+
+        Map<String, Object> old = dml.getOld();
+        try {
+            if (old != null && !old.isEmpty()) {
+                boolean keyChanged = false;
+                List<Map<String, ?>> delValues = new ArrayList<>();
+                StringBuilder deleteSql = new StringBuilder();
+                deleteSql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE ");
+                for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+                    String targetColumnName = entry.getKey();
+                    String srcColumnName = entry.getValue();
+                    if (srcColumnName == null) {
+                        srcColumnName = Util.cleanColumn(targetColumnName);
+                    }
+                    Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+                    if (type != null) {
+                        deleteSql.append(dbMapping.escape(targetColumnName)).append("=? AND ");
+                        // 如果有修改主键的情况
+                        if (old.containsKey(srcColumnName)) {
+                            keyChanged = true;
+                            BatchExecutor.setValue(delValues, type, old.get(srcColumnName));
+                        } else {
+                            BatchExecutor.setValue(delValues, type, data.get(srcColumnName));
+                        }
+                    }
+                }
+                if (keyChanged) {
+                    if (config.isDebug()) {
+                        logger.info("insert into table: {} {}", deleteSql, delValues);
+                    }
+                    batchExecutor.execute(deleteSql.toString(), delValues);
+                }
+            }
+            if (config.isDebug()) {
+                logger.info("insert into table: {} {}", insertSql, values);
+            }
+            batchExecutor.execute(insertSql.toString(), values);
+        } catch (SQLException | RuntimeException e) {
+            logger.warn("Insert into target table, sql: {} {}", insertSql, values ,e);
+            throw e;
+        }
+        if (logger.isTraceEnabled()) {
+            logger.trace("Insert into target table, sql: {}", insertSql);
+        }
+    }
+
+    /**
+     * 删除操作 没有改动
+     *
+     * @param config MappingConfig
+     * @param dml    Single DML
+     */
+    private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
+        Map<String, Object> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        StringBuilder sql = new StringBuilder();
+        sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE ");
+
+        List<Map<String, ?>> values = new ArrayList<>();
+        // 拼接主键
+        appendCondition(dbMapping, sql, ctype, values, data);
+        try {
+            batchExecutor.execute(sql.toString(), values);
+            if (logger.isTraceEnabled()) {
+                logger.trace("Delete from target table, sql: {}", sql);
+            }
+        } catch (SQLException e) {
+            logger.warn("Delete from target error, sql: {} {}", sql, values);
+            throw e;
+        }
+    }
+
+    /**
+     * truncate操作   没有改动
+     *
+     * @param config MappingConfig
+     */
+    private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException {
+        DbMapping dbMapping = config.getDbMapping();
+        StringBuilder sql = new StringBuilder();
+        sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping));
+        batchExecutor.execute(sql.toString(), new ArrayList<>());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Truncate target table, sql: {}", sql);
+        }
+    }
+
+    /**
+     * 获取目标字段类型
+     *
+     * @param conn   sql connection
+     * @param config 映射配置
+     * @return 字段sqlType
+     */
+    private Map<String, Integer> getTargetColumnType(Connection conn, MappingConfig config) {
+        DbMapping dbMapping = config.getDbMapping();
+        String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable();
+        Map<String, Integer> columnType = columnsTypeCache.get(cacheKey);
+        if (columnType == null) {
+            synchronized (PhoenixSyncService.class) {
+                columnType = columnsTypeCache.get(cacheKey);
+                if (columnType == null) {
+                    columnType = new LinkedHashMap<>();
+                    final Map<String, Integer> columnTypeTmp = columnType;
+                    String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2";
+                    try {
+                        Util.sqlRS(conn, sql, rs -> {
+                            try {
+                                ResultSetMetaData rsd = rs.getMetaData();
+                                int columnCount = rsd.getColumnCount();
+                                for (int i = 1; i <= columnCount; i++) {
+                                    columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                                }
+                                columnsTypeCache.put(cacheKey, columnTypeTmp);
+                            } catch (SQLException e) {
+                                logger.error(e.getMessage(), e);
+                            }
+                        });
+                    } catch (RuntimeException e) {
+                        //新增catch 里面做了操作
+                        if (!e.getCause().getClass().getName().endsWith("TableNotFoundException")) {
+                            throw e;
+                        }
+                        if (!PhoenixEtlService.syncSchema(conn, config)) {
+                            throw e;
+                        }
+                        Util.sqlRS(conn, sql, rs -> {
+                            try {
+                                ResultSetMetaData rsd = rs.getMetaData();
+                                int columnCount = rsd.getColumnCount();
+                                for (int i = 1; i <= columnCount; i++) {
+                                    columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                                }
+                                columnsTypeCache.put(cacheKey, columnTypeTmp);
+                            } catch (SQLException e1) {
+                                logger.error(e1.getMessage(), e1);
+                            }
+                        });
+                    }
+                }
+            }
+        }
+        return columnType;
+    }
+
+    /**
+     * 拼接主键 where条件
+     */
+    private void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Integer> ctype,
+                                 List<Map<String, ?>> values, Map<String, Object> d) {
+        // 拼接主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+            sql.append(dbMapping.escape(targetColumnName)).append("=? AND ");
+            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+            }
+            BatchExecutor.setValue(values, type, d.get(srcColumnName));
+        }
+        int len = sql.length();
+        sql.delete(len - 4, len);
+    }
+
+    public static class SyncItem {
+
+        private MappingConfig config;
+        private SingleDml singleDml;
+
+        SyncItem(MappingConfig config, SingleDml singleDml) {
+            this.config = config;
+            this.singleDml = singleDml;
+        }
+    }
+
+    /**
+     * 取主键hash
+     */
+    private int pkHash(DbMapping dbMapping, Map<String, Object> d) {
+        int hash = 0;
+        // 取主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+            Object value = null;
+            if (d != null) {
+                value = d.get(srcColumnName);
+            }
+            if (value != null) {
+                hash += value.hashCode();
+
+            }
+        }
+        hash = Math.abs(hash) % threads;
+        return Math.abs(hash);
+    }
+
+
+    public void close() {
+        for (int i = 0; i < threads; i++) {
+            executorThreads[i].shutdown();
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/BatchExecutor.java

@@ -0,0 +1,108 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.io.Closeable;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * sql批量执行器   基本没有变动
+ */
+public class BatchExecutor implements Closeable {
+
+    private static final Logger logger = LoggerFactory.getLogger(BatchExecutor.class);
+
+    private DataSource          dataSource;
+    private Connection          conn;
+    private AtomicInteger       idx    = new AtomicInteger(0);
+
+    public BatchExecutor(DataSource dataSource){
+        this.dataSource = dataSource;
+    }
+    public BatchExecutor(){
+    }
+
+    public Connection getConn() {
+        if (conn == null) {
+            try {
+                conn=PhoenixAdapter.getPhoenixConnection();
+                this.conn.setAutoCommit(false);
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+        return conn;
+    }
+
+    public static void setValue(List<Map<String, ?>> values, int type, Object value) {
+        Map<String, Object> valueItem = new HashMap<>();
+        valueItem.put("type", type);
+        valueItem.put("value", value);
+        values.add(valueItem);
+    }
+
+    public int executeUpdate(String sql) throws SQLException {
+        logger.debug("execute: {}", sql);
+        Statement statement = getConn().createStatement();
+        int ret = statement.executeUpdate(sql);
+        statement.close();
+        return ret;
+    }
+
+    public void execute(String sql, List<Map<String, ?>> values) throws SQLException {
+        if (logger.isDebugEnabled()) {
+            logger.debug("execute: {} {}", sql, Arrays.toString(values.toArray()));
+        }
+        PreparedStatement pstmt = getConn().prepareStatement(sql);
+        int len = values.size();
+        for (int i = 0; i < len; i++) {
+            int type = (Integer) values.get(i).get("type");
+            Object value = values.get(i).get("value");
+            SyncUtil.setPStmt(type, pstmt, value, i + 1);
+        }
+
+        pstmt.execute();
+        idx.incrementAndGet();
+        pstmt.close();
+    }
+
+    public void commit() throws SQLException {
+        getConn().commit();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor commit " + idx.get() + " rows");
+        }
+        idx.set(0);
+    }
+
+    public void rollback() throws SQLException {
+        getConn().rollback();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor rollback " + idx.get() + " rows");
+        }
+        idx.set(0);
+    }
+
+    @Override
+    public void close() {
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            } finally {
+                conn = null;
+            }
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/PhoenixSupportUtil.java

@@ -0,0 +1,97 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.function.Function;
+
+/**
+ * @author: lihua
+ * @date: 2020/12/30 18:43
+ * @Description:
+ */
+public class PhoenixSupportUtil {
+
+
+    public static final  Logger logger = LoggerFactory.getLogger(PhoenixSupportUtil.class);
+    public static Object sqlRS(Connection dsConnection, String sql, Function<ResultSet, Object> fun) {
+
+        try {
+            Connection conn = dsConnection;
+            Throwable var4 = null;
+
+            Object var9;
+            try {
+                Statement stmt = conn.createStatement(1003, 1007);
+                Throwable var6 = null;
+
+                try {
+                    stmt.setFetchSize(-2147483648);
+                    ResultSet rs = stmt.executeQuery(sql);
+                    Throwable var8 = null;
+
+                    try {
+                        var9 = fun.apply(rs);
+                    } catch (Throwable var56) {
+                        var9 = var56;
+                        var8 = var56;
+                        throw var56;
+                    } finally {
+                        if (rs != null) {
+                            if (var8 != null) {
+                                try {
+                                    rs.close();
+                                } catch (Throwable var55) {
+                                    var8.addSuppressed(var55);
+                                }
+                            } else {
+                                rs.close();
+                            }
+                        }
+
+                    }
+                } catch (Throwable var58) {
+                    var6 = var58;
+                    throw var58;
+                } finally {
+                    if (stmt != null) {
+                        if (var6 != null) {
+                            try {
+                                stmt.close();
+                            } catch (Throwable var54) {
+                                var6.addSuppressed(var54);
+                            }
+                        } else {
+                            stmt.close();
+                        }
+                    }
+
+                }
+            } catch (Throwable var60) {
+                var4 = var60;
+                throw var60;
+            } finally {
+                if (conn != null) {
+                    if (var4 != null) {
+                        try {
+                            conn.close();
+                        } catch (Throwable var53) {
+                            var4.addSuppressed(var53);
+                        }
+                    } else {
+                        conn.close();
+                    }
+                }
+
+            }
+
+            return var9;
+        } catch (Exception var62) {
+            logger.error("sqlRs has error, sql: {} ", sql);
+            throw new RuntimeException(var62);
+        }
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/SingleDml.java

@@ -0,0 +1,97 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 没有改动 单个DML类
+ */
+@SuppressWarnings({"unused", "WeakerAccess"})
+public class SingleDml {
+
+    private String              destination;
+    private String              database;
+    private String              table;
+    private String              type;
+    private Map<String, Object> data;
+    private Map<String, Object> old;
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Map<String, Object> getData() {
+        return data;
+    }
+
+    public void setData(Map<String, Object> data) {
+        this.data = data;
+    }
+
+    public Map<String, Object> getOld() {
+        return old;
+    }
+
+    public void setOld(Map<String, Object> old) {
+        this.old = old;
+    }
+
+    public static List<SingleDml> dml2SingleDmls(Dml dml) {
+        List<SingleDml> singleDmls = new ArrayList<>();
+        if (dml.getData() != null) {
+            int size = dml.getData().size();
+            for (int i = 0; i < size; i++) {
+                SingleDml singleDml = new SingleDml();
+                singleDml.setDestination(dml.getDestination());
+                singleDml.setDatabase(dml.getDatabase());
+                singleDml.setTable(dml.getTable());
+                singleDml.setType(dml.getType());
+                singleDml.setData(dml.getData().get(i));
+                if (dml.getOld() != null) {
+                    singleDml.setOld(dml.getOld().get(i));
+                }
+                singleDmls.add(singleDml);
+            }
+            //MaxWell 中没有对TRUNCATE的DML操作进行解析
+        } else if ("TRUNCATE".equalsIgnoreCase(dml.getType())) {
+            SingleDml singleDml = new SingleDml();
+            singleDml.setDestination(dml.getDestination());
+            singleDml.setDatabase(dml.getDatabase());
+            singleDml.setTable(dml.getTable());
+            singleDml.setType(dml.getType());
+            singleDmls.add(singleDml);
+        }
+        return singleDmls;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/SyncUtil.java

@@ -0,0 +1,274 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class SyncUtil {
+
+    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Map<String, Object> data) {
+        return getColumnsMap(dbMapping, data.keySet());
+    }
+
+    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Collection<String> columns) {
+        Map<String, String> columnsMap;
+        if (dbMapping.getMapAll()) {
+            if (dbMapping.getAllMapColumns() != null) {
+                return dbMapping.getAllMapColumns();
+            }
+            columnsMap = new LinkedHashMap<>();
+            for (String srcColumn : columns) {
+                boolean flag = true;
+                if (dbMapping.getTargetColumns() != null) {
+                    for (Map.Entry<String, String> entry : dbMapping.getTargetColumns().entrySet()) {
+                        if (srcColumn.equals(entry.getValue())) {
+                            columnsMap.put(entry.getKey(), srcColumn);
+                            flag = false;
+                            break;
+                        }
+                    }
+                }
+                //新增 排除掉getExcludeColumns() 去除的列
+                if (flag && !dbMapping.getExcludeColumns().contains(srcColumn)) {
+                    columnsMap.put(srcColumn, srcColumn);
+                }
+            }
+            dbMapping.setAllMapColumns(columnsMap);
+        } else {
+            columnsMap = dbMapping.getTargetColumns();
+        }
+        return columnsMap;
+    }
+
+    /**
+     * 设置 preparedStatement
+     *
+     * @param type sqlType
+     * @param pstmt 需要设置的preparedStatement
+     * @param value 值
+     * @param i 索引号
+     */
+    public static void setPStmt(int type, PreparedStatement pstmt, Object value, int i) throws SQLException {
+        switch (type) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                if (value instanceof Boolean) {
+                    pstmt.setBoolean(i, (Boolean) value);
+                } else if (value instanceof String) {
+                    boolean v = !value.equals("0");
+                    pstmt.setBoolean(i, v);
+                } else if (value instanceof Number) {
+                    boolean v = ((Number) value).intValue() != 0;
+                    pstmt.setBoolean(i, v);
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.CHAR:
+            case Types.NCHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+                if (value instanceof String) {
+                    pstmt.setString(i, (String) value);
+                } else if (value == null) {
+                    pstmt.setNull(i, type);
+                } else {
+                    pstmt.setString(i, value.toString());
+                }
+                break;
+            case Types.TINYINT:
+                if (value instanceof Number) {
+                    pstmt.setByte(i, ((Number) value).byteValue());
+                } else if (value instanceof String) {
+                    pstmt.setByte(i, Byte.parseByte((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.SMALLINT:
+                if (value instanceof Number) {
+                    pstmt.setShort(i, ((Number) value).shortValue());
+                } else if (value instanceof String) {
+                    pstmt.setShort(i, Short.parseShort((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.INTEGER:
+                if (value instanceof Number) {
+                    pstmt.setInt(i, ((Number) value).intValue());
+                } else if (value instanceof String) {
+                    pstmt.setInt(i, Integer.parseInt((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.BIGINT:
+                if (value instanceof Number) {
+                    pstmt.setLong(i, ((Number) value).longValue());
+                } else if (value instanceof String) {
+                    pstmt.setLong(i, Long.parseLong((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+                if (value instanceof BigDecimal) {
+                    pstmt.setBigDecimal(i, (BigDecimal) value);
+                } else if (value instanceof Byte) {
+                    pstmt.setInt(i, ((Byte) value).intValue());
+                } else if (value instanceof Short) {
+                    pstmt.setInt(i, ((Short) value).intValue());
+                } else if (value instanceof Integer) {
+                    pstmt.setInt(i, (Integer) value);
+                } else if (value instanceof Long) {
+                    pstmt.setLong(i, (Long) value);
+                } else if (value instanceof Float) {
+                    pstmt.setBigDecimal(i, new BigDecimal((float) value));
+                } else if (value instanceof Double) {
+                    pstmt.setBigDecimal(i, new BigDecimal((double) value));
+                } else if (value != null) {
+                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.REAL:
+                if (value instanceof Number) {
+                    pstmt.setFloat(i, ((Number) value).floatValue());
+                } else if (value instanceof String) {
+                    pstmt.setFloat(i, Float.parseFloat((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                if (value instanceof Number) {
+                    pstmt.setDouble(i, ((Number) value).doubleValue());
+                } else if (value instanceof String) {
+                    pstmt.setDouble(i, Double.parseDouble((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                if (value instanceof Blob) {
+                    pstmt.setBlob(i, (Blob) value);
+                } else if (value instanceof byte[]) {
+                    pstmt.setBytes(i, (byte[]) value);
+                } else if (value instanceof String) {
+                    pstmt.setBytes(i, ((String) value).getBytes(StandardCharsets.ISO_8859_1));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.CLOB:
+                if (value instanceof Clob) {
+                    pstmt.setClob(i, (Clob) value);
+                } else if (value instanceof byte[]) {
+                    pstmt.setBytes(i, (byte[]) value);
+                } else if (value instanceof String) {
+                    Reader clobReader = new StringReader((String) value);
+                    pstmt.setCharacterStream(i, clobReader);
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.DATE:
+                if (value instanceof Date) {
+                    pstmt.setDate(i, (Date) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setDate(i, new Date(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    if (!v.startsWith("0000-00-00")) {
+                        java.util.Date date = Util.parseDate(v);
+                        if (date != null) {
+                            pstmt.setDate(i, new Date(date.getTime()));
+                        } else {
+                            pstmt.setNull(i, type);
+                        }
+                    } else {
+                        pstmt.setObject(i, value);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.TIME:
+                if (value instanceof Time) {
+                    pstmt.setTime(i, (Time) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setTime(i, new Time(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    java.util.Date date = Util.parseDate(v);
+                    if (date != null) {
+                        pstmt.setTime(i, new Time(date.getTime()));
+                    } else {
+                        pstmt.setNull(i, type);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.TIMESTAMP:
+                if (value instanceof Timestamp) {
+                    pstmt.setTimestamp(i, (Timestamp) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setTimestamp(i, new Timestamp(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    if (!v.startsWith("0000-00-00")) {
+                        java.util.Date date = Util.parseDate(v);
+                        if (date != null) {
+                            pstmt.setTimestamp(i, new Timestamp(date.getTime()));
+                        } else {
+                            pstmt.setNull(i, type);
+                        }
+                    } else {
+                        pstmt.setObject(i, value);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            default:
+                pstmt.setObject(i, value, type);
+        }
+    }
+
+    public static String getDbTableName(MappingConfig.DbMapping dbMapping) {
+        String result = "";
+        if (StringUtils.isNotEmpty(dbMapping.getTargetDb())) {
+            if (dbMapping.isEscapeUpper()) {
+                //新增 字段默认大写加双引号
+                result += "\"" + dbMapping.getTargetDb() + "\".";
+            } else {
+                result += dbMapping.getTargetDb() + ".";
+            }
+        }
+
+        if (dbMapping.isEscapeUpper()) {
+            //新增  字段默认大写加双引号
+            result += "\"" + dbMapping.getTargetTable().replaceAll("\\.", "\".\"") + "\"";
+        } else {
+            result += dbMapping.getTargetTable();
+        }
+        return result;
+    }
+}

client-adapter/phoenix/src/main/java/com/alibaba/otter/canal/client/adapter/phoenix/support/TypeUtil.java

@@ -0,0 +1,142 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.support;
+
+import com.alibaba.druid.sql.ast.SQLDataType;
+import com.alibaba.druid.sql.ast.SQLDataTypeImpl;
+import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Phoenix类型   此类型完全新增
+ */
+public class TypeUtil {
+
+    private static String joinArgs(String type, Object[] args) {
+        if (args.length > 0) {
+            return type + "(" + StringUtils.join(args, ",") + ")";
+        }
+        return type;
+    }
+
+    /**
+     * 根据SQL的定义返回Phoenix的类型定义
+     * @see "https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-type-conversions.html"
+     * @param definition SQL的字段定义
+     * @param limit 是否启用字段长度限制
+     * @return Phoenix字段类型定义
+     */
+    public static String getPhoenixType(SQLColumnDefinition definition, boolean limit) {
+        if (definition == null) return "VARCHAR";
+        SQLDataType sqlDataType = definition.getDataType();
+        SQLDataTypeImpl sqlDataType1 = sqlDataType instanceof SQLDataTypeImpl ? (SQLDataTypeImpl)sqlDataType : null;
+        boolean isUnsigned = sqlDataType1 != null && sqlDataType1.isUnsigned();
+
+        return getPhoenixType(sqlDataType.getName().toUpperCase(), sqlDataType.getArguments().toArray(), isUnsigned, limit);
+    }
+
+    //MySQL类型和Phoenix类型一一对应
+    public static String getPhoenixType(String name, Object[] args, boolean isUnsigned, boolean limit) {
+        switch (name) {
+            case "BIT":
+                if (limit) {
+                    return joinArgs("BINARY", args);
+                }
+                return "BINARY";
+            case "TINYINT":
+                if (isUnsigned) {
+                    return "UNSIGNED_TINYINT";
+                }
+                return "TINYINT";
+            case "BOOLEAN":
+            case "BOOL":
+                return "BOOLEAN";
+            case "SMALLINT":
+                if (isUnsigned) {
+                    return "UNSIGNED_SMALLINT";
+                }
+                return "SMALLINT";
+            case "MEDIUMINT":
+                return "INTEGER";
+            case "INT":
+            case "INTEGER":
+                if (isUnsigned) {
+                    return "UNSIGNED_INT";
+                }
+                return "INTEGER";
+            case "BIGINT":
+                if (isUnsigned) {
+                    return "UNSIGNED_LONG";
+                }
+                return "BIGINT";
+            case "FLOAT":
+                if (isUnsigned) {
+                    return "UNSIGNED_FLOAT";
+                }
+                return "FLOAT";
+            case "DOUBLE":
+                if (isUnsigned) {
+                    return "UNSIGNED_DOUBLE";
+                }
+                return "DOUBLE";
+            case "DECIMAL":
+                if (limit) {
+                    return joinArgs("DECIMAL", args);
+                }
+                return "DECIMAL";
+            case "DATE":
+                if (isUnsigned) {
+                    return "UNSIGNED_DATE";
+                }
+                return "DATE";
+            case "DATETIME":
+            case "TIMESTAMP":
+                if (isUnsigned) {
+                    return "UNSIGNED_TIMESTAMP";
+                }
+                return "TIMESTAMP";
+            case "TIME":
+                if (isUnsigned) {
+                    return "UNSIGNED_TIME";
+                }
+                return "TIME";
+            case "YEAR":
+                return "INTEGER";
+            case "CHAR":
+                if (limit) {
+                    return joinArgs(name, args);
+                }
+                return "VARCHAR";
+            case "VARCHAR":
+                if (limit) {
+                    return joinArgs(name, args);
+                }
+                return "VARCHAR";
+            case "BINARY":
+                if (limit) {
+                    return joinArgs(name, args);
+                }
+                return "VARBINARY";
+            case "VARBINARY":
+                return "VARBINARY";
+            case "TINYBLOB":
+                return "VARBINARY";
+            case "TINYTEXT":
+                return "VARCHAR";
+            case "BLOB":
+                return "VARBINARY";
+            case "TEXT":
+                return "VARCHAR";
+            case "MEDIUMBLOB":
+                return "VARBINARY";
+            case "MEDIUMTEXT":
+                return "VARCHAR";
+            case "LONGBLOB":
+                return "VARBINARY";
+            case "LONGTEXT":
+                return "VARCHAR";
+            case "ENUM":
+            case "SET":
+                return "VARCHAR";
+        }
+        return "VARCHAR";
+    }
+}

client-adapter/phoenix/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+phoenix=com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter

client-adapter/phoenix/src/main/resources/hbase-site.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+  <!--支持NAMESPACE映射 -->
+   <property>
+          <name>phoenix.schema.isNamespaceMappingEnabled</name>
+          <value>true</value>
+   </property>
+   <property>
+          <name>phoenix.schema.mapSystemTablesToNamespace</name>
+          <value>true</value>
+   </property>
+  <!--支持phoenix的二级索引 -->
+   <property>
+          <name>hbase.regionserver.wal.codec</name>
+          <value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
+   </property>
+   <property>  
+        <name>hbase.table.sanity.checks</name>  
+        <value>false</value> 
+   </property> 
+  <!--启动压缩文件检查 -->
+   <property>
+     <name>hbase.regionserver.codecs</name>
+     <value>snappy,lzo</value>
+   </property>
+  <!-- rpc 超时时长 -->
+	<property>
+	  <name>hbase.rpc.timeout</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>hbase.client.operation.timeout</name>
+	  <value>600000</value>
+	</property>
+	<property>
+	  <name>hbase.client.scanner.timeout.period</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>hbase.client.scanner.timeout.period</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	 <name>phoenix.query.timeoutMs</name>
+	 <value>1800000</value>
+	</property>
+	<property>
+	 <name>phoenix.query.keepAliveMs</name>
+	 <value>600000</value>
+	</property>
+	<property>
+	  <name>hbase.client.ipc.pool.type</name>
+	  <value>RoundRobinPool</value>
+	</property>
+	<property>
+	  <name>hbase.client.ipc.pool.size</name>
+	  <value>10</value>
+	</property>
+	<property>
+	  <name>index.builder.threads.keepalivetime</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>index.write.threads.keepalivetime</name>
+	  <value>1200000</value>
+	</property>
+	<property>
+	  <name>hbase.htable.threads.keepalivetime</name>
+	  <value>1200000</value>
+	</property>
+<!--允许写入自定义WAL编辑，确保索引更新的正确性 -->
+	<property>
+	  <name>hbase.regionserver.wal.codec</name>
+	  <value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
+	</property>
+
+</configuration>

client-adapter/phoenix/src/main/resources/phoenix/phoenix_common.properties

@@ -0,0 +1,2 @@
+# ͬ<><CDAC><EFBFBD>߳<EFBFBD><DFB3><EFBFBD>
+threads =3

client-adapter/phoenix/src/main/resources/phoenix/phoenixtest_user.yml

@@ -0,0 +1,22 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+outerAdapterKey: phoenix
+concurrent: true
+dbMapping:
+  database: mytest
+  table: user
+  targetTable: mytest.user
+  escapeUpper: true  # 字段默认大写，并用双引号引起来
+  targetPk:
+    id: ID
+  mapAll: true  # 映射所有字段(默认true，不包含排除的字段)
+  alter: true   # 允许修改表结构(默认true，mapAll=true时可以新增，drop=true时可以删除字段)
+  drop: false   # 允许删除字段(默认false)
+  skipMissing: false # 是否跳过缺失的字段(默认false，允许新增字段时会自动同步缺失的字段;true时跳过缺失的字段)
+  limit: false  # 是否限与数据长度限制一致(默认false，不限制长度避免修改长度而无法修改)
+  targetColumns:
+    id: ID
+    name: NAME
+  excludeColumns: # 排除字段
+    - password

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/PhoenixConnectionTest.java

@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.Properties;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 16:58
+ * @Description:
+ */
+public class PhoenixConnectionTest {
+
+    public static void main(String[] args) {
+        Properties phoenixPro = new Properties();
+        //phoenix内部本身有连接池，不需要使用Druid初始化
+        phoenixPro.setProperty("hbase.rpc.timeout","600000");
+        phoenixPro.setProperty("hbase.client.scanner.timeout.period","600000");
+        phoenixPro.setProperty("dfs.client.socket-timeout","600000");
+        phoenixPro.setProperty("phoenix.query.keepAliveMs","600000");
+        phoenixPro.setProperty("phoenix.query.timeoutMs","3600000");
+        try {
+            Class.forName("org.apache.phoenix.jdbc.PhoenixDriver");
+            Connection connection = DriverManager.getConnection("jdbc:phoenix:zookeeper01,zookeeper02,zookeeper03:2181:/hbase/db", phoenixPro);
+            System.out.println(connection);
+            connection.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/TestConfigLoad.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.config.ConfigLoader;
+import com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Map;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 17:07
+ * @Description:
+ */
+public class TestConfigLoad {
+    @Before
+    public void before() {
+        // 加载数据源连接池
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+    }
+
+    @Test
+    public void testLoad() {
+        Map<String, MappingConfig> configMap = ConfigLoader.load(null);
+        Assert.assertFalse(configMap.isEmpty());
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/TestConstant.java

@@ -0,0 +1,40 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+import java.sql.SQLException;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 17:09
+ * @Description:
+ */
+public class TestConstant {
+    public final static String          jdbcUrl      = "jdbc:mysql://127.0.0.1:3306/canal_adapter?useUnicode=true";
+    public final static String          jdbcUser     = "root";
+    public final static String          jdbcPassword = "!123456";
+
+    public final static DruidDataSource dataSource;
+
+    static {
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUser);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setPoolPreparedStatements(false);
+        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
+        dataSource.setValidationQuery("select 1");
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/sync/Common.java

@@ -0,0 +1,36 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test.sync;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.phoenix.test.TestConstant;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 23:15
+ * @Description:
+ */
+public class Common {
+    public static PhoenixAdapter init() {
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+
+        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
+        outerAdapterConfig.setName("phoenix");
+        outerAdapterConfig.setKey("phoenix");
+        Map<String, String> properties = new HashMap<>();
+        properties.put("jdbc.driverClassName", "org.apache.phoenix.jdbc.PhoenixDriver");
+        properties.put("jdbc.url", "jdbc:phoenix:zookeeper01,zookeeper02,zookeeper03:2181:/hbase/db");
+        outerAdapterConfig.setProperties(properties);
+
+        PhoenixAdapter adapter = new PhoenixAdapter();
+        adapter.init(outerAdapterConfig, null);
+        return adapter;
+    }
+
+    public static void main(String[] args) {
+        init();
+    }
+}

client-adapter/phoenix/src/test/java/com/alibaba/otter/canal/client/adapter/phoenix/test/sync/PhoenixSyncTest.java

@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.client.adapter.phoenix.test.sync;
+
+import com.alibaba.otter.canal.client.adapter.phoenix.PhoenixAdapter;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * @author: lihua
+ * @date: 2021/1/5 23:16
+ * @Description:
+ */
+public class PhoenixSyncTest {
+
+    private PhoenixAdapter phoenixAdapter;
+
+    @Before
+    public void init() {
+        phoenixAdapter = Common.init();
+    }
+
+    @Test
+    public void testEtl() {
+        List<String> param = new ArrayList<>();
+        phoenixAdapter.etl("phoenixtest_user.yml", param);
+    }
+    @Test
+    public void testCount() {
+        phoenixAdapter.count("phoenixtest_user.yml");
+    }
+    @Test
+    public void test01() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1);
+        data.put("name", "sixPulseExcalibur");
+        data.put("password", "123456");
+        dml.setData(dataList);
+
+        phoenixAdapter.sync(Collections.singletonList(dml));
+    }
+
+    @Test
+    public void test02() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1);
+        data.put("name", "sixPulseExcalibur2");
+        dml.setData(dataList);
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("name", "sixPulseExcalibur");
+        dml.setOld(oldList);
+        phoenixAdapter.sync(Collections.singletonList(dml));
+    }
+}

client-adapter/pom.xml

@@ -4,12 +4,12 @@
     <parent>
         <groupId>com.alibaba.otter</groupId>
         <artifactId>canal</artifactId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.client-adapter</artifactId>
-    <version>1.1.5</version>
+    <version>1.1.6-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal client adapter module for otter ${project.version}</name>
 
@@ -32,6 +32,8 @@
         <module>es7x</module>
         <module>escore</module>
         <module>kudu</module>
+        <module>phoenix</module>
+        <module>tablestore</module>
     </modules>
 
     <licenses>
@@ -250,6 +252,25 @@
                 <version>1.9.0</version>
                 <scope>test</scope>
             </dependency>
+
+            <dependency>
+                <groupId>com.aliyun.openservices</groupId>
+                <artifactId>tablestore</artifactId>
+                <version>5.10.3</version>
+                <classifier>jar-with-dependencies</classifier>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.google.protobuf</groupId>
+                        <artifactId>protobuf-java</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpasyncclient</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
+
         </dependencies>
     </dependencyManagement>
 

client-adapter/rdb/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/tablestore/pom.xml

@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.6-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.tablestore</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter rdb module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.aliyun.openservices</groupId>
+            <artifactId>tablestore</artifactId>
+            <version>5.10.3</version>
+            <classifier>jar-with-dependencies</classifier>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpasyncclient</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/classes/tablestore" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/tablestore" erroronmissingdir="true">
+                                        <include name="*.yml" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/TablestoreAdapter.java

@@ -0,0 +1,318 @@
+package com.alibaba.otter.canal.client.adapter.tablestore;
+
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.stream.Collectors;
+
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.client.adapter.tablestore.common.PropertyConstants;
+import com.alibaba.otter.canal.client.adapter.tablestore.config.ConfigLoader;
+import com.alibaba.otter.canal.client.adapter.tablestore.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.tablestore.service.TablestoreEtlService;
+import com.alibaba.otter.canal.client.adapter.tablestore.service.TablestoreSyncService;
+import com.alicloud.openservices.tablestore.DefaultTableStoreWriter;
+import com.alicloud.openservices.tablestore.TableStoreWriter;
+import com.alicloud.openservices.tablestore.core.auth.DefaultCredentials;
+import com.alicloud.openservices.tablestore.core.auth.ServiceCredentials;
+import com.alicloud.openservices.tablestore.writer.WriterConfig;
+import com.alicloud.openservices.tablestore.writer.WriterResult;
+import com.alicloud.openservices.tablestore.writer.enums.BatchRequestType;
+import com.alicloud.openservices.tablestore.writer.enums.DispatchMode;
+import com.alicloud.openservices.tablestore.writer.enums.WriteMode;
+import com.alicloud.openservices.tablestore.writer.enums.WriterRetryStrategy;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
+
+
+@SPI("tablestore")
+public class TablestoreAdapter implements OuterAdapter {
+
+    private static final Logger logger              = LoggerFactory.getLogger(TablestoreAdapter.class);
+
+    private Map<String, MappingConfig>              tablestoreMapping          = new ConcurrentHashMap<>();                // 文件名对应配置
+
+    private Map<String, Map<String, MappingConfig>> mappingConfigCache  = new ConcurrentHashMap<>();
+
+    private Map<String, Map<String, TableStoreWriter>> writerCache  = new ConcurrentHashMap<>();
+
+    private TablestoreSyncService tablestoreSyncService;
+
+    private Properties                              envProperties;
+
+    private OuterAdapterConfig configuration;
+
+
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        this.envProperties = envProperties;
+        this.configuration = configuration;
+        Map<String, MappingConfig> tablestoreMappingTmp = ConfigLoader.load(envProperties);
+        // 过滤不匹配的key的配置
+        tablestoreMappingTmp.forEach((key, mappingConfig) -> {
+            if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
+                    || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
+                    .equalsIgnoreCase(configuration.getKey()))) {
+                tablestoreMapping.put(key, mappingConfig);
+                mappingConfig.getDbMapping().init(mappingConfig);
+            }
+        });
+
+        if (tablestoreMapping.isEmpty()) {
+            throw new RuntimeException("No tablestore adapter found for config key: " + configuration.getKey());
+        }
+
+        Map<String, String> properties = configuration.getProperties();
+
+        for (Map.Entry<String, MappingConfig> entry : tablestoreMapping.entrySet()) {
+            String configName = entry.getKey();
+            MappingConfig mappingConfig = entry.getValue();
+            String key;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+                        + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
+            } else {
+                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
+            }
+            Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
+                    k1 -> new ConcurrentHashMap<>());
+            configMap.put(configName, mappingConfig);
+
+
+            // 构建对应的 TableStoreWriter
+            ServiceCredentials credentials = new DefaultCredentials(
+                    properties.get(PropertyConstants.TABLESTORE_ACCESSSECRETID),
+                    properties.get(PropertyConstants.TABLESTORE_ACCESSSECRETKEY)
+            );
+
+
+            WriterConfig config = getWriterConfig(mappingConfig);
+
+            TableStoreWriter writer = new DefaultTableStoreWriter(
+                    properties.get(PropertyConstants.TABLESTORE_ENDPOINT),
+                    credentials,
+                    properties.get(PropertyConstants.TABLESTORE_INSTANCENAME),
+                    mappingConfig.getDbMapping().getTargetTable(),
+                    config,
+                    null
+            );
+
+            Map<String, TableStoreWriter> config2writerMap = writerCache.computeIfAbsent(key,
+                    k1 -> new ConcurrentHashMap<>());
+            config2writerMap.put(configName, writer);
+
+        }
+
+        tablestoreSyncService = new TablestoreSyncService();
+    }
+
+    /**
+     * 根据配置文件获得tablestorewriter的WriterConfig信息
+     * @param mappingConfig
+     * @return
+     */
+    private WriterConfig getWriterConfig(MappingConfig mappingConfig) {
+        WriterConfig config = new WriterConfig();
+        MappingConfig.DbMapping mapping = mappingConfig.getDbMapping();
+        config.setMaxBatchRowsCount(mapping.getCommitBatch());
+        config.setConcurrency(mappingConfig.getThreads());
+        config.setDispatchMode(DispatchMode.HASH_PRIMARY_KEY);
+        config.setWriteMode(WriteMode.SEQUENTIAL);
+        config.setBatchRequestType(BatchRequestType.BULK_IMPORT);
+        config.setBucketCount(mappingConfig.getThreads());
+        config.setWriterRetryStrategy(WriterRetryStrategy.CERTAIN_ERROR_CODE_NOT_RETRY);
+        config.setAllowDuplicatedRowInBatchRequest(false);
+        return config;
+    }
+
+
+    @Override
+    public void sync(List<Dml> dmls) {
+        if (dmls == null || dmls.isEmpty()) {
+            return;
+        }
+
+        try {
+            Set<TableStoreWriter> writerSet = new HashSet<>();
+            List<Future<WriterResult>> futureList = new ArrayList<>();
+            for (Dml dml : dmls) {
+                String destination = StringUtils.trimToEmpty(dml.getDestination());
+                String groupId = StringUtils.trimToEmpty(dml.getGroupId());
+                String database = dml.getDatabase();
+                String table = dml.getTable();
+                String key;
+                if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                    key = destination + "-" + groupId + "_" + database + "-" + table;
+                } else {
+                    key = destination + "_" + database + "-" + table;
+                }
+                Map<String, MappingConfig> configMap = mappingConfigCache.get(key);
+                if (configMap == null) {
+                    // 可能有dml中涉及到的表并没有出现在配置中，说明此类dml并不需要同步
+                    continue;
+                }
+
+                Map<String, TableStoreWriter> writerMap = writerCache.get(key);
+
+                for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
+                    TableStoreWriter w = writerMap.get(entry.getKey());
+                    // 拿到所有future用于判定失败的记录
+                    Future<WriterResult> futureTemp = tablestoreSyncService.sync(entry.getValue(), dml, w);
+                    if (futureTemp != null) {
+                        writerSet.add(w);
+                        futureList.add(futureTemp);
+                    }
+                }
+            }
+
+            if (writerSet.isEmpty()) {
+                return;
+            }
+
+            writerSet.forEach(e -> e.flush());
+
+            List<WriterResult.RowChangeStatus> totalFailedRows = new ArrayList<>();
+            for (Future<WriterResult> future : futureList) {
+                try {
+                    WriterResult result = future.get();
+                    List<WriterResult.RowChangeStatus> failedRows = result.getFailedRows();
+                    if (!CollectionUtils.isEmpty(failedRows)) {
+                        totalFailedRows.addAll(failedRows);
+                    }
+                } catch (InterruptedException e) {
+                    logger.info("InterruptedException", e);
+                    Thread.currentThread().interrupt();
+                } catch (ExecutionException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            if (!CollectionUtils.isEmpty(totalFailedRows)) {
+                // 认为有失败的请求
+                List<String> msgs = totalFailedRows.stream().map(e -> buildErrorMsgForFailedRowChange(e)).collect(Collectors.toList());
+                throw new RuntimeException("Failed rows:" + org.springframework.util.StringUtils.collectionToDelimitedString(msgs, ",", "[", "]"));
+            }
+
+        } catch (Exception e) {
+            throw e;
+        }
+    }
+
+    /**
+     * 组装失败记录的信息
+     * @param rowChangeStatus
+     * @return
+     */
+    public static String buildErrorMsgForFailedRowChange(WriterResult.RowChangeStatus rowChangeStatus) {
+        StringBuilder sb = new StringBuilder("{Exception:");
+        sb.append(rowChangeStatus.getException().getMessage()).append(",Table:")
+        .append(rowChangeStatus.getRowChange().getTableName()).append(",PrimaryKey:")
+        .append("{").append(rowChangeStatus.getRowChange().getPrimaryKey().toString())
+        .append("}}");
+        return sb.toString();
+    }
+
+
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        MappingConfig config = tablestoreMapping.get(task);
+        if (config == null) {
+            etlResult.setErrorMessage("can not find config for " + task);
+            etlResult.setSucceeded(false);
+            return etlResult;
+        }
+
+        TableStoreWriter writer = null;
+        try {
+            writer = buildEtlWriter(configuration, config);
+
+            TablestoreEtlService rdbEtlService = new TablestoreEtlService(writer, config);
+            rdbEtlService.importData(params);
+
+            etlResult.setSucceeded(true);
+            return etlResult;
+        } catch (Exception e) {
+            logger.error("Error while etl for task " + task, e);
+            etlResult.setSucceeded(false);
+            etlResult.setErrorMessage(e.getMessage());
+            return etlResult;
+        } finally {
+            if (writer != null) {
+                writer.close();
+            }
+        }
+    }
+
+    /**
+     * 构造批量导入的writer
+     * @param configuration
+     * @param mappingConfig
+     * @return
+     */
+    private TableStoreWriter buildEtlWriter(OuterAdapterConfig configuration, MappingConfig mappingConfig) {
+        Map<String, String> properties = configuration.getProperties();
+
+        ServiceCredentials credentials = new DefaultCredentials(
+                properties.get(PropertyConstants.TABLESTORE_ACCESSSECRETID),
+                properties.get(PropertyConstants.TABLESTORE_ACCESSSECRETKEY)
+        );
+
+        WriterConfig config = getWriterConfig(mappingConfig);
+        config.setBucketCount(3);
+        config.setAllowDuplicatedRowInBatchRequest(true);
+        config.setConcurrency(8);
+        config.setWriteMode(WriteMode.PARALLEL);
+
+        TableStoreWriter writer = new DefaultTableStoreWriter(
+                properties.get(PropertyConstants.TABLESTORE_ENDPOINT),
+                credentials,
+                properties.get(PropertyConstants.TABLESTORE_INSTANCENAME),
+                mappingConfig.getDbMapping().getTargetTable(),
+                config,
+                null
+        );
+        return writer;
+    }
+
+
+    @Override
+    public Map<String, Object> count(String task) {
+        throw new RuntimeException("count is not supportted in tablestore");
+    }
+
+
+    @Override
+    public String getDestination(String task) {
+        MappingConfig config = tablestoreMapping.get(task);
+        if (config != null) {
+            return config.getDestination();
+        }
+        return null;
+    }
+
+    @Override
+    public void destroy() {
+        if (tablestoreSyncService != null) {
+            tablestoreSyncService.close();
+        }
+
+        if (writerCache != null) {
+            for (Map<String, TableStoreWriter> tmpMap : writerCache.values()) {
+                if (tmpMap != null) {
+                    for (TableStoreWriter writer : tmpMap.values()) {
+                        writer.close();
+                    }
+                }
+            }
+        }
+    }
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/common/PropertyConstants.java

@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.common;
+
+
+public class PropertyConstants {
+
+    public static final String TABLESTORE_ACCESSSECRETID = "tablestore.accessSecretId";
+
+    public static final String TABLESTORE_ACCESSSECRETKEY = "tablestore.accessSecretKey";
+
+    public static final String TABLESTORE_ENDPOINT = "tablestore.endpoint";
+
+    public static final String TABLESTORE_INSTANCENAME = "tablestore.instanceName";
+
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/config/ConfigLoader.java

@@ -0,0 +1,51 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.config;
+
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.StringUtils;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * RDB表映射配置加载器
+ *
+ * @author rewerma 2018-11-07 下午02:41:34
+ * @version 1.0.0
+ */
+public class ConfigLoader {
+
+    private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
+
+    /**
+     * 加载RDB表映射配置
+     *
+     * @return 配置名/配置文件名--对象
+     */
+    public static Map<String, MappingConfig> load(Properties envProperties) {
+        logger.info("## Start loading tablestore mapping config ... ");
+
+        Map<String, MappingConfig> result = new LinkedHashMap<>();
+
+        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("tablestore");
+        configContentMap.forEach((fileName, content) -> {
+            MappingConfig config = YmlConfigBinder
+                .bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
+            if (config == null) {
+                return;
+            }
+            try {
+                config.validate();
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
+            }
+            result.put(fileName, config);
+        });
+
+        logger.info("## Tablestore mapping config loaded:" + StringUtils.collectionToCommaDelimitedString(result.keySet()));
+        return result;
+    }
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/config/MappingConfig.java

@@ -0,0 +1,318 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.config;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.client.adapter.tablestore.enums.TablestoreFieldType;
+import com.alibaba.otter.canal.client.adapter.tablestore.support.SyncUtil;
+import org.apache.commons.lang.StringUtils;
+
+import java.sql.ResultSetMetaData;
+import java.util.*;
+
+/**
+ * RDB表映射配置
+ *
+ * @author rewerma 2018-11-07 下午02:41:34
+ * @version 1.0.0
+ */
+public class MappingConfig implements AdapterConfig {
+
+    private String    dataSourceKey;      // 数据源key
+
+    private String    destination;        // canal实例或MQ的topic
+
+    private String    groupId;            // groupId
+
+    private String    outerAdapterKey;    // 对应适配器的key
+
+    private DbMapping dbMapping;          // db映射配置
+
+    private Boolean updateChangeColumns = false;
+
+    private Integer threads = 8;
+
+    public String getDataSourceKey() {
+        return dataSourceKey;
+    }
+
+    public void setDataSourceKey(String dataSourceKey) {
+        this.dataSourceKey = dataSourceKey;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getOuterAdapterKey() {
+        return outerAdapterKey;
+    }
+
+    public void setOuterAdapterKey(String outerAdapterKey) {
+        this.outerAdapterKey = outerAdapterKey;
+    }
+
+    public DbMapping getDbMapping() {
+        return dbMapping;
+    }
+
+    public void setDbMapping(DbMapping dbMapping) {
+        this.dbMapping = dbMapping;
+    }
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public AdapterMapping getMapping() {
+        return dbMapping;
+    }
+
+    public Boolean getUpdateChangeColumns() {
+        return updateChangeColumns;
+    }
+
+    public void setUpdateChangeColumns(Boolean updateChangeColumns) {
+        this.updateChangeColumns = updateChangeColumns;
+    }
+
+    public Integer getThreads() {
+        return threads;
+    }
+
+    public void setThreads(Integer threads) {
+        this.threads = threads;
+    }
+
+    public void validate() {
+        if (dbMapping.database == null || dbMapping.database.isEmpty()) {
+            throw new NullPointerException("dbMapping.database");
+        }
+        if (dbMapping.table == null || dbMapping.table.isEmpty()) {
+            throw new NullPointerException("dbMapping.table");
+        }
+        if (dbMapping.targetTable == null || dbMapping.targetTable.isEmpty()) {
+            throw new NullPointerException("dbMapping.targetTable");
+        }
+    }
+
+
+
+
+    public static class ColumnItem {
+        private String targetColumn;
+        private String  column;
+        private TablestoreFieldType type;
+
+        public String getColumn() {
+            return column;
+        }
+
+        public void setColumn(String column) {
+            this.column = column;
+        }
+
+        public TablestoreFieldType getType() {
+            return type;
+        }
+
+        public void setType(TablestoreFieldType type) {
+            this.type = type;
+        }
+
+        public String getTargetColumn() {
+            return targetColumn;
+        }
+
+        public void setTargetColumn(String targetColumn) {
+            this.targetColumn = targetColumn;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ColumnItem that = (ColumnItem) o;
+            return Objects.equals(column, that.column);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(column);
+        }
+    }
+
+
+
+    public static class DbMapping implements AdapterMapping {
+
+        private String              database;                                // 数据库名或schema名
+        private String              table;                                   // 表名
+        private LinkedHashMap<String, String> targetPk        = new LinkedHashMap<>(); // 目标表主键字段
+//        private boolean             mapAll          = false;                 // 映射所有字段
+
+        private String              targetTable;                             // 目标表名
+        private Map<String, String> targetColumns;                           // 目标表字段映射
+        private Map<String, String> targetColumnsParsed;
+        private String              etlCondition;                            // etl条件sql
+
+        private int                 readBatch       = 5000;
+        private int                 commitBatch     = 5000;                  // etl等批量提交大小
+
+        private Map<String, ColumnItem> columnItems        = new LinkedHashMap<>(); // 转换后的字段映射列表
+
+        public String getDatabase() {
+            return database;
+        }
+
+        public void setDatabase(String database) {
+            this.database = database;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
+        public void setTable(String table) {
+            this.table = table;
+        }
+
+
+        public LinkedHashMap<String, String> getTargetPk() {
+            return targetPk;
+        }
+
+        public void setTargetPk(LinkedHashMap<String, String> targetPk) {
+            this.targetPk = targetPk;
+        }
+
+        public String getTargetTable() {
+            return targetTable;
+        }
+
+        public void setTargetTable(String targetTable) {
+            this.targetTable = targetTable;
+        }
+
+        public Map<String, String> getTargetColumns() {
+            return targetColumns;
+        }
+
+
+        public void setTargetColumns(Map<String, String> targetColumns) {
+            this.targetColumns = targetColumns;
+
+        }
+
+        public Map<String, String> getTargetColumnsParsed() {
+            return targetColumnsParsed;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public int getReadBatch() {
+            return readBatch;
+        }
+
+        public void setReadBatch(int readBatch) {
+            this.readBatch = readBatch;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public Map<String, ColumnItem> getColumnItems() {
+            return columnItems;
+        }
+
+        public void setColumnItems(Map<String, ColumnItem> columnItems) {
+            this.columnItems = columnItems;
+        }
+
+        public void init(MappingConfig config) {
+            String splitBy = "$";
+            if (targetColumns != null) {
+                boolean needTypeInference = false;
+                for (Map.Entry<String, String> columnField : targetColumns.entrySet()) {
+                    String field = columnField.getValue();
+                    String type = null;
+                    if (field != null) {
+                        // 解析类型
+                        int i = field.indexOf(splitBy);
+                        if (i > -1) {
+                            type = field.substring(i + 1);
+                            field = field.substring(0, i);
+                        }
+                    }
+                    ColumnItem columnItem = new ColumnItem();
+                    columnItem.setColumn(columnField.getKey());
+                    columnItem.setTargetColumn(StringUtils.isBlank(field) ? columnField.getKey() : field);
+
+                    TablestoreFieldType fieldType = SyncUtil.getTablestoreType(type);
+                    if (fieldType == null) {
+                        needTypeInference = true;
+                    }
+                    columnItem.setType(fieldType);
+                    columnItems.put(columnField.getKey(), columnItem);
+                }
+                if (needTypeInference) {
+                    // 认为有field没有配置映射类型，需要进行类型推断
+                    DruidDataSource sourceDS = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+
+                    Util.sqlRS(sourceDS, "SELECT * FROM " + SyncUtil.getDbTableName(database, table) + " LIMIT 1 ", rs -> {
+                        try {
+                            ResultSetMetaData rsd = rs.getMetaData();
+                            int columnCount = rsd.getColumnCount();
+                            List<String> columns = new ArrayList<>();
+                            for (int i = 1; i <= columnCount; i++) {
+                                String columnName = rsd.getColumnName(i);
+                                if (columnItems.containsKey(columnName) && columnItems.get(columnName).getType() == null) {
+                                    int columnType = rsd.getColumnType(i);
+                                    columnItems.get(columnName).setType(SyncUtil.getDefaultTablestoreType(columnType));
+                                }
+                            }
+                            return true;
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+
+            } else {
+                this.targetColumns = new LinkedHashMap<>();
+            }
+            targetColumnsParsed = new HashMap<>();
+
+            targetColumns.forEach((key, value) -> {
+                if (StringUtils.isEmpty(value)) {
+                    targetColumnsParsed.put(key, key);
+                } else if (value.contains(splitBy) && columnItems.containsKey(key)) {
+                    targetColumnsParsed.put(key, columnItems.get(key).targetColumn);
+                } else {
+                    targetColumnsParsed.put(key, value);
+                }
+            });
+        }
+
+    }
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/enums/TablestoreFieldType.java

@@ -0,0 +1,9 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.enums;
+
+public enum TablestoreFieldType {
+    INT,
+    DOUBLE,
+    BOOL,
+    STRING,
+    BINARY
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/service/TablestoreEtlService.java

@@ -0,0 +1,134 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.service;
+
+
+import com.alibaba.otter.canal.client.adapter.support.*;
+import com.alibaba.otter.canal.client.adapter.tablestore.TablestoreAdapter;
+import com.alibaba.otter.canal.client.adapter.tablestore.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.tablestore.support.SyncUtil;
+import com.alicloud.openservices.tablestore.TableStoreWriter;
+import com.alicloud.openservices.tablestore.model.RowChange;
+import com.alicloud.openservices.tablestore.writer.WriterResult;
+import org.springframework.util.CollectionUtils;
+
+import javax.sql.DataSource;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+public class TablestoreEtlService extends AbstractEtlService {
+
+    private TableStoreWriter writer;
+    private MappingConfig config;
+    private TablestoreSyncService syncService;
+
+
+    public TablestoreEtlService(TableStoreWriter writer, MappingConfig config){
+        super("Tablestore", config);
+        this.writer = writer;
+        this.config = config;
+        syncService = new TablestoreSyncService();
+    }
+
+
+    public EtlResult importData(List<String> params) {
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+        String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping.getDatabase(), dbMapping.getTable());
+        return importData(sql, params);
+    }
+
+    @Override
+    protected boolean executeSqlImport(DataSource srcDS, String sql, List<Object> values,
+                                       AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
+
+        try {
+            MappingConfig.DbMapping dbMapping = (MappingConfig.DbMapping) mapping;
+            Map<String, String> columnsMap = dbMapping.getTargetColumnsParsed();
+
+            Util.sqlRS(srcDS, sql, values, rs -> {
+                int idx = 0;
+                List<Future<WriterResult>> futureList = new ArrayList<>();
+                while (true) {
+                    try {
+                        if (!rs.next()) break;
+                    } catch (SQLException throwables) {
+                        logger.error("Error while get data from srcDs", throwables);
+                        break;
+                    }
+
+                    Dml dml = getDMLByRs(columnsMap, rs);
+
+                    List<RowChange> rowChanges = syncService.getRowChanges(dml, config);
+                    if (CollectionUtils.isEmpty(rowChanges)) {
+                        return null;
+                    }
+
+
+                    Future<WriterResult> future = writer.addRowChangeWithFuture(rowChanges);
+                    if (future != null) {
+                        futureList.add(future);
+                    }
+
+                }
+                writer.flush();
+                for (Future<WriterResult> future : futureList) {
+                    try {
+                        WriterResult result = future.get();
+                        if (result != null && result.isAllSucceed()) {
+                            impCount.incrementAndGet();
+                            idx++;
+                        } else if (result != null && !result.isAllSucceed()) {
+                            List<WriterResult.RowChangeStatus> totalFailedRows = result.getFailedRows();
+                            List<String> msgs = totalFailedRows.stream().map(e -> TablestoreAdapter.buildErrorMsgForFailedRowChange(e)).collect(Collectors.toList());
+                            logger.error("Failed rows when ETL:" + org.springframework.util.StringUtils.collectionToDelimitedString(msgs, ",", "[", "]"));
+                        }
+                    } catch (InterruptedException e) {
+                        logger.info("InterruptedException", e);
+                        errMsg.add(e.getMessage());
+                        Thread.currentThread().interrupt();
+                    } catch (ExecutionException e) {
+                        errMsg.add(e.getMessage());
+                        throw new RuntimeException(e);
+                    }
+                }
+
+                return idx;
+            });
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+    }
+
+
+
+
+    private Dml getDMLByRs(Map<String, String> columnsMap, ResultSet rs) {
+        try {
+            Dml dml = new Dml();
+            dml.setType("INSERT");
+            Map<String, Object> dataMap = new HashMap<>();
+            List<Map<String, Object>> dataList = new ArrayList<>();
+            dataList.add(dataMap);
+            dml.setData(dataList);
+            for (String key : columnsMap.keySet()) {
+                dataMap.put(key, rs.getObject(key));
+            }
+
+            return dml;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+
+
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/service/TablestoreSyncService.java

@@ -0,0 +1,397 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.service;
+
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.client.adapter.tablestore.enums.TablestoreFieldType;
+import com.alibaba.otter.canal.client.adapter.tablestore.support.SyncUtil;
+import com.alicloud.openservices.tablestore.TableStoreWriter;
+import com.alicloud.openservices.tablestore.model.*;
+import com.alicloud.openservices.tablestore.writer.WriterResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.alibaba.otter.canal.client.adapter.tablestore.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import org.springframework.util.CollectionUtils;
+
+
+/**
+ * RDB同步操作业务
+ *
+ * @author rewerma 2018-11-7 下午06:45:49
+ * @version 1.0.0
+ */
+public class TablestoreSyncService {
+
+    private static final Logger               logger  = LoggerFactory.getLogger(TablestoreSyncService.class);
+
+    private Map<String, Map<String, Integer>> columnsTypeCache;
+
+
+    public Map<String, Map<String, Integer>> getColumnsTypeCache() {
+        return columnsTypeCache;
+    }
+
+    public TablestoreSyncService(){
+        this(new ConcurrentHashMap<>());
+    }
+
+    @SuppressWarnings("unchecked")
+    public TablestoreSyncService(Map<String, Map<String, Integer>> columnsTypeCache){
+        this.columnsTypeCache = columnsTypeCache;
+
+    }
+
+    public Future<WriterResult> sync(MappingConfig mappingConfig,
+                     Dml dml,
+                     TableStoreWriter writer) {
+
+        List<RowChange> rowChanges = getRowChanges(dml, mappingConfig);
+        if (CollectionUtils.isEmpty(rowChanges)) {
+            return null;
+        }
+
+        Future<WriterResult> future = writer.addRowChangeWithFuture(rowChanges);
+
+        if (logger.isDebugEnabled()) {
+            logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
+        }
+        return future;
+    }
+
+    public List<RowChange> getRowChanges(Dml dml, MappingConfig config) {
+        String type = dml.getType();
+        boolean updateColume = config.getUpdateChangeColumns();
+        if (type != null && type.equalsIgnoreCase("INSERT")) {
+            return getInsertChanges(dml, updateColume, config);
+        } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
+            return getUpdateChanges(dml, updateColume, config);
+        } else if (type != null && type.equalsIgnoreCase("DELETE")) {
+            return getDeleteChanges(dml, updateColume, config);
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Update 类型下构造rowChangeList
+     * @param dml
+     * @param isColumnUpdate
+     * @param config
+     * @return
+     */
+    private List<RowChange> getUpdateChanges(Dml dml, boolean isColumnUpdate, MappingConfig config) {
+        List<RowChange> changeList = new ArrayList<>();
+        Map<String, String> columnMap = config.getDbMapping().getTargetColumnsParsed();
+        Map<String, MappingConfig.ColumnItem> typeMap = SyncUtil.getTypeMap(config);
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+
+        if (isColumnUpdate) {
+            // 列更新
+            for (int i = 0; i < dml.getData().size(); i++) {
+                Map<String, Object> map = dml.getData().get(i);
+                Map<String, Object> old = dml.getOld().get(i);
+                boolean isPrimaryKeyChange = isPrimaryKeyChange(old, dbMapping.getTargetPk());
+                if (isPrimaryKeyChange) {
+                    // 如果发现主键修改用put delete操作
+                    // 先 delete
+                    RowUpdateChange change = new RowUpdateChange(dbMapping.getTargetTable());
+                    PrimaryKey primaryKey = buildOldPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk(), old);
+                    change.setPrimaryKey(primaryKey);
+                    for (Map.Entry<String, Object> entry : map.entrySet()) {
+                        if (dbMapping.getTargetPk().containsKey(entry.getKey())) {
+                            // 这是个主键, 不需要再次处理
+                            continue;
+                        }
+                        if (!dbMapping.getTargetColumns().containsKey(entry.getKey())) {
+                            // 可能是没有配置的字段
+                            continue;
+                        }
+                        // 非主键
+                        String targetColumn = columnMap.get(entry.getKey());
+                        change.deleteColumns(targetColumn);
+                    }
+                    changeList.add(change);
+
+                    // 然后再put
+                    change = new RowUpdateChange(dbMapping.getTargetTable());
+                    primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                    change.setPrimaryKey(primaryKey);
+
+                    List<Column> columnList = getColumnsWhenPut(columnMap, dbMapping, map, typeMap);
+                    if (!CollectionUtils.isEmpty(columnList)) {
+                        change.put(columnList);
+                        changeList.add(change);
+                    }
+
+
+                } else {
+                    //否则用update
+                    RowUpdateChange change = new RowUpdateChange(dbMapping.getTargetTable());
+                    PrimaryKey primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                    change.setPrimaryKey(primaryKey);
+                    // 部分update 部分delete
+
+                    boolean validData = false;
+                    for (Map.Entry<String, Object> entry : old.entrySet()) {
+                        if (dbMapping.getTargetPk().containsKey(entry.getKey())) {
+                            // 这是个主键, 不需要再次处理
+                            continue;
+                        }
+                        if (!dbMapping.getTargetColumns().containsKey(entry.getKey())) {
+                            // 可能是没有配置的字段
+                            continue;
+                        }
+                        // 非主键
+                        String targetColumn = columnMap.get(entry.getKey());
+                        Object value = map.get(entry.getKey());
+                        validData = true;
+                        if (value == null) {
+                            change.deleteColumns(targetColumn);
+                        } else {
+                            TablestoreFieldType type = typeMap.get(entry.getKey()).getType();
+                            ColumnValue columnValue = SyncUtil.getColumnValue(value, type);
+                            change.put(targetColumn, columnValue);
+                        }
+                    }
+                    if (validData) {
+                        changeList.add(change);
+                    }
+
+                }
+            }
+
+        } else {
+            // 列覆盖
+            for (int i = 0; i < dml.getData().size(); i++) {
+                Map<String, Object> map = dml.getData().get(i);
+
+                RowPutChange change = new RowPutChange(dbMapping.getTargetTable());
+                PrimaryKey primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                change.setPrimaryKey(primaryKey);
+
+                List<Column> columnList = getColumnsWhenPut(columnMap, dbMapping, map, typeMap);
+                if (!CollectionUtils.isEmpty(columnList)) {
+                    change.addColumns(columnList);
+                    changeList.add(change);
+                }
+
+                if (dml.getOld() != null) {
+                    // 如果主键发生修改，需要在tablestore中删除对应的原记录
+                    Map<String, Object> old = dml.getOld().get(i);
+                    if (isPrimaryKeyChange(old, dbMapping.getTargetPk())) {
+                        RowDeleteChange delete = new RowDeleteChange(dbMapping.getTargetTable());
+                        PrimaryKey primaryKeyDelete = buildOldPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk(), old);
+                        delete.setPrimaryKey(primaryKeyDelete);
+                        changeList.add(delete);
+                    }
+
+                }
+
+            }
+        }
+
+        return changeList;
+    }
+
+    /**
+     * Delete 类型下构造rowChangeList
+     * @param dml
+     * @param isColumnUpdate
+     * @param config
+     * @return
+     */
+    private List<RowChange> getDeleteChanges(Dml dml, boolean isColumnUpdate, MappingConfig config) {
+        List<RowChange> changeList = new ArrayList<>();
+        Map<String, String> columnMap = config.getDbMapping().getTargetColumnsParsed();
+        Map<String, MappingConfig.ColumnItem> typeMap = SyncUtil.getTypeMap(config);
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+
+        if (isColumnUpdate) {
+            // 列更新
+            for (Map<String, Object> map : dml.getData()) {
+                RowUpdateChange change = new RowUpdateChange(dbMapping.getTargetTable());
+                PrimaryKey primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                change.setPrimaryKey(primaryKey);
+                boolean validData = false;
+                for (Map.Entry<String, Object> entry : map.entrySet()) {
+                    if (dbMapping.getTargetPk().containsKey(entry.getKey())) {
+                        // 这是个主键, 不需要再次处理
+                        continue;
+                    }
+                    if (!dbMapping.getTargetColumns().containsKey(entry.getKey())) {
+                        // 可能是没有配置的字段
+                        continue;
+                    }
+                    // 非主键
+                    validData = true;
+                    String targetColumn = columnMap.get(entry.getKey());
+                    change.deleteColumns(targetColumn);
+                }
+                if (validData) {
+                    changeList.add(change);
+                }
+            }
+
+        } else {
+            // 列覆盖
+            for (Map<String, Object> map : dml.getData()) {
+                RowDeleteChange change = new RowDeleteChange(dbMapping.getTargetTable());
+                PrimaryKey primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                change.setPrimaryKey(primaryKey);
+
+                changeList.add(change);
+            }
+        }
+
+        return changeList;
+    }
+
+    /**
+     * Insert 类型下构造rowChangeList
+     * @param dml
+     * @param isColumnUpdate
+     * @param config
+     * @return
+     */
+    private List<RowChange> getInsertChanges(Dml dml, boolean isColumnUpdate, MappingConfig config) {
+        List<RowChange> changeList = new ArrayList<>();
+
+        Map<String, String> columnMap = config.getDbMapping().getTargetColumnsParsed();
+        Map<String, MappingConfig.ColumnItem> typeMap = SyncUtil.getTypeMap(config);
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+        if (isColumnUpdate) {
+            // 列更新
+            for (Map<String, Object> map : dml.getData()) {
+                RowUpdateChange change = new RowUpdateChange(dbMapping.getTargetTable());
+                PrimaryKey primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                change.setPrimaryKey(primaryKey);
+
+                List<Column> columnList = getColumnsWhenPut(columnMap, dbMapping, map, typeMap);
+                if (!CollectionUtils.isEmpty(columnList)) {
+                    change.put(columnList);
+                    changeList.add(change);
+                }
+            }
+
+        } else {
+            // 列覆盖
+            for (Map<String, Object> map : dml.getData()) {
+                RowPutChange change = new RowPutChange(dbMapping.getTargetTable());
+                PrimaryKey primaryKey = buildPrimaryKey(map, typeMap, columnMap, dbMapping.getTargetPk());
+                change.setPrimaryKey(primaryKey);
+
+                List<Column> columnList = getColumnsWhenPut(columnMap, dbMapping, map, typeMap);
+                if (!CollectionUtils.isEmpty(columnList)) {
+                    change.addColumns(columnList);
+                }
+
+                changeList.add(change);
+            }
+        }
+        return changeList;
+    }
+
+
+    /**
+     * 组装rowChange的主键
+     * @param map 数据map
+     * @param typeMap  类型映射
+     * @param columnMap 字段名称映射
+     * @param targetPk  主键map
+     * @return
+     */
+    private PrimaryKey buildPrimaryKey(Map<String, Object> map, Map<String, MappingConfig.ColumnItem> typeMap, Map<String, String> columnMap, LinkedHashMap<String, String> targetPk) {
+        List primaryKeyList = new ArrayList<>();
+        for (Map.Entry<String, String> entry : targetPk.entrySet()) {
+            // build primary key
+            String targetColumn = columnMap.get(entry.getKey());
+            Object value = map.get(entry.getKey());
+            TablestoreFieldType type = typeMap.get(entry.getKey()).getType();
+            PrimaryKeyValue keyValue = SyncUtil.getPrimaryKeyValue(value, type);
+            PrimaryKeyColumn primaryKeyColumn = new PrimaryKeyColumn(targetColumn, keyValue);
+            primaryKeyList.add(primaryKeyColumn);
+        }
+        return new PrimaryKey(primaryKeyList);
+    }
+
+
+
+    private PrimaryKey buildOldPrimaryKey(Map<String, Object> map,
+                                          Map<String, MappingConfig.ColumnItem> typeMap,
+                                          Map<String, String> columnMap,
+                                          LinkedHashMap<String, String> targetPk,
+                                          Map<String, Object> old) {
+        List primaryKeyList = new ArrayList<>();
+        for (Map.Entry<String, String> entry : targetPk.entrySet()) {
+            // build primary key
+            String targetColumn = columnMap.get(entry.getKey());
+            Object value = old != null && old.containsKey(entry.getKey()) && !old.get(entry.getKey()).equals(map.get(entry.getKey())) ? old.get(entry.getKey()) : map.get(entry.getKey());
+            TablestoreFieldType type = typeMap.get(entry.getKey()).getType();
+            PrimaryKeyValue keyValue = SyncUtil.getPrimaryKeyValue(value, type);
+            PrimaryKeyColumn primaryKeyColumn = new PrimaryKeyColumn(targetColumn, keyValue);
+            primaryKeyList.add(primaryKeyColumn);
+        }
+        return new PrimaryKey(primaryKeyList);
+    }
+
+    /**
+     * 用于获得全量覆盖时的非主键列对应的columnlist
+     * @param columnMap
+     * @param dbMapping
+     * @param map
+     * @param typeMap
+     * @return
+     */
+    private List<Column> getColumnsWhenPut(Map<String, String> columnMap,
+                                           MappingConfig.DbMapping dbMapping,
+                                           Map<String, Object> map,
+                                           Map<String, MappingConfig.ColumnItem> typeMap) {
+        List<Column> columnList = new ArrayList<>();
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            if (dbMapping.getTargetPk().containsKey(entry.getKey())) {
+                // 这是个主键, 不需要再次处理
+                continue;
+            }
+            if (!dbMapping.getTargetColumns().containsKey(entry.getKey())) {
+                // 可能是没有配置的字段
+                continue;
+            }
+            // 非主键
+            String targetColumn = columnMap.get(entry.getKey());
+            Object value = entry.getValue();
+            if (value == null) {
+                // insert时空值过滤掉
+                continue;
+            }
+            TablestoreFieldType type = typeMap.get(entry.getKey()).getType();
+            ColumnValue columnValue = SyncUtil.getColumnValue(value, type);
+            columnList.add(new Column(targetColumn, columnValue));
+        }
+        return columnList;
+    }
+
+    /**
+     * 检查是否主键被修改
+     * @param old       old中的数据
+     * @param targetPk  主键map
+     * @return
+     */
+    private boolean isPrimaryKeyChange(Map<String, Object> old, Map<String, String> targetPk) {
+        for (String pkCol : targetPk.keySet()) {
+            if (old.containsKey(pkCol)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public void close() {
+
+    }
+}

client-adapter/tablestore/src/main/java/com/alibaba/otter/canal/client/adapter/tablestore/support/SyncUtil.java

@@ -0,0 +1,211 @@
+package com.alibaba.otter.canal.client.adapter.tablestore.support;
+
+
+import com.alibaba.otter.canal.client.adapter.tablestore.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.tablestore.enums.TablestoreFieldType;
+import com.alicloud.openservices.tablestore.model.ColumnValue;
+import com.alicloud.openservices.tablestore.model.PrimaryKeyValue;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.util.Map;
+
+public class SyncUtil {
+    private static final Logger logger  = LoggerFactory.getLogger(SyncUtil.class);
+
+
+    public static PrimaryKeyValue getPrimaryKeyValue(Object value, TablestoreFieldType fieldType) {
+        Object tablestoreValue;
+        switch (fieldType) {
+            case STRING:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return PrimaryKeyValue.fromString((String)tablestoreValue);
+            case INT:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return PrimaryKeyValue.fromLong((long)tablestoreValue);
+            case BINARY:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return PrimaryKeyValue.fromBinary((byte[])tablestoreValue);
+
+            default:
+                return PrimaryKeyValue.fromString(value.toString());
+
+        }
+    }
+
+    private static Object getTablestoreValue(Object value, TablestoreFieldType type) {
+        switch (type) {
+            case STRING:
+                if (value instanceof byte[]) {
+                    return new String(((byte[])value));
+                }
+                return value.toString();
+            case INT:
+                if (value instanceof Number) {
+                    return ((Number) value).longValue();
+                } else if (value instanceof Timestamp) {
+                    return ((Timestamp) value).getTime();
+                } else if (value instanceof String) {
+                    try {
+                        return Long.parseLong((String) value);
+                    } catch (NumberFormatException e) {
+                        logger.error("Error while parse long:" + value.toString(), e);
+                        throw e;
+                    }
+                } else if (value instanceof Date) {
+                    return ((Date) value).getTime();
+                } else if (value instanceof Time) {
+                    return ((Time) value).getTime();
+                } else if (value instanceof java.util.Date) {
+                    return ((java.util.Date) value).getTime();
+                } else if (value instanceof Boolean) {
+                    Boolean ob =  ((Boolean)value);
+                    return ob ? 1L : 0L;
+                }
+                return null;
+            case BINARY:
+                if (value instanceof byte[]) {
+                    return value;
+                } else if (value instanceof Blob) {
+                    Blob item = ((Blob) value);
+                    int length;
+                    try {
+                        length = (int) item.length();
+                        return item.getBytes(1, length);
+                    } catch (SQLException e) {
+                        logger.error("Error while convert blob to binary, blob:" + item.toString(), e);
+                        throw new RuntimeException(e);
+                    }
+                } else if (value instanceof String) {
+                    return ((String) value).getBytes(StandardCharsets.ISO_8859_1);
+                } else if (value instanceof Clob) {
+                    return value.toString().getBytes(StandardCharsets.ISO_8859_1);
+                }
+                return null;
+            case BOOL:
+                if (value instanceof Boolean) {
+                    return value;
+                } else if (value instanceof String) {
+                    return !value.equals("0");
+                } else if (value instanceof Number) {
+                    return ((Number) value).intValue() != 0;
+                }
+                return null;
+            case DOUBLE:
+                if (value instanceof Number) {
+                    return ((Number)value).doubleValue();
+                } else if (value instanceof String) {
+                    try {
+                        return Double.parseDouble((String) value);
+                    } catch (NumberFormatException e) {
+                        logger.error("Error while parse double:" + value.toString(), e);
+                        throw e;
+                    }
+                }
+                return null;
+            default:
+                return value;
+        }
+    }
+
+    /**
+     * 解析配置的字段映射类型
+     * @param type
+     * @return
+     */
+    public static TablestoreFieldType getTablestoreType(String type) {
+        if (type != null) {
+            if (type.equalsIgnoreCase("string")) {
+                return TablestoreFieldType.STRING;
+            } else if (type.equalsIgnoreCase("int") || type.equalsIgnoreCase("integer")) {
+                return TablestoreFieldType.INT;
+            } else if (type.equalsIgnoreCase("bool") || type.equalsIgnoreCase("boolean")) {
+                return TablestoreFieldType.BOOL;
+            } else if (type.equalsIgnoreCase("binary")) {
+                return TablestoreFieldType.BINARY;
+            } else if (type.equalsIgnoreCase("double") || type.equalsIgnoreCase("float") || type.equalsIgnoreCase("decimal")) {
+                return TablestoreFieldType.DOUBLE;
+            } else {
+                return null;
+            }
+        } else {
+            return null;
+        }
+    }
+
+
+    public static ColumnValue getColumnValue(Object value, TablestoreFieldType fieldType) {
+        Object tablestoreValue;
+        switch (fieldType) {
+            case STRING:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return ColumnValue.fromString((String)tablestoreValue);
+            case INT:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return ColumnValue.fromLong((long)tablestoreValue);
+            case BINARY:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return ColumnValue.fromBinary((byte[])tablestoreValue);
+            case DOUBLE:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return ColumnValue.fromDouble((double)tablestoreValue);
+            case BOOL:
+                tablestoreValue = getTablestoreValue(value, fieldType);
+                return ColumnValue.fromBoolean((boolean)tablestoreValue);
+            default:
+                return ColumnValue.fromString(value.toString());
+        }
+
+
+    }
+
+    public static Map<String, MappingConfig.ColumnItem> getTypeMap(MappingConfig config) {
+        return config.getDbMapping().getColumnItems();
+    }
+
+    public static TablestoreFieldType getDefaultTablestoreType(int sqlType) {
+        switch (sqlType) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                return TablestoreFieldType.BOOL;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                return TablestoreFieldType.BINARY;
+            case Types.TINYINT:
+            case Types.SMALLINT:
+            case Types.INTEGER:
+            case Types.BIGINT:
+                return TablestoreFieldType.INT;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+            case Types.REAL:
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                return TablestoreFieldType.DOUBLE;
+            case Types.CHAR:
+            case Types.NCHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+            case Types.CLOB:
+            case Types.DATE:
+            case Types.TIME:
+            case Types.TIMESTAMP:
+                return TablestoreFieldType.STRING;
+            default:
+                return TablestoreFieldType.STRING;
+        }
+    }
+
+    public static String getDbTableName(String db,String table) {
+        String result = "";
+        if (StringUtils.isNotEmpty(db)) {
+            result += ("`" + db + "`.");
+        }
+        result += ("`" + table + "`");
+        return result;
+    }
+}

client-adapter/tablestore/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+tablestore=com.alibaba.otter.canal.client.adapter.tablestore.TablestoreAdapter

client-adapter/tablestore/src/main/resources/tablestore/test.yml

@@ -0,0 +1,21 @@
+dataSourceKey: defaultDS
+destination: test_ots
+groupId: g1
+outerAdapterKey: mysql1
+threads: 1
+updateChangeColumns: false
+dbMapping:
+  database: test_ots
+  table: test
+  targetTable: canal_target
+  targetPk:
+    oId: oId
+  targetColumns:
+    oId:
+    c_id:
+    p_price:
+    p_count:
+  etlCondition: 
+  commitBatch: 1 # 批量提交的大小
+
+

client/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.5</version>
+		<version>1.1.6-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>
@@ -132,6 +132,12 @@
 			<version>1.1.1</version>
 			<scope>provided</scope>
 		</dependency>
+		<!-- Pulsar -->
+		<dependency>
+			<groupId>org.apache.pulsar</groupId>
+			<artifactId>pulsar-client</artifactId>
+			<scope>provided</scope>
+		</dependency>
 
 		<!-- junit -->
 		<dependency>

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaOffsetCanalConnector.java

@@ -6,7 +6,6 @@ import com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
-import com.google.common.collect.Lists;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -43,7 +42,7 @@ public class KafkaOffsetCanalConnector extends KafkaCanalConnector {
     public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
         waitClientRunning();
         if (!running) {
-            return Lists.newArrayList();
+            return new ArrayList<>();
         }
 
         if (offset > -1) {
@@ -61,7 +60,7 @@ public class KafkaOffsetCanalConnector extends KafkaCanalConnector {
             }
             return messages;
         }
-        return Lists.newArrayList();
+        return new ArrayList<>();
     }
 
     /**
@@ -76,7 +75,7 @@ public class KafkaOffsetCanalConnector extends KafkaCanalConnector {
     public List<KafkaFlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
         waitClientRunning();
         if (!running) {
-            return Lists.newArrayList();
+            return new ArrayList<>();
         }
 
         if (offset > -1) {
@@ -96,7 +95,7 @@ public class KafkaOffsetCanalConnector extends KafkaCanalConnector {
 
             return flatMessages;
         }
-        return Lists.newArrayList();
+        return new ArrayList<>();
     }
 
     /**

client/src/main/java/com/alibaba/otter/canal/client/pulsarmq/PulsarMQCanalConnector.java

@@ -0,0 +1,458 @@
+package com.alibaba.otter.canal.client.pulsarmq;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.client.CanalMQConnector;
+import com.alibaba.otter.canal.client.CanalMessageDeserializer;
+import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
+import com.alibaba.otter.canal.common.utils.MQUtil;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.client.api.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * PulsarMQ的连接
+ *
+ * <pre>
+ * 注意点:
+ * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
+ * </pre>
+ * todo 重复消费的概率相当高。一次批处理中，只要有一个消息处理失败，则该批次全部重试
+ *
+ * @since 1.1.1
+ */
+public class PulsarMQCanalConnector implements CanalMQConnector {
+
+    private static final Logger logger = LoggerFactory.getLogger(PulsarMQCanalConnector.class);
+
+    private volatile Messages<byte[]> lastGetBatchMessage;
+
+    /**
+     * 连接pulsar客户端
+     */
+    private PulsarClient pulsarClient;
+    /**
+     * 消费者
+     */
+    private Consumer<byte[]> consumer;
+    /**
+     * 是否扁平化Canal消息内容
+     */
+    private boolean isFlatMessage = false;
+    /**
+     * 主题名称
+     */
+    private String topic;
+    /**
+     * 环境连接URL
+     */
+    private String serviceUrl;
+    /**
+     * 角色认证token
+     */
+    private String roleToken;
+    /**
+     * 订阅客户端名称
+     */
+    private String subscriptName;
+    /**
+     * 每次批量获取数据的最大条目数，默认30
+     */
+    private int batchSize = 30;
+    /**
+     * 与{@code batchSize}一起决定批量获取的数据大小
+     * 当：
+     * <p>
+     * 1. {@code batchSize} 条消息未消费时<br/>
+     * 2. 距上一次批量消费时间达到{@code batchTimeoutSeconds}秒时
+     * </p>
+     * 任一条件满足，即执行批量消费
+     */
+    private int getBatchTimeoutSeconds = 30;
+    /**
+     * 批量处理消息时，一次批量处理的超时时间秒数
+     * <p>
+     * 该时间应该根据{@code batchSize}和{@code batchTimeoutSeconds}合理设置
+     * </p>
+     */
+    private int batchProcessTimeoutSeconds = 60;
+    /**
+     * 消费失败后的重试秒数，默认60秒
+     */
+    private int redeliveryDelaySeconds = 60;
+    /**
+     * 当客户端接收到消息，30秒还没有返回ack给服务端时，ack超时，会重新消费该消息
+     */
+    private int ackTimeoutSeconds = 30;
+    /**
+     * 是否开启消息失败重试功能，默认开启
+     */
+    private boolean isRetry = true;
+    /**
+     * <p>
+     * true重试(-RETRY)和死信队列(-DLQ)后缀为大写，有些地方创建的为小写，需确保正确
+     * </p>
+     */
+    private boolean isRetryDLQUpperCase = false;
+    /**
+     * 最大重试次数
+     */
+    private int maxRedeliveryCount = 128;
+    /**
+     * 连接标识位，在连接或关闭连接后改变值
+     */
+    private boolean connected = false;
+
+    /**
+     * 除必要参数外，其他参数使用默认值
+     * <p>
+     * 由于pulsar会根据subscriptName来区分消费实例，并且已经分配的指定实例的消息会固定到该实例的retry（重试）和dlq（死信）队列中，
+     * 所以subscriptName必传，且必须跟之前的一致，否则会导致之前消费失败的消息不会重消费。
+     * </p>
+     *
+     * @param isFlatMessage true使用扁平消息
+     * @param serviceUrl    pulsar服务连接地址，通常为：pulsar:host:ip或http://host:ip
+     * @param roleToken     有对应topic的消费者权限的角色token
+     * @param topic         订阅主题
+     * @param subscriptName 订阅和客户端名称，同一个订阅名视为同一个消费实例
+     * @date 2021/9/18 08:54
+     * @author chad
+     * @since 1 by chad at 2021/9/18 完善
+     */
+    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic
+            , String subscriptName) {
+        this.isFlatMessage = isFlatMessage;
+        this.serviceUrl = serviceUrl;
+        this.roleToken = roleToken;
+        this.topic = topic;
+        this.subscriptName = subscriptName;
+        if (StringUtils.isEmpty(this.subscriptName)) {
+            throw new RuntimeException("Pulsar Consumer subscriptName required");
+        }
+    }
+
+    /**
+     * 完全自定义的消费实例参数
+     *
+     * @date 2021/9/18 10:20
+     * @author chad
+     * @since 1 by chad at 2021/9/18 完善
+     */
+    public PulsarMQCanalConnector(boolean isFlatMessage, String serviceUrl, String roleToken, String topic
+            , String subscriptName, int batchSize, int getBatchTimeoutSeconds, int batchProcessTimeoutSeconds
+            , int redeliveryDelaySeconds, int ackTimeoutSeconds, boolean isRetry, boolean isRetryDLQUpperCase
+            , int maxRedeliveryCount) {
+        this.isFlatMessage = isFlatMessage;
+        this.serviceUrl = serviceUrl;
+        this.roleToken = roleToken;
+        this.topic = topic;
+        this.subscriptName = subscriptName;
+        if (StringUtils.isEmpty(this.subscriptName)) {
+            throw new RuntimeException("Pulsar Consumer subscriptName required");
+        }
+        this.batchSize = batchSize;
+        this.getBatchTimeoutSeconds = getBatchTimeoutSeconds;
+        this.batchProcessTimeoutSeconds = batchProcessTimeoutSeconds;
+        this.redeliveryDelaySeconds = redeliveryDelaySeconds;
+        this.ackTimeoutSeconds = ackTimeoutSeconds;
+        this.isRetry = isRetry;
+        this.isRetryDLQUpperCase = isRetryDLQUpperCase;
+        this.maxRedeliveryCount = maxRedeliveryCount;
+    }
+
+    @Override
+    public void connect() throws CanalClientException {
+        // 连接创建客户端
+        try {
+            pulsarClient = PulsarClient.builder()
+                    .serviceUrl(serviceUrl)
+                    .authentication(AuthenticationFactory.token(roleToken))
+                    .build();
+        } catch (PulsarClientException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void disconnect() throws CanalClientException {
+        try {
+            if (null != this.consumer && this.consumer.isConnected()) {
+                this.consumer.close();
+            }
+        } catch (PulsarClientException e) {
+            logger.error("close pulsar consumer error", e);
+        }
+        try {
+            if (null != this.pulsarClient) {
+                this.pulsarClient.close();
+            }
+        } catch (PulsarClientException e) {
+            logger.error("close pulsar client error", e);
+        }
+
+        this.connected = false;
+    }
+
+    @Override
+    public boolean checkValid() throws CanalClientException {
+        return connected;
+    }
+
+    @Override
+    public synchronized void subscribe(String filter) throws CanalClientException {
+        if (connected) {
+            return;
+        }
+        ConsumerBuilder<byte[]> builder = pulsarClient.newConsumer();
+
+        if (MQUtil.isPatternTopic(this.topic)) {
+            // 正则
+            builder.topicsPattern(this.topic);
+        } else {// 多个topic
+            builder.topic(this.topic);
+        }
+        // 为保证消息的有序性，仅支持单消费实例模式
+        // 灾备模式，一个分区只能有一个消费者，如果当前消费者不可用，自动切换到其他消费者
+        builder.subscriptionType(SubscriptionType.Failover);
+
+        builder
+                // 调用consumer.negativeAcknowledge(message) （即nack）来表示消费失败的消息
+                // 在指定的时间进行重新消费，默认是1分钟。
+                .negativeAckRedeliveryDelay(this.redeliveryDelaySeconds, TimeUnit.SECONDS)
+                .subscriptionName(this.subscriptName)
+        ;
+        if (this.isRetry) {
+            DeadLetterPolicy.DeadLetterPolicyBuilder dlqBuilder = DeadLetterPolicy.builder()
+                    // 最大重试次数
+                    .maxRedeliverCount(this.maxRedeliveryCount);
+            // 指定重试队列，不是多个或通配符topic才能判断重试队列
+            if (!MQUtil.isPatternTag(this.topic)) {
+                String retryTopic = this.topic + (this.isRetryDLQUpperCase ? "-RETRY" : "-retry");
+                dlqBuilder.retryLetterTopic(retryTopic);
+                String dlqTopic = this.topic + (this.isRetryDLQUpperCase ? "-DLQ" : "-dlq");
+                dlqBuilder.deadLetterTopic(dlqTopic);
+            }
+
+            //默认关闭，如果需要重试则开启
+            builder.enableRetry(true)
+                    .deadLetterPolicy(dlqBuilder.build());
+        }
+
+        // ack超时
+        builder.ackTimeout(this.ackTimeoutSeconds, TimeUnit.SECONDS);
+
+        // pulsar批量获取消息设置
+        builder.batchReceivePolicy(new BatchReceivePolicy.Builder()
+                .maxNumMessages(this.batchSize)
+                .timeout(this.getBatchTimeoutSeconds, TimeUnit.SECONDS)
+                .build());
+
+        try {
+            this.consumer = builder.subscribe();
+        } catch (PulsarClientException e) {
+            throw new RuntimeException(e);
+        }
+        connected = true;
+    }
+
+
+    @Override
+    public void subscribe() throws CanalClientException {
+        this.subscribe(null);
+    }
+
+    @Override
+    public void unsubscribe() throws CanalClientException {
+        try {
+            if (null != this.consumer) {
+                this.consumer.unsubscribe();
+            }
+        } catch (PulsarClientException e) {
+            throw new CanalClientException(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 不关注业务执行结果，只要收到消息即认识消费成功，自动ack
+     *
+     * @param timeout 阻塞获取消息的超时时间
+     * @param unit    时间单位
+     * @return java.util.List<com.alibaba.otter.canal.protocol.Message>
+     * @date 2021/9/13 22:24
+     * @author chad
+     * @since 1 by chad at 2021/9/13 添加注释
+     */
+    @Override
+    public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
+        List<Message> messages = getListWithoutAck(timeout, unit);
+        if (messages != null && !messages.isEmpty()) {
+            ack();
+        }
+        return messages;
+    }
+
+    /**
+     * 关心业务执行结果，业务侧根据执行结果调用 {@link PulsarMQCanalConnector#ack()}或{@link PulsarMQCanalConnector#rollback()}
+     * <p>
+     * 本方法示支持多线程，在MQ保障顺序的前提下，也无法提供单Topic多线程
+     * </p>
+     *
+     * @param timeout 阻塞获取消息的超时时间
+     * @param unit    时间单位
+     * @return java.util.List<com.alibaba.otter.canal.protocol.Message>
+     * @date 2021/9/13 22:26
+     * @author chad
+     * @since 1 by chad at 2021/9/13 添加注释
+     */
+    @Override
+    public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
+        return getListWithoutAck();
+    }
+
+    @Override
+    public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
+        List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
+        if (messages != null && !messages.isEmpty()) {
+            ack();
+        }
+        return messages;
+    }
+
+    @Override
+    public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
+        return getListWithoutAck();
+    }
+
+    /**
+     * 获取泛型数据，供其他方法调用
+     * <p>
+     * 不支持多线程调用
+     * </p>
+     *
+     * @return java.util.List<T>
+     * @date 2021/9/14 15:20
+     * @author chad
+     * @since 1 by chad at 2021/9/14 供{@link PulsarMQCanalConnector#getListWithoutAck(Long, TimeUnit)}
+     * 和{@link PulsarMQCanalConnector#getFlatListWithoutAck(Long, TimeUnit)}调用
+     */
+    private <T> List<T> getListWithoutAck() {
+        if (null != this.lastGetBatchMessage) {
+            throw new CanalClientException("mq get/ack not support concurrent & async ack");
+        }
+        List messageList = Lists.newArrayList();
+
+        try {
+            this.lastGetBatchMessage = consumer.batchReceive();
+            if (null == this.lastGetBatchMessage || this.lastGetBatchMessage.size() < 1) {
+                this.lastGetBatchMessage = null;
+                return messageList;
+            }
+        } catch (PulsarClientException e) {
+            logger.error("Receiver Pulsar MQ message error", e);
+            throw new CanalClientException(e);
+        }
+
+        for (org.apache.pulsar.client.api.Message<byte[]> msgExt : this.lastGetBatchMessage) {
+            byte[] data = msgExt.getData();
+            if (data == null) {
+                logger.warn("Received message data is null");
+                continue;
+            }
+            try {
+                if (isFlatMessage) {
+                    FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class);
+                    messageList.add(flatMessage);
+                } else {
+                    Message message = CanalMessageDeserializer.deserializer(data);
+                    messageList.add(message);
+                }
+            } catch (Exception ex) {
+                logger.error("Add message error", ex);
+                throw new CanalClientException(ex);
+            }
+        }
+
+        return messageList;
+    }
+
+    /**
+     * 当业务侧执行成功时，需要手动执行消息的ack操作
+     *
+     * @return void
+     * @date 2021/9/13 22:27
+     * @author chad
+     * @since 1 by chad at 2021/9/13 添加注释
+     */
+    @Override
+    public void ack() throws CanalClientException {
+        // 为什么要一个批次要么全部成功要么全部失败
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.consumer.acknowledge(this.lastGetBatchMessage);
+            }
+        } catch (Throwable e) {
+            if (this.lastGetBatchMessage != null) {
+                this.consumer.negativeAcknowledge(this.lastGetBatchMessage);
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    /**
+     * 当业务侧执行失败时，需要手动执行消息的rollback操作，从而让消息重新消费
+     *
+     * @return void
+     * @date 2021/9/13 22:28
+     * @author chad
+     * @since 1 by chad at 2021/9/13 添加注释
+     */
+    @Override
+    public void rollback() throws CanalClientException {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.consumer.negativeAcknowledge(this.lastGetBatchMessage);
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public Message get(int batchSize) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message getWithoutAck(int batchSize) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public void ack(long batchId) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+    @Override
+    public void rollback(long batchId) throws CanalClientException {
+        throw new CanalClientException("mq not support this method");
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/rabbitmq/RabbitMQCanalConnector.java

@@ -13,6 +13,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -239,7 +240,7 @@ public class RabbitMQCanalConnector implements CanalMQConnector {
         if (logger.isDebugEnabled()) {
             logger.debug("Get Message: {}", new String(messageData));
         }
-        List messageList = Lists.newArrayList();
+        List messageList = new ArrayList<>();
         if (!flatMessage) {
             Message message = CanalMessageDeserializer.deserializer(messageData);
             messageList.add(message);

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.client.rocketmq;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -165,7 +166,7 @@ public class RocketMQCanalConnector implements CanalMQConnector {
         if (logger.isDebugEnabled()) {
             logger.debug("Get Message: {}", messageExts);
         }
-        List messageList = Lists.newArrayList();
+        List messageList = new ArrayList<>();
         for (MessageExt messageExt : messageExts) {
             byte[] data = messageExt.getBody();
             if (data != null) {

common/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.5</version>
+		<version>1.1.6-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<artifactId>canal.common</artifactId>

common/src/main/java/com/alibaba/otter/canal/common/utils/MQUtil.java

@@ -0,0 +1,97 @@
+package com.alibaba.otter.canal.common.utils;
+
+
+public class MQUtil {
+    /**
+     * topic是否是正则表达式
+     *
+     * @param topic
+     * @return true是正则表达式
+     */
+    public static boolean isPatternTopic(String topic) {
+        return !topic.matches("^[0-9a-z:/-]+$");
+    }
+
+    /**
+     * 检查topic有效性
+     *
+     * @param topic
+     * @return
+     */
+    public static boolean checkTopic(String topic) {
+        return topic.matches("^[0-9a-z:/.*-]+$");
+    }
+
+    /**
+     * 检查tag有效性
+     *
+     * @param tag
+     * @return
+     */
+    public static boolean checkTag(String tag) {
+        return tag.matches("^[0-9a-zA-Z.*]+$");
+    }
+
+    /**
+     * 判断tag是否是正则
+     *
+     * @param tag
+     * @return
+     */
+    public static boolean isPatternTag(String tag) {
+        return !tag.matches("^[0-9a-zA-Z]+$");
+    }
+
+    /**
+     * 检查topic有效性
+     *
+     * @param topics
+     */
+    public static void checkTopicWithErr(String... topics) {
+        if (null == topics || 0 == topics.length) {
+            throw new NullPointerException("topic cannot null");
+        }
+
+        if (1 == topics.length) {
+            boolean ok = checkTopic(topics[0]);
+            if (ok) {
+                return;
+            }
+            throw new RuntimeException("topic invalid: " + topics[0]);
+        }
+
+        for (String t : topics) {
+            if (!checkTopic(t)) {
+                throw new IllegalArgumentException("topic invalid: " + t);
+            }
+            if (isPatternTopic(t)) {
+                throw new RuntimeException("pattern topic cannot multi: " + t);
+            }
+        }
+    }
+
+    /**
+     * 检查tag有效性
+     *
+     * @param tags
+     */
+    public static void checkTagWithErr(String... tags) {
+        // 空表示不使用tag
+        if (null == tags || 0 == tags.length) {
+            return;
+        }
+
+//        if (1 == tags.length && (null == tags[0] || 0 == tags[0].trim().length())) {
+//            throw new NullPointerException("tag cannot null");
+//        }
+
+        for (String t : tags) {
+            if (!checkTag(t)) {
+                throw new IllegalArgumentException("tag invalid: " + t);
+            }
+            if (isPatternTag(t)) {
+                throw new RuntimeException("pattern tag cannot multi: " + t);
+            }
+        }
+    }
+}

connector/core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.connector</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java

@@ -15,6 +15,7 @@ public class MQDestination {
     private String  partitionHash;
     private String  dynamicTopic;
     private String  dynamicTopicPartitionNum;
+    private Boolean enableDynamicQueuePartition;
 
     public String getCanalDestination() {
         return canalDestination;
@@ -71,4 +72,12 @@ public class MQDestination {
     public void setDynamicTopicPartitionNum(String dynamicTopicPartitionNum) {
         this.dynamicTopicPartitionNum = dynamicTopicPartitionNum;
     }
+
+    public Boolean getEnableDynamicQueuePartition() {
+        return enableDynamicQueuePartition;
+    }
+
+    public void setEnableDynamicQueuePartition(Boolean enableDynamicQueuePartition) {
+        this.enableDynamicQueuePartition = enableDynamicQueuePartition;
+    }
 }

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java

@@ -36,7 +36,7 @@ public class MQMessageUtils {
     private static Map<String, List<PartitionData>>    partitionDatas    = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
                                                                              .softValues(),
                                                                              pkHashConfigs -> {
-                                                                                 List<PartitionData> datas = Lists.newArrayList();
+                                                                                 List<PartitionData> datas = new ArrayList<>();
 
                                                                                  String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
                                                                                      ",",
@@ -75,7 +75,7 @@ public class MQMessageUtils {
     private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
                                                                              .softValues(),
                                                                               pkHashConfigs -> {
-                                                                                List<DynamicTopicData> datas = Lists.newArrayList();
+                                                                                List<DynamicTopicData> datas = new ArrayList<>();
                                                                                 String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
                                                                                     ",",
                                                                                     ";"),
@@ -102,7 +102,7 @@ public class MQMessageUtils {
     private static Map<String, List<TopicPartitionData>> topicPartitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
                                                                                 .softValues(),
                                                                                 tPConfigs -> {
-                                                                                    List<TopicPartitionData> datas = Lists.newArrayList();
+                                                                                    List<TopicPartitionData> datas = new ArrayList<>();
                                                                                     String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs,
                                                                                             ",",
                                                                                             ";"),
@@ -258,7 +258,7 @@ public class MQMessageUtils {
         List<Entry>[] partitionEntries = new List[partitionsNum];
         for (int i = 0; i < partitionsNum; i++) {
             // 注意一下并发
-            partitionEntries[i] = Collections.synchronizedList(Lists.newArrayList());
+            partitionEntries[i] = Collections.synchronizedList(new ArrayList<>());
         }
 
         for (EntryRowData data : datas) {
@@ -671,17 +671,34 @@ public class MQMessageUtils {
     }
 
     public static class PartitionData {
-
+        /**
+         * 当{schemaName.tableName}没有正则时，就匹配{schemaName.tableName}
+         */
         public String             simpleName;
+        /**
+         * 当{schemaName.tableName}有正则时，匹配正则
+         */
         public AviaterRegexFilter regexFilter;
+        /**
+         * hash模式
+         */
         public HashMode           hashMode = new HashMode();
     }
 
     public static class HashMode {
-
+        /**
+         * 当{schemaName.tableName}:$pk$时，使用自动主键hash
+         */
         public boolean      autoPkHash = false;
+        /**
+         * 当表达式仅为{schemaName.tableName}，没有:时，仅使用table hash
+         */
         public boolean      tableHash  = false;
-        public List<String> pkNames    = Lists.newArrayList();
+
+        /**
+         * 当表达式为{schemaName.tableName}:id^name^age时，pkNames为：id, name, age三个
+         */
+        public List<String> pkNames    = new ArrayList<>();
     }
 
     public static class DynamicTopicData {

connector/kafka-connector/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.connector</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java

@@ -9,6 +9,7 @@ import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
+import com.alibaba.otter.canal.common.utils.PropertiesUtils;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -53,6 +54,8 @@ public class CanalKafkaConsumer implements CanalMsgConsumer {
             String k = (String) entry.getKey();
             Object v = entry.getValue();
             if (k.startsWith(PREFIX_KAFKA_CONFIG) && v != null) {
+                // check env config
+                v = PropertiesUtils.getProperty(properties, k);
                 kafkaProperties.put(k.substring(PREFIX_KAFKA_CONFIG.length()), v);
             }
         }

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java

@@ -102,6 +102,8 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
             String key = (String) entry.getKey();
             Object value = entry.getValue();
             if (key.startsWith(PREFIX_KAFKA_CONFIG) && value != null) {
+                // check env config
+                value = PropertiesUtils.getProperty(properties, key);
                 key = key.substring(PREFIX_KAFKA_CONFIG.length());
                 kafkaProperties.put(key, value);
             }

connector/pom.xml

@@ -4,13 +4,13 @@
     <parent>
         <groupId>com.alibaba.otter</groupId>
         <artifactId>canal</artifactId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.connector</artifactId>
-    <version>1.1.5</version>
+    <version>1.1.6-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal connector module for otter ${project.version}</name>
 
@@ -92,10 +92,11 @@
     </repositories>
 
     <modules>
-        <module>kafka-connector</module>
         <module>core</module>
+        <module>kafka-connector</module>
         <module>rocketmq-connector</module>
         <module>rabbitmq-connector</module>
+        <module>pulsarmq-connector</module>
         <module>tcp-connector</module>
     </modules>
 
@@ -126,4 +127,4 @@
             <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
         </repository>
     </distributionManagement>
-</project>
+</project>

connector/pulsarmq-connector/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.6-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.pulsarmq</artifactId>
+    <packaging>jar</packaging>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.pulsar</groupId>
+            <artifactId>pulsar-client</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.pulsarmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.pulsarmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQConstants.java

@@ -0,0 +1,62 @@
+package com.alibaba.otter.canal.connector.pulsarmq.config;
+
+import org.joda.time.Seconds;
+
+/**
+ * PulsarMQ配置
+ *
+ * @author chad
+ * @date 2021/9/15 11:13
+ * @since 1 by chad at 2021/9/15 新增配置文件
+ */
+public class PulsarMQConstants {
+    public static final String ROOT = "pulsarmq";
+    /**
+     * pulsar服务连接地址
+     */
+    public static final String PULSARMQ_SERVER_URL = ROOT + "." + "serverUrl";
+    /**
+     * pulsar服务角色token，需要有对应token的生产者权限
+     */
+    public static final String PULSARMQ_ROLE_TOKEN = ROOT + "." + "roleToken";
+    /**
+     * topic前缀
+     */
+    public static final String PULSARMQ_TOPIC_TENANT_PREFIX = ROOT + "." + "topicTenantPrefix";
+
+    /**** 消费者 *****/
+    /**
+     * 获取批量消息超时等待时间
+     */
+    public static final String PULSARMQ_GET_BATCH_TIMEOUT_SECONDS = ROOT + "." + "getBatchTimeoutSeconds";
+    /**
+     * 批量处理超时时间
+     */
+    public static final String PULSARMQ_BATCH_PROCESS_TIMEOUT = ROOT + "." + "batchProcessTimeout";
+    /**
+     * 消费都订阅名称，将以该名称为消费者身份标识，同一个subscriptName，认为是同一个消费实例
+     */
+    public static final String PULSARMQ_SUBSCRIPT_NAME = ROOT + "." + "subscriptName";
+    /**
+     * 重试间隔秒数
+     */
+    public static final String PULSARMQ_REDELIVERY_DELAY_SECONDS = ROOT + "." + "redeliveryDelaySeconds";
+    /**
+     * ACK超时秒数
+     */
+    public static final String PULSARMQ_ACK_TIMEOUT_SECONDS = ROOT + "." + "ackTimeoutSeconds";
+    /**
+     * 是否开启消费重试
+     */
+    public static final String PULSARMQ_IS_RETRY = ROOT + "." + "isRetry";
+    /**
+     * 自动生成的 retry dlq队列名称后缀是否大写
+     */
+    public static final String PULSARMQ_IS_RETRY_DLQ_UPPERCASE = ROOT + "." + "isRetryDLQUpperCase";
+    /**
+     * 最大重试次数
+     */
+    public static final String PULSARMQ_MAX_REDELIVERY_COUNT = ROOT + "." + "maxRedeliveryCount";
+
+
+}

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/config/PulsarMQProducerConfig.java

@@ -0,0 +1,57 @@
+package com.alibaba.otter.canal.connector.pulsarmq.config;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * Pulsar生产者配置
+ * @author chad
+ * @date 2021/9/15 11:23
+ * @since 1 by chad at 2021/9/15 新增
+ */
+public class PulsarMQProducerConfig extends MQProperties {
+    /**
+     * pulsar服务连接地址
+     * <p>
+     *     broker: pulsar://localhost:6650<br/>
+     *     httpUrl: http://localhost:8080
+     * </p>
+     */
+    private String serverUrl;
+    /**
+     * pulsar topic前缀
+     * <p>
+     * 正常的pulsar topic全路径为：persistent://{tenant}/{namespace}/{TOPIC}，
+     * 而为了方便，在配置文件中仅需要配置 {TOPIC} 段即可，persistent://{tenant}/{namespace}由此参数控制。
+     * 在发送消息时会自动拼接上
+     * </p>
+     */
+    private String topicTenantPrefix;
+    /**
+     * 生产者角色权限，请确保该角色有canal使用的所有topic生产者权限（最低要求）
+     */
+    private String roleToken;
+
+    public String getServerUrl() {
+        return serverUrl;
+    }
+
+    public void setServerUrl(String serverUrl) {
+        this.serverUrl = serverUrl;
+    }
+
+    public String getRoleToken() {
+        return roleToken;
+    }
+
+    public void setRoleToken(String roleToken) {
+        this.roleToken = roleToken;
+    }
+
+    public String getTopicTenantPrefix() {
+        return topicTenantPrefix;
+    }
+
+    public void setTopicTenantPrefix(String topicTenantPrefix) {
+        this.topicTenantPrefix = topicTenantPrefix;
+    }
+}

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumer.java

@@ -0,0 +1,305 @@
+package com.alibaba.otter.canal.connector.pulsarmq.consumer;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.common.utils.MQUtil;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.client.api.*;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Pulsar consumer SPI 实现
+ *
+ * @author rewerma @ 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("pulsarmq")
+public class CanalPulsarMQConsumer implements CanalMsgConsumer {
+    /**
+     * 连接pulsar客户端
+     */
+    private PulsarClient pulsarClient;
+    private Consumer<byte[]> pulsarMQConsumer;
+    /**
+     * 是否为扁平消息
+     */
+    private boolean flatMessage = false;
+    /**
+     * 主题名称
+     */
+    private String topic;
+    /**
+     * 单线程控制
+     */
+    private volatile Messages<byte[]> lastGetBatchMessage;
+    /**
+     * 环境连接URL
+     */
+    private String serviceUrl;
+    /**
+     * 角色认证token
+     */
+    private String roleToken;
+    /**
+     * 订阅客户端名称
+     */
+    private String subscriptName;
+    /**
+     * 每次批量获取数据的最大条目数，默认30
+     */
+    private int batchSize = 30;
+    /**
+     * 与{@code batchSize}一起决定批量获取的数据大小
+     * 当：
+     * <p>
+     * 1. {@code batchSize} 条消息未消费时<br/>
+     * 2. 距上一次批量消费时间达到{@code batchTimeoutSeconds}秒时
+     * </p>
+     * 任一条件满足，即执行批量消费
+     */
+    private int getBatchTimeoutSeconds = 30;
+    /**
+     * 批量处理消息时，一次批量处理的超时时间
+     * <p>
+     * 该时间应该根据{@code batchSize}和{@code batchTimeoutSeconds}合理设置
+     * </p>
+     */
+    private long batchProcessTimeout = 60 * 1000;
+    /**
+     * 消费失败后的重试秒数，默认60秒
+     */
+    private int redeliveryDelaySeconds = 60;
+    /**
+     * 当客户端接收到消息，30秒还没有返回ack给服务端时，ack超时，会重新消费该消息
+     */
+    private int ackTimeoutSeconds = 30;
+    /**
+     * 是否开启消息失败重试功能，默认开启
+     */
+    private boolean isRetry = true;
+    /**
+     * <p>
+     * true重试(-RETRY)和死信队列(-DLQ)后缀为大写，有些地方创建的为小写，需确保正确
+     * </p>
+     */
+    private boolean isRetryDLQUpperCase = false;
+    /**
+     * 最大重试次数
+     */
+    private int maxRedeliveryCount = 128;
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        this.topic = topic;
+        String flatMessageStr = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        if (StringUtils.isNotEmpty(flatMessageStr)) {
+            this.flatMessage = Boolean.parseBoolean(flatMessageStr);
+        }
+        this.serviceUrl = properties.getProperty(PulsarMQConstants.PULSARMQ_SERVER_URL);
+        this.roleToken = properties.getProperty(PulsarMQConstants.PULSARMQ_ROLE_TOKEN);
+        this.subscriptName = properties.getProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME);
+        if (StringUtils.isEmpty(this.subscriptName)) {
+            throw new RuntimeException("Pulsar Consumer subscriptName required");
+        }
+        String batchSizeStr = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
+        if (StringUtils.isNotEmpty(batchSizeStr)) {
+            this.batchSize = Integer.parseInt(batchSizeStr);
+        }
+        String getBatchTimeoutSecondsStr = properties.getProperty(PulsarMQConstants.PULSARMQ_GET_BATCH_TIMEOUT_SECONDS);
+        if (StringUtils.isNotEmpty(getBatchTimeoutSecondsStr)) {
+            this.getBatchTimeoutSeconds = Integer.parseInt(getBatchTimeoutSecondsStr);
+        }
+        String batchProcessTimeoutStr = properties.getProperty(PulsarMQConstants.PULSARMQ_BATCH_PROCESS_TIMEOUT);
+        if (StringUtils.isNotEmpty(batchProcessTimeoutStr)) {
+            this.batchProcessTimeout = Integer.parseInt(batchProcessTimeoutStr);
+        }
+        String redeliveryDelaySecondsStr = properties.getProperty(PulsarMQConstants.PULSARMQ_REDELIVERY_DELAY_SECONDS);
+        if (StringUtils.isNotEmpty(redeliveryDelaySecondsStr)) {
+            this.redeliveryDelaySeconds = Integer.parseInt(redeliveryDelaySecondsStr);
+        }
+        String ackTimeoutSecondsStr = properties.getProperty(PulsarMQConstants.PULSARMQ_ACK_TIMEOUT_SECONDS);
+        if (StringUtils.isNotEmpty(ackTimeoutSecondsStr)) {
+            this.ackTimeoutSeconds = Integer.parseInt(ackTimeoutSecondsStr);
+        }
+        String isRetryStr = properties.getProperty(PulsarMQConstants.PULSARMQ_IS_RETRY);
+        if (StringUtils.isNotEmpty(isRetryStr)) {
+            this.isRetry = Boolean.parseBoolean(isRetryStr);
+        }
+        String isRetryDLQUpperCaseStr = properties.getProperty(PulsarMQConstants.PULSARMQ_IS_RETRY_DLQ_UPPERCASE);
+        if (StringUtils.isNotEmpty(isRetryDLQUpperCaseStr)) {
+            this.isRetryDLQUpperCase = Boolean.parseBoolean(isRetryDLQUpperCaseStr);
+        }
+        String maxRedeliveryCountStr = properties.getProperty(PulsarMQConstants.PULSARMQ_MAX_REDELIVERY_COUNT);
+        if (StringUtils.isNotEmpty(maxRedeliveryCountStr)) {
+            this.maxRedeliveryCount = Integer.parseInt(maxRedeliveryCountStr);
+        }
+    }
+
+    @Override
+    public void connect() {
+        if (isConsumerActive()) {
+            return;
+        }
+        // 连接创建客户端
+        try {
+            pulsarClient = PulsarClient.builder()
+                    .serviceUrl(serviceUrl)
+                    .authentication(AuthenticationFactory.token(roleToken))
+                    .build();
+        } catch (PulsarClientException e) {
+            throw new RuntimeException(e);
+        }
+        ConsumerBuilder<byte[]> builder = pulsarClient.newConsumer();
+        if (MQUtil.isPatternTopic(this.topic)) {
+            // 正则只支持一个
+            builder.topicsPattern(this.topic);
+        } else {// 多个topic
+            builder.topic(this.topic);
+        }
+        // 为保证消息的有序性，仅支持单消费实例模式
+        // 灾备模式，一个分区只能有一个消费者，如果当前消费者不可用，自动切换到其他消费者
+        builder.subscriptionType(SubscriptionType.Failover);
+
+        builder
+                // 调用consumer.negativeAcknowledge(message) （即nack）来表示消费失败的消息
+                // 在指定的时间进行重新消费，默认是1分钟。
+                .negativeAckRedeliveryDelay(this.redeliveryDelaySeconds, TimeUnit.SECONDS)
+                .subscriptionName(this.subscriptName)
+        ;
+        if (this.isRetry) {
+            DeadLetterPolicy.DeadLetterPolicyBuilder dlqBuilder = DeadLetterPolicy.builder()
+                    // 最大重试次数
+                    .maxRedeliverCount(this.maxRedeliveryCount);
+            // 指定重试队列，不是多个或通配符topic才能判断重试队列
+            if (!MQUtil.isPatternTag(this.topic)) {
+                String retryTopic = this.topic + (this.isRetryDLQUpperCase ? "-RETRY" : "-retry");
+                dlqBuilder.retryLetterTopic(retryTopic);
+                String dlqTopic = this.topic + (this.isRetryDLQUpperCase ? "-DLQ" : "-dlq");
+                dlqBuilder.deadLetterTopic(dlqTopic);
+            }
+
+            //默认关闭，如果需要重试则开启
+            builder.enableRetry(true)
+                    .deadLetterPolicy(dlqBuilder.build());
+        }
+
+        // ack超时
+        builder.ackTimeout(this.ackTimeoutSeconds, TimeUnit.SECONDS);
+
+        // pulsar批量获取消息设置
+        builder.batchReceivePolicy(new BatchReceivePolicy.Builder()
+                .maxNumMessages(this.batchSize)
+                .timeout(this.getBatchTimeoutSeconds, TimeUnit.SECONDS)
+                .build());
+
+        try {
+            this.pulsarMQConsumer = builder.subscribe();
+        } catch (PulsarClientException e) {
+            throw new CanalClientException("Subscript pulsar consumer error", e);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        List<CommonMessage> messageList = Lists.newArrayList();
+        try {
+            Messages<byte[]> messages = pulsarMQConsumer.batchReceive();
+
+            if (null == messages || messages.size() == 0) {
+                return messageList;
+            }
+            // 保存当前消费记录，用于ack和rollback
+            this.lastGetBatchMessage = messages;
+            for (org.apache.pulsar.client.api.Message<byte[]> msg : messages) {
+                byte[] data = msg.getData();
+                if (!this.flatMessage) {
+                    Message message = CanalMessageSerializerUtil.deserializer(data);
+                    List<CommonMessage> list = MessageUtil.convert(message);
+                    messageList.addAll(list);
+                } else {
+                    CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
+                    messageList.add(commonMessage);
+                }
+            }
+        } catch (PulsarClientException e) {
+            throw new CanalClientException("Receive pulsar batch message error", e);
+        }
+
+        return messageList;
+    }
+
+    @Override
+    public void rollback() {
+        try {
+            if (isConsumerActive() && hasLastMessages()) {
+                // 回滚所有消息
+                this.pulsarMQConsumer.negativeAcknowledge(this.lastGetBatchMessage);
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        try {
+            if (isConsumerActive() && hasLastMessages()) {
+                // 确认所有消息
+                this.pulsarMQConsumer.acknowledge(this.lastGetBatchMessage);
+            }
+        } catch (PulsarClientException e) {
+            if (isConsumerActive() && hasLastMessages()) {
+                this.pulsarMQConsumer.negativeAcknowledge(this.lastGetBatchMessage);
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        if (null == this.pulsarMQConsumer || !this.pulsarMQConsumer.isConnected()) {
+            return;
+        }
+        try {
+            this.pulsarMQConsumer.unsubscribe();
+            this.pulsarClient.close();
+        } catch (PulsarClientException e) {
+            throw new CanalClientException("Disconnect pulsar consumer error", e);
+        }
+    }
+
+    /**
+     * 是否消费可用
+     *
+     * @return true消费者可用
+     */
+    private boolean isConsumerActive() {
+        return null != this.pulsarMQConsumer && this.pulsarMQConsumer.isConnected();
+    }
+
+    /**
+     * 是否有未确认消息
+     *
+     * @return true有正在消费的待确认消息
+     */
+    private boolean hasLastMessages() {
+        return null != this.lastGetBatchMessage && this.lastGetBatchMessage.size() > 0;
+    }
+}

connector/pulsarmq-connector/src/main/java/com/alibaba/otter/canal/connector/pulsarmq/producer/CanalPulsarMQProducer.java

@@ -0,0 +1,422 @@
+package com.alibaba.otter.canal.connector.pulsarmq.producer;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.common.utils.PropertiesUtils;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
+import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQProducerConfig;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.client.api.*;
+import org.apache.pulsar.shade.com.google.gson.JsonParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * PulsarMQ Producer SPI 实现
+ *
+ * @author chad 2021/9/2
+ * @version 1.0.0
+ */
+@SPI("pulsarmq")
+public class CanalPulsarMQProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger logger = LoggerFactory.getLogger(CanalPulsarMQProducer.class);
+
+    private static final Map<String, Producer<byte[]>> PRODUCERS = new HashMap<>();
+
+    protected ThreadPoolExecutor sendPartitionExecutor;
+    /**
+     * 消息体分区属性名称
+     */
+    public static final String MSG_PROPERTY_PARTITION_NAME = "partitionNum";
+    /**
+     * pulsar客户端，管理连接
+     */
+    protected PulsarClient client;
+
+    @Override
+    public void init(Properties properties) {
+        // 加载配置
+        PulsarMQProducerConfig pulsarMQProducerConfig = new PulsarMQProducerConfig();
+        this.mqProperties = pulsarMQProducerConfig;
+        super.init(properties);
+        loadPulsarMQProperties(properties);
+
+        // 初始化连接客户端
+        try {
+            client = PulsarClient.builder()
+                    // 填写pulsar的连接地址
+                    .serviceUrl(pulsarMQProducerConfig.getServerUrl())
+                    // 角色权限认证的token
+                    .authentication(AuthenticationFactory.token(pulsarMQProducerConfig.getRoleToken()))
+                    .build();
+        } catch (PulsarClientException e) {
+            throw new RuntimeException(e);
+        }
+        // 加载所有生产者 --> topic可能有正则或表名，无法确认所有topic，在使用时再加载
+
+        int parallelPartitionSendThreadSize = mqProperties.getParallelSendThreadSize();
+        sendPartitionExecutor = new ThreadPoolExecutor(parallelPartitionSendThreadSize,
+                parallelPartitionSendThreadSize,
+                0,
+                TimeUnit.SECONDS,
+                new ArrayBlockingQueue<>(parallelPartitionSendThreadSize * 2),
+                new NamedThreadFactory("MQ-Parallel-Sender-Partition"),
+                new ThreadPoolExecutor.CallerRunsPolicy());
+    }
+
+    /**
+     * 加载配置
+     *
+     * @param properties
+     * @return void
+     * @date 2021/9/15 11:22
+     * @author chad
+     * @since 1 by chad at 2021/9/15 新增
+     */
+    private void loadPulsarMQProperties(Properties properties) {
+        PulsarMQProducerConfig tmpProperties = (PulsarMQProducerConfig) this.mqProperties;
+        String serverUrl = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_SERVER_URL);
+        if (!StringUtils.isEmpty(serverUrl)) {
+            tmpProperties.setServerUrl(serverUrl);
+        }
+
+        String roleToken = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_ROLE_TOKEN);
+        if (!StringUtils.isEmpty(roleToken)) {
+            tmpProperties.setRoleToken(roleToken);
+        }
+        String topicTenantPrefix = PropertiesUtils.getProperty(properties, PulsarMQConstants.PULSARMQ_TOPIC_TENANT_PREFIX);
+        if (!StringUtils.isEmpty(topicTenantPrefix)) {
+            tmpProperties.setTopicTenantPrefix(topicTenantPrefix);
+        }
+        if (logger.isDebugEnabled()) {
+            logger.debug("Load pulsar properties ==> {}", JSON.toJSON(this.mqProperties));
+        }
+    }
+
+    /**
+     * 发送消息，处理的任务：
+     * <p>
+     * 1. 动态 Topic，根据schema.table或schema来匹配topic配置，将改变发送到指定的一个或多个具体的Topic<br/>
+     * 2. 使用线程池发送多个消息，单个消息不使用线程池
+     * </p>
+     *
+     * @param destination 消息目标信息
+     * @param message     消息
+     * @param callback    消息发送结果回调
+     * @return void
+     * @date 2021/9/2 22:01
+     * @author chad
+     * @since 1.0.0 by chad at 2021/9/2: 新增
+     */
+    @Override
+    public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
+
+        ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
+                        destination.getTopic(),
+                        destination.getDynamicTopic());
+
+                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                    String topicName = entry.getKey().replace('.', '_');
+                    com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+                    template.submit(() -> {
+                        try {
+                            send(destination, topicName, messageSub);
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+
+                template.waitForResult();
+            } else {
+                send(destination, destination.getTopic(), message);
+            }
+
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    /**
+     * 发送单条消息到指定topic。区分是否发送扁平消息
+     *
+     * @param destination
+     * @param topicName
+     * @param message
+     * @return void
+     * @date 2021/9/2 22:05
+     * @author chad
+     * @since 1.0.0 by chad at 2021/9/2: 新增
+     */
+    public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
+
+        // 获取当前topic的分区数
+        Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
+                destination.getDynamicTopicPartitionNum());
+        if (partitionNum == null) {
+            partitionNum = destination.getPartitionsNum();
+        }
+        ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
+        // 并发构造
+        MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
+        if (!mqProperties.isFlatMessage()) {
+            // 动态计算目标分区
+            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                for (MQMessageUtils.EntryRowData r : datas) {
+                    CanalEntry.Entry entry = r.entry;
+                    if (null == entry) {
+                        continue;
+                    }
+                    // 串行分区
+                    com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
+                            message.getId(),
+                            partitionNum,
+                            destination.getPartitionHash(),
+                            mqProperties.isDatabaseHash());
+                    // 发送
+                    int len = messages.length;
+                    for (int i = 0; i < len; i++) {
+                        final int partition = i;
+                        com.alibaba.otter.canal.protocol.Message m = messages[i];
+                        template.submit(() -> {
+                            sendMessage(topicName, partition, m);
+                        });
+                    }
+                }
+            } else {
+                // 默认分区
+                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
+                sendMessage(topicName, partition, message);
+            }
+        } else {
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
+
+            // 初始化分区合并队列
+            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
+                int len = partitionNum;
+                for (int i = 0; i < len; i++) {
+                    partitionFlatMessages.add(new ArrayList<>());
+                }
+
+                for (FlatMessage flatMessage : flatMessages) {
+                    FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
+                            partitionNum,
+                            destination.getPartitionHash(),
+                            mqProperties.isDatabaseHash());
+                    int length = partitionFlatMessage.length;
+                    for (int i = 0; i < length; i++) {
+                        // 增加null判断,issue #3267
+                        if (partitionFlatMessage[i] != null) {
+                            partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
+                        }
+                    }
+                }
+
+                for (int i = 0; i < len; i++) {
+                    final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
+                    if (flatMessagePart != null && flatMessagePart.size() > 0) {
+                        final int partition = i;
+                        template.submit(() -> {
+                            // 批量发送
+                            sendMessage(topicName, partition, flatMessagePart);
+                        });
+                    }
+                }
+
+                // 批量等所有分区的结果
+                template.waitForResult();
+            } else {
+                // 默认分区
+                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
+                sendMessage(topicName, partition, flatMessages);
+            }
+        }
+    }
+
+    /**
+     * 发送原始消息，需要做分区处理
+     *
+     * @param topic        topic
+     * @param partitionNum 目标分区
+     * @param msg          原始消息内容
+     * @return void
+     * @date 2021/9/10 17:55
+     * @author chad
+     * @since 1 by chad at 2021/9/10 新增
+     */
+    private void sendMessage(String topic, int partitionNum, com.alibaba.otter.canal.protocol.Message msg) {
+        Producer<byte[]> producer = getProducer(topic);
+        byte[] msgBytes = CanalMessageSerializerUtil.serializer(msg, mqProperties.isFilterTransactionEntry());
+        try {
+            MessageId msgResultId = producer.newMessage()
+                    .property(MSG_PROPERTY_PARTITION_NAME, String.valueOf(partitionNum))
+                    .value(msgBytes).send();
+            // todo 判断发送结果
+            if (logger.isDebugEnabled()) {
+                logger.debug("Send Message to topic:{} Result: {}", topic, msgResultId);
+            }
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 发送扁平消息
+     *
+     * @param topic        topic主题
+     * @param flatMessages 扁平消息
+     * @return void
+     * @date 2021/9/10 18:22
+     * @author chad
+     * @since 1 by chad at 2021/9/10 新增
+     */
+    private void sendMessage(String topic, int partition, List<FlatMessage> flatMessages) {
+        Producer<byte[]> producer = getProducer(topic);
+        for (FlatMessage f : flatMessages) {
+            try {
+                MessageId msgResultId = producer
+                        .newMessage()
+                        .property(MSG_PROPERTY_PARTITION_NAME, String.valueOf(partition))
+                        .value(JSON.toJSONBytes(f, SerializerFeature.WriteMapNullValue))
+                        .send()
+                        //
+                        ;
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Send Messages to topic:{} Result: {}", topic, msgResultId);
+                }
+            } catch (Throwable e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    /**
+     * 获取指定topic的生产者，并且使用缓存
+     *
+     * @param topic
+     * @return org.apache.pulsar.client.api.Producer<byte [ ]>
+     * @date 2021/9/10 11:21
+     * @author chad
+     * @since 1 by chad at 2021/9/10 新增
+     */
+    private Producer<byte[]> getProducer(String topic) {
+        Producer producer = PRODUCERS.get(topic);
+
+        if (null == producer) {
+            try {
+                synchronized (PRODUCERS) {
+                    producer = PRODUCERS.get(topic);
+                    if (null != producer) {
+                        return producer;
+                    }
+
+                    // 拼接topic前缀
+                    PulsarMQProducerConfig pulsarMQProperties = (PulsarMQProducerConfig) this.mqProperties;
+                    String prefix = pulsarMQProperties.getTopicTenantPrefix();
+                    String fullTopic = topic;
+                    if (!StringUtils.isEmpty(prefix)) {
+                        if (!prefix.endsWith("/")) {
+                            fullTopic = "/" + fullTopic;
+                        }
+                        fullTopic = pulsarMQProperties.getTopicTenantPrefix() + fullTopic;
+                    }
+
+                    // 创建指定topic的生产者
+                    producer = client.newProducer()
+                            .topic(fullTopic)
+                            // 指定路由器
+                            .messageRouter(new MessageRouterImpl(topic))
+                            .create();
+                    // 放入缓存
+                    PRODUCERS.put(topic, producer);
+                }
+            } catch (PulsarClientException e) {
+                logger.error("create producer failed for topic: " + topic, e);
+                throw new RuntimeException(e);
+            }
+        }
+
+        return producer;
+    }
+
+    /**
+     * Pulsar自定义路由策略
+     *
+     * @author chad
+     * @version 1
+     * @since 1 by chad at 2021/9/10 新增
+     * @since 2 by chad at 2021/9/17 修改为msg自带目标分区
+     */
+    private static class MessageRouterImpl implements MessageRouter {
+        private String topicLocal;
+
+        public MessageRouterImpl(String topicLocal) {
+            this.topicLocal = topicLocal;
+        }
+
+        @Override
+        public int choosePartition(Message<?> msg, TopicMetadata metadata) {
+            String partitionStr = msg.getProperty(MSG_PROPERTY_PARTITION_NAME);
+            int partition = 0;
+            if (!StringUtils.isEmpty(partitionStr)) {
+                try {
+                    partition = Integer.parseInt(partitionStr);
+                } catch (NumberFormatException e) {
+                    logger.warn("Parse msg {} property failed for value: {}", MSG_PROPERTY_PARTITION_NAME, partitionStr);
+                }
+            }
+            // topic创建时设置的分区数
+            Integer partitionNum = metadata.numPartitions();
+            // 如果 partition 超出 partitionNum，取余数
+            if (null != partitionNum && partition >= partitionNum) {
+                partition = partition % partitionNum;
+            }
+            return partition;
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RocketMQ producer##");
+
+        for (Producer p : PRODUCERS.values()) {
+            try {
+                if (null != p && p.isConnected()) {
+                    p.close();
+                }
+            } catch (PulsarClientException e) {
+                logger.warn("close producer name: {}, topic: {}, error: {}", p.getProducerName(), p.getTopic(), e.getMessage());
+            }
+        }
+
+        super.stop();
+    }
+}

connector/pulsarmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+pulsarmq=com.alibaba.otter.canal.connector.pulsarmq.producer.CanalPulsarMQProducer

connector/pulsarmq-connector/src/test/java/com/alibaba/otter/canal/connector/pulsarmq/consumer/CanalPulsarMQConsumerTest.java

@@ -0,0 +1,76 @@
+package com.alibaba.otter.canal.connector.pulsarmq.consumer;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.pulsarmq.config.PulsarMQConstants;
+import org.apache.commons.lang.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author chad
+ * @date 2021/9/17 22:32
+ * @since
+ */
+public class CanalPulsarMQConsumerTest {
+
+    private Properties properties;
+
+    private CanalPulsarMQConsumer consumer;
+
+    @Before
+    public void before() {
+        properties = new Properties();
+        properties.setProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE, String.valueOf(true));
+        properties.setProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE, String.valueOf(30));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_GET_BATCH_TIMEOUT_SECONDS, String.valueOf(5));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_BATCH_PROCESS_TIMEOUT, String.valueOf(30));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_SERVER_URL, "pulsar://192.168.1.200:6650");
+        properties.setProperty(PulsarMQConstants.PULSARMQ_ROLE_TOKEN, "123456");
+        properties.setProperty(PulsarMQConstants.PULSARMQ_SUBSCRIPT_NAME, "test-for-canal-pulsar-consumer");
+        properties.setProperty(PulsarMQConstants.PULSARMQ_REDELIVERY_DELAY_SECONDS, String.valueOf(30));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_ACK_TIMEOUT_SECONDS, String.valueOf(30));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_IS_RETRY, String.valueOf(true));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_IS_RETRY_DLQ_UPPERCASE, String.valueOf(true));
+        properties.setProperty(PulsarMQConstants.PULSARMQ_MAX_REDELIVERY_COUNT, String.valueOf(16));
+
+        consumer = new CanalPulsarMQConsumer();
+        consumer.init(this.properties, "persistent://public/canal/test-topics-partition", "groupid");
+    }
+
+    @After
+    public void after() {
+        consumer.disconnect();
+    }
+
+    @Test
+    public void getMessage() {
+        consumer.connect();
+
+        while (true) {
+            List<CommonMessage> list = consumer.getMessage(-1L, TimeUnit.SECONDS);
+            String time = LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"));
+            if (null == list || list.isEmpty()) {
+                System.out.println(time + " Receive empty");
+                continue;
+            }
+            for (CommonMessage m : list) {
+                System.out.println(time + " Receive ==> " + JSON.toJSONString(m));
+            }
+
+            consumer.ack();
+        }
+    }
+}

connector/rabbitmq-connector/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.connector</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.connector.rabbitmq.consumer;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.BlockingQueue;
@@ -23,7 +24,6 @@ import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
 import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
-import com.google.common.collect.Lists;
 import com.rabbitmq.client.AMQP;
 import com.rabbitmq.client.Channel;
 import com.rabbitmq.client.Connection;
@@ -137,7 +137,7 @@ public class CanalRabbitMQConsumer implements CanalMsgConsumer {
         if (logger.isDebugEnabled()) {
             logger.debug("Get Message: {}", new String(messageData));
         }
-        List<CommonMessage> messageList = Lists.newArrayList();
+        List<CommonMessage> messageList = new ArrayList<>();
         if (!flatMessage) {
             Message message = CanalMessageSerializerUtil.deserializer(messageData);
             messageList.addAll(MessageUtil.convert(message));

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java

@@ -175,8 +175,8 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
     public void stop() {
         logger.info("## Stop RabbitMQ producer##");
         try {
-            this.connect.close();
             this.channel.close();
+            this.connect.close();
             super.stop();
         } catch (AlreadyClosedException ex) {
             logger.error("Connection is already closed", ex);

connector/rocketmq-connector/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.connector</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.connector.rocketmq.consumer;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.BlockingQueue;
@@ -30,7 +31,6 @@ import com.alibaba.otter.canal.connector.core.util.MessageUtil;
 import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
-import com.google.common.collect.Lists;
 
 /**
  * RocketMQ consumer SPI 实现
@@ -142,7 +142,7 @@ public class CanalRocketMQConsumer implements CanalMsgConsumer {
         if (logger.isDebugEnabled()) {
             logger.debug("Get Message: {}", messageExts);
         }
-        List<CommonMessage> messageList = Lists.newArrayList();
+        List<CommonMessage> messageList = new ArrayList<>();
         for (MessageExt messageExt : messageExts) {
             byte[] data = messageExt.getBody();
             if (data != null) {

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java

@@ -1,29 +1,5 @@
 package com.alibaba.otter.canal.connector.rocketmq.producer;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.rocketmq.acl.common.AclClientRPCHook;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.client.AccessChannel;
-import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
-import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
-import org.apache.rocketmq.client.producer.DefaultMQProducer;
-import org.apache.rocketmq.client.producer.SendResult;
-import org.apache.rocketmq.common.message.Message;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.remoting.RPCHook;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
 import com.alibaba.otter.canal.common.CanalException;
@@ -40,6 +16,29 @@ import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
 import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
 import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
 import com.alibaba.otter.canal.protocol.FlatMessage;
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.AccessChannel;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 /**
  * RocketMQ Producer SPI 实现
@@ -47,17 +46,15 @@ import com.alibaba.otter.canal.protocol.FlatMessage;
  * @author rewerma 2020-01-27
  * @version 1.0.0
  */
-@SPI("rocketmq")
-public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
+@SPI("rocketmq") public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
 
-    private static final Logger  logger               = LoggerFactory.getLogger(CanalRocketMQProducer.class);
+    private static final Logger logger = LoggerFactory.getLogger(CanalRocketMQProducer.class);
 
-    private DefaultMQProducer    defaultMQProducer;
-    private static final String  CLOUD_ACCESS_CHANNEL = "cloud";
-    protected ThreadPoolExecutor sendPartitionExecutor;
+    private              DefaultMQProducer  defaultMQProducer;
+    private static final String             CLOUD_ACCESS_CHANNEL = "cloud";
+    protected            ThreadPoolExecutor sendPartitionExecutor;
 
-    @Override
-    public void init(Properties properties) {
+    @Override public void init(Properties properties) {
         RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
         this.mqProperties = rocketMQProperties;
         super.init(properties);
@@ -72,9 +69,9 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
         }
 
         defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
-            rpcHook,
-            rocketMQProperties.isEnableMessageTrace(),
-            rocketMQProperties.getCustomizedTraceTopic());
+                rpcHook,
+                rocketMQProperties.isEnableMessageTrace(),
+                rocketMQProperties.getCustomizedTraceTopic());
         if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
             defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
         }
@@ -93,12 +90,12 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
 
         int parallelPartitionSendThreadSize = mqProperties.getParallelSendThreadSize();
         sendPartitionExecutor = new ThreadPoolExecutor(parallelPartitionSendThreadSize,
-            parallelPartitionSendThreadSize,
-            0,
-            TimeUnit.SECONDS,
-            new ArrayBlockingQueue<>(parallelPartitionSendThreadSize * 2),
-            new NamedThreadFactory("MQ-Parallel-Sender-Partition"),
-            new ThreadPoolExecutor.CallerRunsPolicy());
+                parallelPartitionSendThreadSize,
+                0,
+                TimeUnit.SECONDS,
+                new ArrayBlockingQueue<>(parallelPartitionSendThreadSize * 2),
+                new NamedThreadFactory("MQ-Parallel-Sender-Partition"),
+                new ThreadPoolExecutor.CallerRunsPolicy());
     }
 
     private void loadRocketMQProperties(Properties properties) {
@@ -114,12 +111,12 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
             rocketMQProperties.setProducerGroup(producerGroup);
         }
         String enableMessageTrace = PropertiesUtils.getProperty(properties,
-            RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
+                RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
         if (!StringUtils.isEmpty(enableMessageTrace)) {
             rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
         }
         String customizedTraceTopic = PropertiesUtils.getProperty(properties,
-            RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
+                RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
         if (!StringUtils.isEmpty(customizedTraceTopic)) {
             rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
         }
@@ -136,7 +133,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
             rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
         }
         String vipChannelEnabled = PropertiesUtils.getProperty(properties,
-            RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
+                RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
         if (!StringUtils.isEmpty(vipChannelEnabled)) {
             rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
         }
@@ -146,15 +143,15 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
         }
     }
 
-    @Override
-    public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
+    @Override public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message,
+                               Callback callback) {
         ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
         try {
             if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
                 // 动态topic
                 Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
-                    destination.getTopic(),
-                    destination.getDynamicTopic());
+                        destination.getTopic(),
+                        destination.getDynamicTopic());
 
                 for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
                     String topicName = entry.getKey().replace('.', '_');
@@ -182,10 +179,17 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
         }
     }
 
-    public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
+    public void send(final MQDestination destination, String topicName,
+                     com.alibaba.otter.canal.protocol.Message message) {
         // 获取当前topic的分区数
         Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName,
-            destination.getDynamicTopicPartitionNum());
+                destination.getDynamicTopicPartitionNum());
+
+        // 获取topic的队列数为分区数
+        if (partitionNum == null) {
+            partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
+        }
+
         if (partitionNum == null) {
             partitionNum = destination.getPartitionsNum();
         }
@@ -195,10 +199,10 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
                 MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
                 // 串行分区
                 com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
-                    message.getId(),
-                    partitionNum,
-                    destination.getPartitionHash(),
-                    mqProperties.isDatabaseHash());
+                        message.getId(),
+                        partitionNum,
+                        destination.getPartitionHash(),
+                        mqProperties.isDatabaseHash());
                 int length = messages.length;
 
                 ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
@@ -208,9 +212,9 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
                         final int index = i;
                         template.submit(() -> {
                             Message data = new Message(topicName,
-                                ((RocketMQProducerConfig) this.mqProperties).getTag(),
-                                CanalMessageSerializerUtil.serializer(dataPartition,
-                                    mqProperties.isFilterTransactionEntry()));
+                                    ((RocketMQProducerConfig) this.mqProperties).getTag(),
+                                    CanalMessageSerializerUtil.serializer(dataPartition,
+                                            mqProperties.isFilterTransactionEntry()));
                             sendMessage(data, index);
                         });
                     }
@@ -220,8 +224,8 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
             } else {
                 final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
                 Message data = new Message(topicName,
-                    ((RocketMQProducerConfig) this.mqProperties).getTag(),
-                    CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
+                        ((RocketMQProducerConfig) this.mqProperties).getTag(),
+                        CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
                 sendMessage(data, partition);
             }
         } else {
@@ -232,15 +236,15 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
             // 初始化分区合并队列
             if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
                 List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
-                for (int i = 0; i < destination.getPartitionsNum(); i++) {
+                for (int i = 0; i < partitionNum; i++) {
                     partitionFlatMessages.add(new ArrayList<>());
                 }
 
                 for (FlatMessage flatMessage : flatMessages) {
                     FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
-                        partitionNum,
-                        destination.getPartitionHash(),
-                        mqProperties.isDatabaseHash());
+                            partitionNum,
+                            destination.getPartitionHash(),
+                            mqProperties.isDatabaseHash());
                     int length = partitionFlatMessage.length;
                     for (int i = 0; i < length; i++) {
                         // 增加null判断,issue #3267
@@ -257,10 +261,10 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
                         final int index = i;
                         template.submit(() -> {
                             List<Message> messages = flatMessagePart.stream()
-                                .map(flatMessage -> new Message(topicName,
-                                    ((RocketMQProducerConfig) this.mqProperties).getTag(),
-                                    JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
-                                .collect(Collectors.toList());
+                                    .map(flatMessage -> new Message(topicName,
+                                            ((RocketMQProducerConfig) this.mqProperties).getTag(),
+                                            JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
+                                    .collect(Collectors.toList());
                             // 批量发送
                             sendMessage(messages, index);
                         });
@@ -272,10 +276,10 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
             } else {
                 final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
                 List<Message> messages = flatMessages.stream()
-                    .map(flatMessage -> new Message(topicName,
-                        ((RocketMQProducerConfig) this.mqProperties).getTag(),
-                        JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
-                    .collect(Collectors.toList());
+                        .map(flatMessage -> new Message(topicName,
+                                ((RocketMQProducerConfig) this.mqProperties).getTag(),
+                                JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
+                        .collect(Collectors.toList());
                 // 批量发送
                 sendMessage(messages, partition);
             }
@@ -300,8 +304,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
         }
     }
 
-    @SuppressWarnings("deprecation")
-    private void sendMessage(List<Message> messages, int partition) {
+    @SuppressWarnings("deprecation") private void sendMessage(List<Message> messages, int partition) {
         if (messages.isEmpty()) {
             return;
         }
@@ -331,6 +334,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
                 }
 
                 try {
+                    // 阿里云RocketMQ暂不支持批量发送消息，当canal.mq.flatMessage = true时，会发送失败
                     SendResult sendResult = this.defaultMQProducer.send(messages, queue);
                     if (logger.isDebugEnabled()) {
                         logger.debug("Send Message Result: {}", sendResult);
@@ -342,8 +346,7 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
         }
     }
 
-    @Override
-    public void stop() {
+    @Override public void stop() {
         logger.info("## Stop RocketMQ producer##");
         this.defaultMQProducer.shutdown();
         if (sendPartitionExecutor != null) {
@@ -352,4 +355,18 @@ public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQ
 
         super.stop();
     }
+
+    private Integer getTopicDynamicQueuesSize(Boolean enable, String topicName) {
+        if (enable != null && enable) {
+            topicName = this.defaultMQProducer.withNamespace(topicName);
+            DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
+            TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(topicName);
+            if (topicInfo == null) {
+                return null;
+            } else {
+                return topicInfo.getMessageQueueList().size();
+            }
+        }
+        return null;
+    }
 }

connector/tcp-connector/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.connector</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

dbsync/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.5</version>
+		<version>1.1.6-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

deployer/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>com.alibaba.otter</groupId>
         <artifactId>canal</artifactId>
-        <version>1.1.5</version>
+        <version>1.1.6-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>com.alibaba.otter</groupId>

deployer/src/main/bin/startup.bat

@@ -7,7 +7,8 @@ if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
 
 set conf_dir=%ENV_PATH%\..\conf
 set canal_conf=%conf_dir%\canal.properties
-@rem set canal_local_conf=%conf_dir%\canal_local.properties
+@rem set canal_conf=%conf_dir%\canal_local.properties
+if "%1" == "local" set canal_conf=%conf_dir%\canal_local.properties
 set logback_configurationFile=%conf_dir%\logback.xml
 
 set CLASSPATH=%conf_dir%
@@ -23,4 +24,4 @@ set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
 set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher
 echo start cmd : %CMD_STR%
 
-java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher
+java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher

deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.deployer.monitor;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
@@ -77,9 +78,9 @@ public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle impleme
         }
 
         final List<String> is = Lists.newArrayList(StringUtils.split(instances, ','));
-        List<String> start = Lists.newArrayList();
-        List<String> stop = Lists.newArrayList();
-        List<String> restart = Lists.newArrayList();
+        List<String> start = new ArrayList<>();
+        List<String> stop = new ArrayList<>();
+        List<String> restart = new ArrayList<>();
         for (String instance : is) {
             if (!configs.containsKey(instance)) {
                 PlainCanal newPlainCanal = configClient.findInstance(instance, null);

deployer/src/main/resources/canal.properties

@@ -25,7 +25,7 @@ canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, rocketMQ, rabbitMQ
+# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -171,4 +171,12 @@ rabbitmq.virtual.host =
 rabbitmq.exchange =
 rabbitmq.username =
 rabbitmq.password =
-rabbitmq.deliveryMode =
+rabbitmq.deliveryMode =
+
+
+##################################################
+######### 		      Pulsar         #############
+##################################################
+pulsarmq.serverUrl =
+pulsarmq.roleToken =
+pulsarmq.topicTenantPrefix =

deployer/src/main/resources/logback.xml

@@ -85,6 +85,10 @@
 		<level value="INFO" />
 		<appender-ref ref="CANAL-ROOT" />
 	</logger>
+	<logger name="com.alibaba.otter.canal.connector.pulsarmq" additivity="false">
+		<level value="INFO" />
+		<appender-ref ref="CANAL-ROOT" />
+	</logger>
 	<root level="WARN">
 		<!-- <appender-ref ref="STDOUT"/>  -->
 		<appender-ref ref="CANAL-ROOT" />

deployer/src/main/resources/spring/default-instance.xml

@@ -206,5 +206,6 @@
 		<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
 		<property name="partitionHash" value="${canal.mq.partitionHash}" />
 		<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
+		<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
 	</bean>
 </beans>

deployer/src/main/resources/spring/file-instance.xml

@@ -192,5 +192,6 @@
         <property name="partitionsNum" value="${canal.mq.partitionsNum}" />
         <property name="partitionHash" value="${canal.mq.partitionHash}" />
 		<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
+		<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
 	</bean>
 </beans>

deployer/src/main/resources/spring/group-instance.xml

@@ -287,5 +287,6 @@
         <property name="partitionsNum" value="${canal.mq.partitionsNum}" />
         <property name="partitionHash" value="${canal.mq.partitionHash}" />
 		<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
+		<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
     </bean>
 </beans>

deployer/src/main/resources/spring/memory-instance.xml

@@ -180,5 +180,6 @@
 		<property name="partitionsNum" value="${canal.mq.partitionsNum}" />
 		<property name="partitionHash" value="${canal.mq.partitionHash}" />
 		<property name="dynamicTopicPartitionNum" value="${canal.mq.dynamicTopicPartitionNum}" />
+		<property name="enableDynamicQueuePartition" value="${canal.mq.enableDynamicQueuePartition}" />
 	</bean>
 </beans>

docker/image/app.sh

@@ -141,6 +141,6 @@ waitterm
 echo "==> STOP"
 
 stop_canal
-start_exporter
+stop_exporter
 
-echo "==> STOP SUCCESSFUL ..."
+echo "==> STOP SUCCESSFUL ..."

driver/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.5</version>
+		<version>1.1.6-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java

@@ -46,7 +46,7 @@ public class MysqlConnector {
 
     private SocketChannel       channel;
     private volatile boolean    dumping           = false;
-    // mysql connectinnId
+    // mysql connectionId
     private long                connectionId      = -1;
     private AtomicBoolean       connected         = new AtomicBoolean(false);
     // serverVersion
@@ -169,7 +169,7 @@ public class MysqlConnector {
             } else if (body[0] == -2) {
                 throw new IOException("Unexpected EOF packet at handshake phase.");
             } else {
-                throw new IOException("unpexpected packet with field_count=" + body[0]);
+                throw new IOException("Unexpected packet with field_count=" + body[0]);
             }
         }
         HandshakeInitializationPacket handshakePacket = new HandshakeInitializationPacket();
@@ -273,7 +273,7 @@ public class MysqlConnector {
                 err.fromBytes(body);
                 throw new IOException("Error When doing Client Authentication:" + err.toString());
             } else {
-                throw new IOException("unpexpected packet with field_count=" + body[0]);
+                throw new IOException("Unexpected packet with field_count=" + body[0]);
             }
         }
     }
@@ -304,7 +304,7 @@ public class MysqlConnector {
                 err.fromBytes(body);
                 throw new IOException("Error When doing Client Authentication:" + err.toString());
             default:
-                throw new IOException("unpexpected packet with field_count=" + body[0]);
+                throw new IOException("Unexpected packet with field_count=" + body[0]);
         }
     }
 

example/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.5</version>
+		<version>1.1.6-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>

example/src/main/java/com/alibaba/otter/canal/example/BaseCanalClientTest.java

@@ -132,14 +132,14 @@ public class BaseCanalClientTest {
             }
 
             if (entry.getEntryType() == EntryType.ROWDATA) {
-                RowChange rowChage = null;
+                RowChange rowChange = null;
                 try {
-                    rowChage = RowChange.parseFrom(entry.getStoreValue());
+                    rowChange = RowChange.parseFrom(entry.getStoreValue());
                 } catch (Exception e) {
                     throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                 }
 
-                EventType eventType = rowChage.getEventType();
+                EventType eventType = rowChange.getEventType();
 
                 logger.info(row_format,
                     new Object[] { entry.getHeader().getLogfileName(),
@@ -148,13 +148,13 @@ public class BaseCanalClientTest {
                             String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date),
                             entry.getHeader().getGtid(), String.valueOf(delayTime) });
 
-                if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
-                    logger.info("ddl : " + rowChage.getIsDdl() + " ,  sql ----> " + rowChage.getSql() + SEP);
+                if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
+                    logger.info("ddl : " + rowChange.getIsDdl() + " ,  sql ----> " + rowChange.getSql() + SEP);
                     continue;
                 }
 
-                printXAInfo(rowChage.getPropsList());
-                for (RowData rowData : rowChage.getRowDatasList()) {
+                printXAInfo(rowChange.getPropsList());
+                for (RowData rowData : rowChange.getRowDatasList()) {
                     if (eventType == EventType.DELETE) {
                         printColumn(rowData.getBeforeColumnsList());
                     } else if (eventType == EventType.INSERT) {
@@ -217,7 +217,7 @@ public class BaseCanalClientTest {
 
     /**
      * 获取当前Entry的 GTID信息示例
-     * 
+     *
      * @param header
      * @return
      */
@@ -235,7 +235,7 @@ public class BaseCanalClientTest {
 
     /**
      * 获取当前Entry的 GTID Sequence No信息示例
-     * 
+     *
      * @param header
      * @return
      */
@@ -253,7 +253,7 @@ public class BaseCanalClientTest {
 
     /**
      * 获取当前Entry的 GTID Last Committed信息示例
-     * 
+     *
      * @param header
      * @return
      */

example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientExample.java

@@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
 
 /**
- * Kafka client example
+ * RocketMQ client example
  *
  * @author machengyuan @ 2018-6-12
  * @version 1.0.0

example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientFlatMessageExample.java

@@ -11,7 +11,7 @@ import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 
 /**
- * Kafka client example
+ * RocketMQ client example
  *
  * @author machengyuan @ 2018-6-12
  * @version 1.0.0

filter/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.1.5</version>
+		<version>1.1.6-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>