Revert "修复测试类报错 (#4516)"

This reverts commit d899981354.

README.md

@@ -4,8 +4,6 @@
 ![license](https://img.shields.io/github/license/alibaba/canal.svg)
 [![average time to resolve an issue](http://isitmaintained.com/badge/resolution/alibaba/canal.svg)](http://isitmaintained.com/project/alibaba/canal "average time to resolve an issue")
 [![percentage of issues still open](http://isitmaintained.com/badge/open/alibaba/canal.svg)](http://isitmaintained.com/project/alibaba/canal "percentage of issues still open")
-[![Leaderboard](https://img.shields.io/badge/Canal-%E6%9F%A5%E7%9C%8B%E8%B4%A1%E7%8C%AE%E6%8E%92%E8%A1%8C%E6%A6%9C-orange)](https://opensource.alibaba.com/contribution_leaderboard/details?projectValue=canal)
-
 
 ## 简介
 
@@ -87,16 +85,11 @@ canal 特别设计了 client-server 模式，交互协议使用 protobuf 3.0 , c
 - canal php客户端: [https://github.com/xingwenge/canal-php](https://github.com/xingwenge/canal-php)
 - canal Python客户端：[https://github.com/haozi3156666/canal-python](https://github.com/haozi3156666/canal-python)
 - canal Rust客户端：[https://github.com/laohanlinux/canal-rs](https://github.com/laohanlinux/canal-rs)
-- canal Nodejs客户端：[https://github.com/marmot-z/canal-nodejs](https://github.com/marmot-z/canal-nodejs)
 
 canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递到 MQ 系统中，比如 Kafka/RocketMQ，可以借助于 MQ 的多语言能力 
 
 - 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart)
 
-## 基于canal开发的工具
-
--  canal2sql(基于binlog生成SQL) : [https://github.com/zhuchao941/canal2sql]
-
 ## 相关开源&产品
 
 - [canal 消费端开源项目: Otter](http://github.com/alibaba/otter)
@@ -107,5 +100,3 @@ canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递
 
 ## 问题反馈
 - 报告 issue: [github issues](https://github.com/alibaba/canal/issues)
-
-本项目的Issues会被同步沉淀至[阿里云开发者社区](https://developer.aliyun.com/ask)

admin/admin-ui/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <artifactId>canal.admin</artifactId>
     <groupId>com.alibaba.otter</groupId>
-    <version>1.1.8-SNAPSHOT</version>
+    <version>1.1.7-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

admin/admin-ui/src/views/canalServer/CanalInstanceUpdate.vue

@@ -59,7 +59,7 @@ export default {
       canalInstanceDetail(this.$route.query.id).then(response => {
         const data = response.data
         this.form.id = data.id
-        this.form.name = data.name + '/instance.properties'
+        this.form.name = data.name + '/instance.propertios'
         this.form.content = data.content
         this.form.clusterServerId = data.clusterServerId
       })

admin/admin-ui/src/views/login/index.vue

@@ -66,8 +66,8 @@ export default {
       }
     }
     const validatePassword = (rule, value, callback) => {
-      if (value.length < 5) {
+      if (value.length < 6) {
-        callback(new Error('The password can not be less than 5 digits'))
+        callback(new Error('The password can not be less than 6 digits'))
       } else {
         callback()
       }

admin/admin-web/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.admin</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>canal.admin-web</artifactId>
@@ -30,8 +30,6 @@
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
-            <!-- 支持arm64需要mysql 8.0,升级为8.0的新版驱动 -->
-            <version>8.0.33</version>
         </dependency>
         <dependency>
             <groupId>com.github.ben-manes.caffeine</groupId>

admin/admin-web/src/main/bin/startup.sh

@@ -59,7 +59,7 @@ esac
 JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk  -F '.' '{print $1}'`
 str=`file -L $JAVA | grep 64-bit`
 
-JAVA_OPTS="$JAVA_OPTS -Xss1m -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
+JAVA_OPTS="$JAVA_OPTS -Xss256k -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
 if [ $JavaVersion -ge 11 ] ; then
   #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
   JAVA_OPTS="$JAVA_OPTS"
@@ -69,12 +69,9 @@ else
 fi
 
 if [ -n "$str" ]; then
-	if [ $JavaVersion -ge 11 ] ; then
+	# JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
-    # For G1
+  # For G1
-    JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
+  JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
-  else
-	  JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
-	fi
 else
 	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
 fi

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/EbeanConfig.java

@@ -10,7 +10,6 @@ import java.util.List;
 
 import javax.sql.DataSource;
 
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/WebConfig.java

@@ -5,8 +5,6 @@ import java.io.PrintWriter;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.commons.lang.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
@@ -28,24 +26,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 @Configuration
 public class WebConfig implements WebMvcConfigurer {
 
-    @Value(value = "${canal.adminUser}")
-    private String user;
-
-    @Value(value = "${canal.adminPasswd}")
-    private String passwd;
-
     @Override
     public void addInterceptors(InterceptorRegistry registry) {
-        if (StringUtils.isEmpty(user)) {
-            throw new IllegalArgumentException(
-                "canal.adminUser is empty , pls check https://github.com/alibaba/canal/issues/4941");
-        }
-
-        if (StringUtils.isEmpty(passwd)) {
-            throw new IllegalArgumentException(
-                "canal.adminPasswd is empty , pls check https://github.com/alibaba/canal/issues/4941");
-        }
-
         registry.addInterceptor(new HandlerInterceptor() {
 
             @Override

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java

@@ -38,12 +38,12 @@ public class SimpleAdminConnectors {
     }
 
     private static String getDesensitizationPassword(String password) {
-        String defaultPassword = "*****";
+        String defaultPassword = "******";
 
-        if (Strings.isNullOrEmpty(password) || password.length() < 5) {
+        if (Strings.isNullOrEmpty(password) || password.length() < 6) {
             return defaultPassword;
         } else {
-            return String.format("%s*****", password.substring(0, 3));
+            return String.format("%s******", password.substring(0, 3));
         }
     }
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/PollingConfigController.java

@@ -2,11 +2,15 @@ package com.alibaba.otter.canal.admin.controller;
 
 import java.security.NoSuchAlgorithmException;
 
-import org.apache.commons.lang.RandomStringUtils;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestHeader;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
 
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalConfig;
@@ -24,7 +28,7 @@ import com.alibaba.otter.canal.protocol.SecurityUtil;
 @RequestMapping("/api/{env}/config")
 public class PollingConfigController {
 
-    private static final byte[] seeds = RandomStringUtils.random(16).getBytes();
+    private static final byte[] seeds = "canal is best!".getBytes();
 
     @Value(value = "${canal.adminUser}")
     String                      user;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/UserServiceImpl.java

@@ -21,7 +21,7 @@ public class UserServiceImpl implements UserService {
 
     private static byte[] seeds = "canal is best!".getBytes();
 
-    private static final Integer PASSWORD_LENGTH = 5;
+    private static final Integer PASSWORD_LENGTH = 6;
 
     public User find4Login(String username, String password) {
         if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) {

admin/admin-web/src/main/resources/application.yml

@@ -18,4 +18,4 @@ spring.datasource:
 
 canal:
   adminUser: admin
-  adminPasswd:
+  adminPasswd: admin

admin/admin-web/src/main/resources/canal-template.properties

@@ -25,7 +25,7 @@ canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
+# tcp, kafka, rocketMQ, rabbitMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -148,13 +148,8 @@ kafka.max.in.flight.requests.per.connection = 1
 kafka.retries = 0
 
 kafka.kerberos.enable = false
-kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
+kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
-kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf
+kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
-
-# sasl demo
-# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
-# kafka.sasl.mechanism = SCRAM-SHA-512
-# kafka.security.protocol = SASL_PLAINTEXT
 
 ##################################################
 ######### 		    RocketMQ	     #############
@@ -177,11 +172,3 @@ rabbitmq.exchange =
 rabbitmq.username =
 rabbitmq.password =
 rabbitmq.deliveryMode =
-
-
-##################################################
-######### 		      Pulsar         #############
-##################################################
-pulsarmq.serverUrl =
-pulsarmq.roleToken =
-pulsarmq.topicTenantPrefix =

admin/pom.xml

@@ -1,15 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <groupId>com.alibaba.otter</groupId>
-        <artifactId>canal</artifactId>
-        <version>1.1.8-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.admin</artifactId>
-    <version>1.1.8-SNAPSHOT</version>
+    <version>1.1.7-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal admin module for otter ${project.version}</name>
     <modules>
@@ -24,6 +18,7 @@
         <java_source_version>1.8</java_source_version>
         <java_target_version>1.8</java_target_version>
         <file_encoding>UTF-8</file_encoding>
+        <log4j_version>2.17.0</log4j_version>
     </properties>
 
     <dependencyManagement>
@@ -31,25 +26,87 @@
             <dependency>
                 <groupId>org.springframework.cloud</groupId>
                 <artifactId>spring-cloud-context</artifactId>
-                <version>3.0.6</version>
+                <version>2.0.0.RELEASE</version>
             </dependency>
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-core</artifactId>
+                <version>5.0.5.RELEASE</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-aop</artifactId>
+                <version>5.0.5.RELEASE</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-context</artifactId>
+                <version>5.0.5.RELEASE</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-jdbc</artifactId>
+                <version>5.0.5.RELEASE</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-orm</artifactId>
+                <version>5.0.5.RELEASE</version>
+            </dependency>
+            <dependency>
+                <groupId>org.springframework</groupId>
+                <artifactId>spring-test</artifactId>
+                <version>5.0.5.RELEASE</version>
+                <scope>test</scope>
+            </dependency>
+
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>
-                <version>2.5.4</version>
+                <version>2.0.1.RELEASE</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>mysql</groupId>
+                <artifactId>mysql-connector-java</artifactId>
+                <version>5.1.48</version>
+            </dependency>
             <dependency>
                 <groupId>com.github.ben-manes.caffeine</groupId>
                 <artifactId>caffeine</artifactId>
                 <version>2.6.2</version>
             </dependency>
+            <dependency>
+                <groupId>commons-beanutils</groupId>
+                <artifactId>commons-beanutils</artifactId>
+                <version>1.8.2</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-lang</groupId>
+                <artifactId>commons-lang</artifactId>
+                <version>2.6</version>
+            </dependency>
             <dependency>
                 <groupId>io.ebean</groupId>
                 <artifactId>ebean</artifactId>
                 <version>11.41.1</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-core</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-api</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-to-slf4j</artifactId>
+                <version>${log4j_version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

client-adapter/clickhouse/pom.xml

@@ -1,84 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>canal.client-adapter</artifactId>
-        <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>com.alibaba.otter</groupId>
-    <artifactId>client-adapter.clickhouse</artifactId>
-    <packaging>jar</packaging>
-    <name>canal client adapter clickhouse module for otter ${project.version}</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>client-adapter.common</artifactId>
-            <version>${project.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>ru.yandex.clickhouse</groupId>
-            <artifactId>clickhouse-jdbc</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <configuration>
-                            <tasks>
-                                <copy todir="${project.basedir}/../launcher/target/classes/clickhouse" overwrite="true">
-                                    <fileset dir="${project.basedir}/target/classes/clickhouse" erroronmissingdir="true">
-                                        <include name="*.yml" />
-                                    </fileset>
-                                </copy>
-                            </tasks>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseAdapter.java

@@ -1,340 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.druid.filter.stat.StatFilter;
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.druid.util.JdbcUtils;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.ConfigLoader;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MirrorDbConfig;
-import com.alibaba.otter.canal.client.adapter.clickhouse.monitor.ClickHouseConfigMonitor;
-import com.alibaba.otter.canal.client.adapter.clickhouse.service.ClickHouseBatchSyncService;
-import com.alibaba.otter.canal.client.adapter.clickhouse.service.ClickHouseEtlService;
-import com.alibaba.otter.canal.client.adapter.clickhouse.service.ClickHouseMirrorDbBatchSyncService;
-import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.*;
-
-/**
- * ClickHouse Adapter implementation
- *
- * @author: Xander
- * @date: Created in 2023/11/10 1:13
- * @email: zhrunxin33@gmail.com
- */
-@SPI
-public class ClickHouseAdapter implements OuterAdapter {
-
-    private static final Logger                         logger = LoggerFactory.getLogger(ClickHouseAdapter.class);
-
-    // Store the mapping of filename and configuration, load yml files below
-    // resource path
-    private Map<String, MappingConfig>              clickHouseMapping   = new ConcurrentHashMap<>();
-
-    // Schema -> Table -> MappingConfig
-    private Map<String, Map<String, MappingConfig>> mappingConfigCache  = new ConcurrentHashMap<>();
-
-    // Mirror DB Configuration, don't need to load column mapping
-    private Map<String, MirrorDbConfig>             mirrorDbConfigCache = new ConcurrentHashMap<>();
-
-    private DruidDataSource                             dataSource;
-
-    private ClickHouseBatchSyncService clickHouseBatchSyncService;
-
-    private ClickHouseMirrorDbBatchSyncService clickHouseMirrorDbBatchSyncService;
-
-    private Properties                                  envProperties;
-
-    // Launch configuration
-    private OuterAdapterConfig                      configuration;
-
-    private ClickHouseConfigMonitor clickHouseConfigMonitor;
-
-    public Map<String, MappingConfig> getClickHouseMapping() {
-        return clickHouseMapping;
-    }
-
-    @Override
-    public void init(OuterAdapterConfig configuration, Properties envProperties) {
-        this.envProperties = envProperties;
-        this.configuration = configuration;
-        // Load DB type from adapter.launch/bootstrap.yml
-        Map<String, String> properties = configuration.getProperties();
-        String dbType = JdbcUtils.getDbType(properties.get("jdbc.url"), null);
-        // 当.yml文件编码格式存在问题，此处clickhouse yml文件构建 可能会抛出异常
-        Map<String, MappingConfig> clickHouseMappingTmp = ConfigLoader.load(envProperties);
-        // 过滤不匹配的key的配置
-        clickHouseMappingTmp.forEach((key, config) -> {
-            addConfig(key, config);
-        });
-
-        if (clickHouseMapping.isEmpty()) {
-            throw new RuntimeException("No clickhouse adapter found for config key: " + configuration.getKey());
-        }
-
-        // 初始化连接池
-        dataSource = new DruidDataSource();
-        dataSource.setDriverClassName(properties.get("jdbc.driverClassName"));
-        dataSource.setUrl(properties.get("jdbc.url"));
-        dataSource.setUsername(properties.get("jdbc.username"));
-        dataSource.setPassword(properties.get("jdbc.password"));
-        dataSource.setInitialSize(1);
-        dataSource.setMinIdle(1);
-        dataSource.setMaxActive(30);
-        dataSource.setMaxWait(60000);
-        dataSource.setTimeBetweenEvictionRunsMillis(60000);
-        dataSource.setMinEvictableIdleTimeMillis(300000);
-        dataSource.setUseUnfairLock(true);
-        dataSource.setDefaultAutoCommit(false); // disable auto commit or disable Transactional
-        dataSource.setDbType(dbType);
-
-        if ("true".equals(properties.getOrDefault("druid.stat.enable", "true"))) {
-            StatFilter statFilter = new StatFilter();
-            statFilter.setSlowSqlMillis(Long.parseLong(properties.getOrDefault("druid.stat.slowSqlMillis", "1000")));
-            statFilter.setMergeSql(true);
-            statFilter.setLogSlowSql(true);
-            dataSource.setProxyFilters(Collections.singletonList(statFilter));
-        }
-
-        try {
-            dataSource.init();
-        } catch (SQLException e) {
-            logger.error("ERROR ## failed to initial datasource: " + properties.get("jdbc.url"), e);
-        }
-
-        String threads = properties.get("threads");
-        String scheduleTime = properties.get("scheduleTime");
-        String batchSize = properties.get("batchSize");
-
-        boolean skipDupException = BooleanUtils.toBoolean(configuration.getProperties()
-                .getOrDefault("skipDupException", "true"));
-        clickHouseBatchSyncService = new ClickHouseBatchSyncService(dataSource,
-                threads != null ? Integer.valueOf(threads) : null,
-                batchSize != null ? Integer.valueOf(batchSize) : null,
-                scheduleTime != null ? Long.valueOf(scheduleTime) : null,
-                skipDupException);
-
-        clickHouseMirrorDbBatchSyncService = new ClickHouseMirrorDbBatchSyncService(mirrorDbConfigCache,
-                dataSource,
-                threads != null ? Integer.valueOf(threads) : null,
-                batchSize != null ? Integer.valueOf(batchSize) : null,
-                scheduleTime != null ? Long.valueOf(scheduleTime) : null,
-                clickHouseBatchSyncService.getColumnsTypeCache(),
-                skipDupException);
-
-        clickHouseConfigMonitor = new ClickHouseConfigMonitor();
-        clickHouseConfigMonitor.init(configuration.getKey(), this, envProperties);
-    }
-
-    /**
-     * Sync main entrance
-     *
-     * @param dmls 数据包
-     */
-    @Override
-    public void sync(List<Dml> dmls) {
-        if (dmls == null || dmls.isEmpty()) {
-            return;
-        }
-        try {
-            // If mappingConfigCache(column mapping) is empty, that must be mirroring synchronize
-            if (!mappingConfigCache.isEmpty()) {
-                clickHouseBatchSyncService.sync(mappingConfigCache, dmls, envProperties);
-            }
-            clickHouseMirrorDbBatchSyncService.sync(dmls);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * Destroy
-     */
-    @Override
-    public void destroy() {
-        if (clickHouseConfigMonitor != null) {
-            clickHouseConfigMonitor.destroy();
-        }
-
-        if (clickHouseBatchSyncService != null) {
-            clickHouseBatchSyncService.close();
-        }
-
-        if (dataSource != null) {
-            dataSource.close();
-        }
-    }
-
-
-    /**
-     * ETL方法
-     *
-     * @param task 任务名, 对应配置名
-     * @param params etl筛选条件
-     * @return ETL结果
-     */
-    @Override
-    public EtlResult etl(String task, List<String> params) {
-        EtlResult etlResult = new EtlResult();
-        MappingConfig config = clickHouseMapping.get(task);
-        ClickHouseEtlService clickhouseEtlService = new ClickHouseEtlService(dataSource, config);
-        if (config != null) {
-            return clickhouseEtlService.importData(params);
-        } else {
-            StringBuilder resultMsg = new StringBuilder();
-            boolean resSucc = true;
-            for (MappingConfig configTmp : clickHouseMapping.values()) {
-                // 取所有的destination为task的配置
-                if (configTmp.getDestination().equals(task)) {
-                    EtlResult etlRes = clickhouseEtlService.importData(params);
-                    if (!etlRes.getSucceeded()) {
-                        resSucc = false;
-                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
-                    } else {
-                        resultMsg.append(etlRes.getResultMessage()).append("\n");
-                    }
-                }
-            }
-            if (resultMsg.length() > 0) {
-                etlResult.setSucceeded(resSucc);
-                if (resSucc) {
-                    etlResult.setResultMessage(resultMsg.toString());
-                } else {
-                    etlResult.setErrorMessage(resultMsg.toString());
-                }
-                return etlResult;
-            }
-        }
-        etlResult.setSucceeded(false);
-        etlResult.setErrorMessage("Task not found");
-        return etlResult;
-    }
-
-    /**
-     * 获取总数方法
-     *
-     * @param task 任务名, 对应配置名
-     * @return 总数
-     */
-    @Override
-    public Map<String, Object> count(String task) {
-        MappingConfig config = clickHouseMapping.get(task);
-        MappingConfig.DbMapping dbMapping = config.getDbMapping();
-        String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType());
-        Connection conn = null;
-        Map<String, Object> res = new LinkedHashMap<>();
-        try {
-            conn = dataSource.getConnection();
-            Util.sqlRS(conn, sql, rs -> {
-                try {
-                    if (rs.next()) {
-                        Long rowCount = rs.getLong("cnt");
-                        res.put("count", rowCount);
-                    }
-                } catch (SQLException e) {
-                    logger.error(e.getMessage(), e);
-                }
-            });
-        } catch (SQLException e) {
-            logger.error(e.getMessage(), e);
-        } finally {
-            if (conn != null) {
-                try {
-                    conn.close();
-                } catch (SQLException e) {
-                    logger.error(e.getMessage(), e);
-                }
-            }
-        }
-        res.put("targetTable", SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()));
-
-        return res;
-    }
-
-    /**
-     * 获取对应canal instance name 或 mq topic
-     *
-     * @param task 任务名, 对应配置名
-     * @return destination
-     */
-    @Override
-    public String getDestination(String task) {
-        MappingConfig config = clickHouseMapping.get(task);
-        if (config != null) {
-            return config.getDestination();
-        }
-        return null;
-    }
-
-    private void addSyncConfigToCache(String configName, MappingConfig mappingConfig) {
-        if (!mappingConfig.getDbMapping().getMirrorDb()) {
-            String key;
-            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
-                        + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
-                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
-            } else {
-                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
-                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
-            }
-            Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
-                    k1 -> new ConcurrentHashMap<>());
-            configMap.put(configName, mappingConfig);
-        } else {
-            // mirrorDB
-            String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
-                    + mappingConfig.getDbMapping().getDatabase();
-            mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
-        }
-    }
-
-    public boolean addConfig(String fileName, MappingConfig config) {
-        if (match(config)) {
-            clickHouseMapping.put(fileName, config);
-            addSyncConfigToCache(fileName, config);
-            FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName,
-                    configuration.getKey());
-            return true;
-        }
-        return false;
-    }
-
-    public void updateConfig(String fileName, MappingConfig config) {
-        if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey()
-                .equals(configuration.getKey())) {
-            // 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
-            throw new RuntimeException("not allow to change outAdapterKey");
-        }
-        clickHouseMapping.put(fileName, config);
-        addSyncConfigToCache(fileName, config);
-    }
-
-    public void deleteConfig(String fileName) {
-        clickHouseMapping.remove(fileName);
-        for (Map<String, MappingConfig> configMap : mappingConfigCache.values()) {
-            if (configMap != null) {
-                configMap.remove(fileName);
-            }
-        }
-        FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
-    }
-
-    private boolean match(MappingConfig config) {
-        boolean sameMatch = config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
-                .equalsIgnoreCase(configuration.getKey());
-        boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
-                .startsWith(StringUtils
-                        .join(new String[]{Util.AUTO_GENERATED_PREFIX, config.getDestination(),
-                                config.getGroupId()}, '-'));
-        return sameMatch || prefixMatch;
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/config/ConfigLoader.java

@@ -1,52 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.config;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
-
-/**
- * CLICKHOUSE表映射配置加载器
- *
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-public class ConfigLoader {
-
-    private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
-
-    /**
-     * 加载CLICKHOUSE表映射配置
-     *
-     * @return 配置名/配置文件名--对象
-     */
-    public static Map<String, MappingConfig> load(Properties envProperties) {
-        logger.info("## Start loading clickhouse mapping config ... ");
-
-        Map<String, MappingConfig> result = new LinkedHashMap<>();
-
-        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("clickhouse");
-        configContentMap.forEach((fileName, content) -> {
-            MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
-            if (config == null) {
-                return;
-            }
-            try {
-                config.validate();
-            } catch (Exception e) {
-                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
-            }
-            result.put(fileName, config);
-        });
-
-        logger.info("## ClickHouse mapping config loaded");
-        return result;
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/config/MappingConfig.java

@@ -1,227 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.config;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-
-/**
- * CLICKHOUSE表映射配置
- *
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-public class MappingConfig implements AdapterConfig {
-
-    private String    dataSourceKey;      // 数据源key
-
-    private String    destination;        // canal实例或MQ的topic
-
-    private String    groupId;            // groupId
-
-    private String    outerAdapterKey;    // 对应适配器的key
-
-    private boolean   concurrent = false; // 是否并行同步
-
-    private DbMapping dbMapping;          // db映射配置
-
-    public String getDataSourceKey() {
-        return dataSourceKey;
-    }
-
-    public void setDataSourceKey(String dataSourceKey) {
-        this.dataSourceKey = dataSourceKey;
-    }
-
-    public String getGroupId() {
-        return groupId;
-    }
-
-    public void setGroupId(String groupId) {
-        this.groupId = groupId;
-    }
-
-    public String getOuterAdapterKey() {
-        return outerAdapterKey;
-    }
-
-    public void setOuterAdapterKey(String outerAdapterKey) {
-        this.outerAdapterKey = outerAdapterKey;
-    }
-
-    public boolean getConcurrent() {
-        return concurrent;
-    }
-
-    public void setConcurrent(boolean concurrent) {
-        this.concurrent = concurrent;
-    }
-
-    public DbMapping getDbMapping() {
-        return dbMapping;
-    }
-
-    public void setDbMapping(DbMapping dbMapping) {
-        this.dbMapping = dbMapping;
-    }
-
-    public String getDestination() {
-        return destination;
-    }
-
-    public void setDestination(String destination) {
-        this.destination = destination;
-    }
-
-    public AdapterMapping getMapping() {
-        return dbMapping;
-    }
-
-    public void validate() {
-        if (dbMapping.database == null || dbMapping.database.isEmpty()) {
-            throw new NullPointerException("dbMapping.database");
-        }
-        if (!dbMapping.getMirrorDb() && (dbMapping.table == null || dbMapping.table.isEmpty())) {
-            throw new NullPointerException("dbMapping.table");
-        }
-        if (!dbMapping.getMirrorDb() && (dbMapping.targetTable == null || dbMapping.targetTable.isEmpty())) {
-            throw new NullPointerException("dbMapping.targetTable");
-        }
-    }
-
-    public static class DbMapping implements AdapterMapping {
-
-        private boolean             mirrorDb        = false;                 // 是否镜像库
-        private String              database;                                // 数据库名或schema名
-        private String              table;                                   // 表名
-        private Map<String, String> targetPk        = new LinkedHashMap<>(); // 目标表主键字段
-        private boolean             mapAll          = false;                 // 映射所有字段
-        private String              targetDb;                                // 目标库名
-        private String              targetTable;                             // 目标表名
-        private Map<String, String> targetColumns;                           // 目标表字段映射
-
-        private boolean             caseInsensitive = false;                 // 目标表不区分大小写，默认是否
-
-        private String              etlCondition;                            // etl条件sql
-
-        private int                 readBatch       = 5000;
-        private int                 commitBatch     = 5000;                  // etl等批量提交大小
-
-        private Map<String, String> allMapColumns;
-
-        public boolean getMirrorDb() {
-            return mirrorDb;
-        }
-
-        public void setMirrorDb(boolean mirrorDb) {
-            this.mirrorDb = mirrorDb;
-        }
-
-        public String getDatabase() {
-            return database;
-        }
-
-        public void setDatabase(String database) {
-            this.database = database;
-        }
-
-        public String getTable() {
-            return table;
-        }
-
-        public void setTable(String table) {
-            this.table = table;
-        }
-
-        public Map<String, String> getTargetPk() {
-            return targetPk;
-        }
-
-        public void setTargetPk(Map<String, String> targetPk) {
-            this.targetPk = targetPk;
-        }
-
-        public Boolean getMapAll() {
-            return mapAll;
-        }
-
-        public void setMapAll(Boolean mapAll) {
-            this.mapAll = mapAll;
-        }
-
-        public String getTargetDb() {
-            return targetDb;
-        }
-
-        public void setTargetDb(String targetDb) {
-            this.targetDb = targetDb;
-        }
-
-        public String getTargetTable() {
-            return targetTable;
-        }
-
-        public void setTargetTable(String targetTable) {
-            this.targetTable = targetTable;
-        }
-
-        public Map<String, String> getTargetColumns() {
-            if (targetColumns != null) {
-                targetColumns.forEach((key, value) -> {
-                    if (StringUtils.isEmpty(value)) {
-                        targetColumns.put(key, key);
-                    }
-                });
-            }
-            return targetColumns;
-        }
-
-        public void setTargetColumns(Map<String, String> targetColumns) {
-            this.targetColumns = targetColumns;
-        }
-
-        public boolean isCaseInsensitive() {
-            return caseInsensitive;
-        }
-
-        public void setCaseInsensitive(boolean caseInsensitive) {
-            this.caseInsensitive = caseInsensitive;
-        }
-
-        public String getEtlCondition() {
-            return etlCondition;
-        }
-
-        public void setEtlCondition(String etlCondition) {
-            this.etlCondition = etlCondition;
-        }
-
-        public int getReadBatch() {
-            return readBatch;
-        }
-
-        public void setReadBatch(int readBatch) {
-            this.readBatch = readBatch;
-        }
-
-        public int getCommitBatch() {
-            return commitBatch;
-        }
-
-        public void setCommitBatch(int commitBatch) {
-            this.commitBatch = commitBatch;
-        }
-
-        public Map<String, String> getAllMapColumns() {
-            return allMapColumns;
-        }
-
-        public void setAllMapColumns(Map<String, String> allMapColumns) {
-            this.allMapColumns = allMapColumns;
-        }
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/config/MirrorDbConfig.java

@@ -1,51 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.config;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-
-public class MirrorDbConfig {
-
-    private String                     fileName;
-    private MappingConfig              mappingConfig;
-    private Map<String, MappingConfig> tableConfig = new ConcurrentHashMap<>();
-
-    public static MirrorDbConfig create(String fileName, MappingConfig mappingConfig) {
-        return new MirrorDbConfig(fileName, mappingConfig);
-    }
-
-    public MirrorDbConfig(String fileName, MappingConfig mappingConfig){
-        this.fileName = fileName;
-        this.mappingConfig = mappingConfig;
-    }
-
-    public String getFileName() {
-        return fileName;
-    }
-
-    public void setFileName(String fileName) {
-        this.fileName = fileName;
-    }
-
-    public MappingConfig getMappingConfig() {
-        return mappingConfig;
-    }
-
-    public void setMappingConfig(MappingConfig mappingConfig) {
-        this.mappingConfig = mappingConfig;
-    }
-
-    public Map<String, MappingConfig> getTableConfig() {
-        return tableConfig;
-    }
-
-    public void setTableConfig(Map<String, MappingConfig> tableConfig) {
-        this.tableConfig = tableConfig;
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/monitor/ClickHouseConfigMonitor.java

@@ -1,131 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.monitor;
-
-import java.io.File;
-import java.util.Properties;
-
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
-import org.apache.commons.io.monitor.FileAlterationMonitor;
-import org.apache.commons.io.monitor.FileAlterationObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
-
-/**
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-
-public class ClickHouseConfigMonitor {
-
-    private static final Logger   logger      = LoggerFactory.getLogger(ClickHouseConfigMonitor.class);
-
-    private static final String   adapterName = "clickhouse";
-
-    private String                key;
-
-    private ClickHouseAdapter     clickHouseAdapter;
-
-    private Properties            envProperties;
-
-    private FileAlterationMonitor fileMonitor;
-
-    public void init(String key, ClickHouseAdapter clickHouseAdapter, Properties envProperties) {
-        this.key = key;
-        this.clickHouseAdapter = clickHouseAdapter;
-        this.envProperties = envProperties;
-        File confDir = Util.getConfDirPath(adapterName);
-        try {
-            FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
-            FileListener listener = new FileListener();
-            observer.addListener(listener);
-            fileMonitor = new FileAlterationMonitor(3000, observer);
-            fileMonitor.start();
-
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    public void destroy() {
-        try {
-            fileMonitor.stop();
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    private class FileListener extends FileAlterationListenerAdaptor {
-
-        @Override
-        public void onFileCreate(File file) {
-            super.onFileCreate(file);
-            try {
-                // 加载新增的配置文件
-                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                MappingConfig config = YamlUtils
-                    .ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
-                if (config == null) {
-                    return;
-                }
-                config.validate();
-                boolean result = clickHouseAdapter.addConfig(file.getName(), config);
-                if (result) {
-                    logger.info("Add a new clickhouse mapping config: {} to canal adapter", file.getName());
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        @Override
-        public void onFileChange(File file) {
-            super.onFileChange(file);
-
-            try {
-                if (clickHouseAdapter.getClickHouseMapping().containsKey(file.getName())) {
-                    // 加载配置文件
-                    String configContent = MappingConfigsLoader
-                        .loadConfig(adapterName + File.separator + file.getName());
-                    if (configContent == null) {
-                        onFileDelete(file);
-                        return;
-                    }
-                    MappingConfig config = YamlUtils
-                        .ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
-                    if (config == null) {
-                        return;
-                    }
-                    config.validate();
-                    clickHouseAdapter.updateConfig(file.getName(), config);
-                    logger.info("Change a clickhouse mapping config: {} of canal adapter", file.getName());
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        @Override
-        public void onFileDelete(File file) {
-            super.onFileDelete(file);
-
-            try {
-                if (clickHouseAdapter.getClickHouseMapping().containsKey(file.getName())) {
-                    clickHouseAdapter.deleteConfig(file.getName());
-
-                    logger.info("Delete a clickhouse mapping config: {} of canal adapter", file.getName());
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseBatchSyncService.java

@@ -1,555 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.service;
-
-import java.sql.Connection;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Types;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.fastjson2.JSON;
-import com.alibaba.fastjson2.JSONWriter.Feature;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig.DbMapping;
-import com.alibaba.otter.canal.client.adapter.clickhouse.support.BatchExecutor;
-import com.alibaba.otter.canal.client.adapter.clickhouse.support.SingleDml;
-import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * ClickHouse batch synchronize
- *
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-public class ClickHouseBatchSyncService {
-
-    private static final Logger                         logger  = LoggerFactory.getLogger(ClickHouseBatchSyncService.class);
-
-    private DruidDataSource                             dataSource;
-
-    private Map<String, Map<String, Integer>>           columnsTypeCache;     // Cache of instance.schema.table -> <columnName, jdbcType>
-
-    private Map<MappingConfig, List<SingleDml>>[]       bufferPools;          // Data buffer pool store sync data, List<Dml> dispersed as arrays according to hash value
-
-    private BatchExecutor[]                             batchExecutors;       // Batch Executor
-
-    private BatchExecutor                               alterExecutors;       // Alter Single Executor(update/delete/truncate)
-
-    private ExecutorService[]                           executorThreads;      // Be initialized once
-
-    private ScheduledExecutorService[]                  scheduledExecutors;
-
-    private int                                         threads = 3;          // Default parallel thread count
-    private int                                         batchSize = 1000;
-    private long                                        scheduleTime = 10;
-    private boolean                                     skipDupException;
-
-    public Map<String, Map<String, Integer>> getColumnsTypeCache() {
-        return columnsTypeCache;
-    }
-
-    public ClickHouseBatchSyncService(DruidDataSource dataSource, Integer threads, Integer batchSize, Long scheduleTime, boolean skipDupException){
-        this(dataSource, threads, batchSize, scheduleTime, new ConcurrentHashMap<>(), skipDupException);
-    }
-
-    @SuppressWarnings("unchecked")
-    public ClickHouseBatchSyncService(DruidDataSource dataSource, Integer threads, Integer batchSize, Long scheduleTime, Map<String, Map<String, Integer>> columnsTypeCache,
-                                      boolean skipDupException){
-        this.dataSource = dataSource;
-        this.columnsTypeCache = columnsTypeCache;
-        this.skipDupException = skipDupException;
-        try {
-            if (threads != null) {
-                this.threads = threads;
-            }
-            if (batchSize != null) {
-                this.batchSize = batchSize;
-            }
-            if (scheduleTime != null) {
-                this.scheduleTime = scheduleTime;
-            }
-            this.alterExecutors = new BatchExecutor(dataSource);
-            this.bufferPools = new ConcurrentHashMap[this.threads];
-            this.batchExecutors = new BatchExecutor[this.threads];
-            this.executorThreads = new ExecutorService[this.threads];
-            this.scheduledExecutors = new ScheduledExecutorService[this.threads];
-            for (int i = 0; i < this.threads; i++) {
-                bufferPools[i] = new ConcurrentHashMap<>();
-                batchExecutors[i] = new BatchExecutor(dataSource);
-                executorThreads[i] = Executors.newSingleThreadExecutor();
-                scheduledExecutors[i] = Executors.newSingleThreadScheduledExecutor();
-            }
-            scheduleBatchSync();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * Timing-driven event
-     * start schedule batch sync threadPool
-     */
-    private void scheduleBatchSync() {
-        for (int i = 0; i < scheduledExecutors.length; i++) {
-            int index = i;
-            scheduledExecutors[i].scheduleAtFixedRate(()->{
-                List<Future<Boolean>> futures = new ArrayList<>();
-                for (MappingConfig mappingConfig : bufferPools[index].keySet()) {
-                    List<SingleDml> dmls = bufferPools[index].get(mappingConfig);
-                    if (dmls == null || dmls.isEmpty()) {
-                        return;
-                    }
-                    List<SingleDml> tempDmls;
-                    synchronized (dmls) {
-                        tempDmls = new ArrayList<>(dmls);
-                        dmls.clear();
-                    }
-                    futures.add(executorThreads[index].submit(()->{
-                        try {
-                            insert(batchExecutors[index], mappingConfig, tempDmls);
-                            batchExecutors[index].commit();
-                            return true;
-                        } catch (Exception e){
-                            batchExecutors[index].rollback();
-                            throw new RuntimeException(e);
-                        }
-                    }));
-                }
-            }, 0, scheduleTime, TimeUnit.SECONDS);
-        }
-        logger.info("Schedule batch executors has started successfully!");
-    }
-
-    /**
-     * 批量同步回调
-     *
-     * @param dmls 批量 DML
-     * @param function 回调方法
-     */
-    public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
-        boolean toExecute = false;
-        for (Dml dml : dmls) {
-            if (!toExecute) {
-                toExecute = function.apply(dml);
-            } else {
-                function.apply(dml);
-            }
-        }
-    }
-
-    /**
-     * Distribute dmls into different partition
-     *
-     * @param mappingConfig {@link com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter#mappingConfigCache }
-     * @param dmls received DML
-     */
-    public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls, Properties envProperties) {
-        sync(dmls, dml -> {
-            if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
-                // DDL(Cache need to update when DDL was executed)
-            columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
-            return false;
-        } else {
-            // DML
-            String destination = StringUtils.trimToEmpty(dml.getDestination());
-            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
-            String database = dml.getDatabase();
-            String table = dml.getTable();
-            Map<String, MappingConfig> configMap;
-            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
-            } else {
-                configMap = mappingConfig.get(destination + "_" + database + "-" + table);
-            }
-
-            if (configMap == null) {
-                return false;
-            }
-
-            if (configMap.values().isEmpty()) {
-                return false;
-            }
-
-            for (MappingConfig config : configMap.values()) {
-                distributeDml(config, dml);
-            }
-            return true;
-        }
-    }   );
-    }
-
-    /**
-     * Dml distributor
-     */
-    private void distributeDml(MappingConfig config, Dml dml) {
-        if (config != null) {
-            try {
-                String type = dml.getType();
-                if (type == null) return;
-
-                if (type.equalsIgnoreCase("INSERT")) {
-                    appendDmlBufferPartition(config, dml);
-                } else {
-                    boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
-                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
-
-                    if (type.equalsIgnoreCase("UPDATE")) {
-                        for (SingleDml singleDml : singleDmls) {
-                            update(alterExecutors, config, singleDml);
-                        }
-                    } else if (type.equalsIgnoreCase("DELETE")) {
-                        for (SingleDml singleDml : singleDmls) {
-                            delete(alterExecutors, config, singleDml);
-                        }
-                    } else if (type.equalsIgnoreCase("TRUNCATE")) {
-                        truncate(alterExecutors, config);
-                    }
-                }
-                if (logger.isDebugEnabled()) {
-                    logger.debug("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
-                }
-            } catch (SQLException e) {
-                throw new RuntimeException(e);
-            }
-        }
-    }
-
-    public void appendDmlBufferPartition(MappingConfig config, Dml dml) {
-        boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
-        List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
-
-        singleDmls.forEach(singleDml -> {
-            int hash = mappingHash(config.getDbMapping());
-            if (!config.getConcurrent()) {
-                hash = 0;
-            }
-            List<SingleDml> dmls = bufferPools[hash].computeIfAbsent(config, k -> new ArrayList<>());
-            synchronized (dmls) {
-                dmls.add(singleDml);
-                logger.info("Append one data into pool, id {}", singleDml.getData().get("id"));
-            }
-            // Check the size of the List, achieve when it reaches the maximum value
-            if (dmls.size() >= batchSize) syncToClickHouse(config, hash);
-        });
-    }
-
-    /**
-     * sync when size of list{@link #bufferPools } reaches the maximum value
-     *
-     * @param config key
-     * @param index parallel thread index
-     */
-    private void syncToClickHouse(MappingConfig config, int index) {
-        List<SingleDml> dmls = bufferPools[index].get(config);
-        logger.info("schema:{} table:{} reaches the maximum value, ready to synchronize, size {}", config.getDbMapping().getDatabase(), config.getDbMapping().getTable(), dmls.size());
-        if (dmls ==null || dmls.isEmpty()) {
-            return;
-        }
-        List<SingleDml> tempDmls;
-        synchronized (dmls) {
-            tempDmls = new ArrayList<>(dmls);
-            dmls.clear();
-        }
-        executorThreads[index].submit(() -> {
-            try {
-                insert(batchExecutors[index], config, tempDmls);
-                batchExecutors[index].commit();
-                return true;
-            } catch (Exception e) {
-                batchExecutors[index].rollback();
-                throw new RuntimeException(e);
-            }
-        });
-    }
-
-    /**
-     * Insert
-     *
-     * @param batchExecutor batch translational executor
-     * @param config corresponding configuration object
-     * @param dmls DMLs
-     */
-    private void insert(BatchExecutor batchExecutor, MappingConfig config, List<SingleDml> dmls) throws SQLException {
-        if (dmls == null || dmls.isEmpty()) {
-            return;
-        }
-        List<SingleDml> clearDmls = dmls.stream().filter(e -> e.getData() != null && !e.getData().isEmpty()).collect(Collectors.toList());
-        if (clearDmls == null || clearDmls.isEmpty()) {
-            return;
-        }
-
-        DbMapping dbMapping = config.getDbMapping();
-        String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
-        Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, clearDmls.get(0).getData());
-
-        StringBuilder insertSql = new StringBuilder();
-        insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())).append(" (");
-
-        columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick)
-            .append(targetColumnName)
-            .append(backtick)
-            .append(","));
-        int len = insertSql.length();
-        insertSql.delete(len - 1, len).append(") VALUES (");
-        int mapLen = columnsMap.size();
-        for (int i = 0; i < mapLen; i++) {
-            insertSql.append("?,");
-        }
-        len = insertSql.length();
-        insertSql.delete(len - 1, len).append(")");
-
-        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
-
-        List<List<Map<String, ?>>> values = new ArrayList<>();
-        boolean flag = false;
-        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
-            String targetColumnName = entry.getKey();
-            String srcColumnName = entry.getValue();
-            if (srcColumnName == null) {
-                srcColumnName = Util.cleanColumn(targetColumnName);
-            }
-
-            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
-            if (type == null) {
-                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
-            }
-            for (int i = 0; i < clearDmls.size(); i++) {
-                Map<String, Object> dmlData = clearDmls.get(i).getData();
-                List<Map<String, ?>> item;
-                if (flag == false) {
-                    item = new ArrayList<>();
-                    values.add(item);
-                } else {
-                    item = values.get(i);
-                }
-                Object value = dmlData.get(srcColumnName);
-                BatchExecutor.setValue(item, type, value);
-            }
-            flag = true;
-        }
-
-        try {
-            batchExecutor.batchExecute(insertSql.toString(), values);
-        } catch (SQLException e) {
-            if (skipDupException
-                && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001:"))) {
-                // ignore
-                // TODO 增加更多关系数据库的主键冲突的错误码
-            } else {
-                throw e;
-            }
-        }
-        if (logger.isTraceEnabled()) {
-            logger.trace("Insert into target table, sql: {}", insertSql);
-        }
-
-    }
-
-    /**
-     * 更新操作
-     *
-     * @param config 配置项
-     * @param dml DML数据
-     */
-    private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
-        Map<String, Object> data = dml.getData();
-        if (data == null || data.isEmpty()) {
-            return;
-        }
-
-        Map<String, Object> old = dml.getOld();
-        if (old == null || old.isEmpty()) {
-            return;
-        }
-
-        DbMapping dbMapping = config.getDbMapping();
-        String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
-        Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
-
-        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
-
-        StringBuilder updateSql = new StringBuilder();
-        updateSql.append("ALTER TABLE ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())).append(" UPDATE ");
-        List<Map<String, ?>> values = new ArrayList<>();
-        boolean hasMatched = false;
-        for (String srcColumnName : old.keySet()) {
-            List<String> targetColumnNames = new ArrayList<>();
-            columnsMap.forEach((targetColumn, srcColumn) -> {
-                if (srcColumnName.equalsIgnoreCase(srcColumn)) {
-                    targetColumnNames.add(targetColumn);
-                }
-            });
-            if (!targetColumnNames.isEmpty()) {
-                hasMatched = true;
-                for (String targetColumnName : targetColumnNames) {
-                    updateSql.append(backtick).append(targetColumnName).append(backtick).append("=?, ");
-                    Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
-                    if (type == null) {
-                        throw new RuntimeException("Target column: " + targetColumnName + " not matched");
-                    }
-                    BatchExecutor.setValue(values, type, data.get(srcColumnName));
-                }
-            }
-        }
-        if (!hasMatched) {
-            logger.warn("Did not matched any columns to update ");
-            return;
-        }
-        int len = updateSql.length();
-        updateSql.delete(len - 2, len).append(" WHERE ");
-
-        // 拼接主键
-        appendCondition(dbMapping, updateSql, ctype, values, data, old);
-        batchExecutor.execute(updateSql.toString(), values);
-        if (logger.isTraceEnabled()) {
-            logger.trace("Alter update target table, sql: {}", updateSql);
-        }
-    }
-
-    /**
-     * 删除操作
-     *
-     * @param config
-     * @param dml
-     */
-    private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
-        Map<String, Object> data = dml.getData();
-        if (data == null || data.isEmpty()) {
-            return;
-        }
-
-        DbMapping dbMapping = config.getDbMapping();
-        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
-
-        StringBuilder sql = new StringBuilder();
-        sql.append("ALTER TABLE ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())).append(" DELETE WHERE ");
-
-        List<Map<String, ?>> values = new ArrayList<>();
-        // 拼接主键
-        appendCondition(dbMapping, sql, ctype, values, data);
-        batchExecutor.execute(sql.toString(), values);
-        if (logger.isTraceEnabled()) {
-            logger.trace("Alter delete from target table, sql: {}", sql);
-        }
-    }
-
-    /**
-     * truncate操作
-     *
-     * @param config
-     */
-    private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException {
-        DbMapping dbMapping = config.getDbMapping();
-        StringBuilder sql = new StringBuilder();
-        sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()));
-        batchExecutor.execute(sql.toString(), new ArrayList<>());
-        if (logger.isTraceEnabled()) {
-            logger.trace("Truncate target table, sql: {}", sql);
-        }
-    }
-
-    /**
-     * 获取目标字段类型
-     *
-     * @param conn sql connection
-     * @param config 映射配置
-     * @return 字段sqlType
-     */
-    private Map<String, Integer> getTargetColumnType(Connection conn, MappingConfig config) {
-        DbMapping dbMapping = config.getDbMapping();
-        String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable();
-        Map<String, Integer> columnType = columnsTypeCache.get(cacheKey);
-        if (columnType == null) {
-            synchronized (ClickHouseBatchSyncService.class) {
-                columnType = columnsTypeCache.get(cacheKey);
-                if (columnType == null) {
-                    columnType = new LinkedHashMap<>();
-                    final Map<String, Integer> columnTypeTmp = columnType;
-                    String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " WHERE 1=2";
-                    Util.sqlRS(conn, sql, rs -> {
-                        try {
-                            ResultSetMetaData rsd = rs.getMetaData();
-                            int columnCount = rsd.getColumnCount();
-                            for (int i = 1; i <= columnCount; i++) {
-                                int colType = rsd.getColumnType(i);
-                                // 修复year类型作为date处理时的data truncated问题
-                                if ("YEAR".equals(rsd.getColumnTypeName(i))) {
-                                    colType = Types.VARCHAR;
-                                }
-                                columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), colType);
-                            }
-                            columnsTypeCache.put(cacheKey, columnTypeTmp);
-                        } catch (SQLException e) {
-                            logger.error(e.getMessage(), e);
-                        }
-                    });
-                }
-            }
-        }
-        return columnType;
-    }
-
-    /**
-     * 拼接主键 where条件
-     */
-    private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map<String, Integer> ctype,
-                                 List<Map<String, ?>> values, Map<String, Object> d) {
-        appendCondition(dbMapping, sql, ctype, values, d, null);
-    }
-
-    private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map<String, Integer> ctype,
-                                 List<Map<String, ?>> values, Map<String, Object> d, Map<String, Object> o) {
-        String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
-
-        // 拼接主键
-        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
-            String targetColumnName = entry.getKey();
-            String srcColumnName = entry.getValue();
-            if (srcColumnName == null) {
-                srcColumnName = Util.cleanColumn(targetColumnName);
-            }
-            sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND ");
-            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
-            if (type == null) {
-                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
-            }
-            // 如果有修改主键的情况
-            if (o != null && o.containsKey(srcColumnName)) {
-                BatchExecutor.setValue(values, type, o.get(srcColumnName));
-            } else {
-                BatchExecutor.setValue(values, type, d.get(srcColumnName));
-            }
-        }
-        int len = sql.length();
-        sql.delete(len - 4, len);
-    }
-
-    /**
-     * make sure the same table in one index
-     *
-     * @param dbMapping
-     * @return
-     */
-    private int mappingHash(MappingConfig.DbMapping dbMapping) {
-        int hash = dbMapping.getDatabase().toLowerCase().hashCode() + dbMapping.getTable().toLowerCase().hashCode();
-        hash = Math.abs(hash) % threads;
-        return Math.abs(hash);
-    }
-
-    public void close() {
-        for (int i = 0; i < threads; i++) {
-            executorThreads[i].shutdown();
-        }
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseEtlService.java

@@ -1,197 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.service;
-
-import java.sql.*;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.sql.DataSource;
-
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig.DbMapping;
-import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.*;
-
-/**
- * ClickHouse ETL 操作业务类
- *
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-public class ClickHouseEtlService extends AbstractEtlService {
-
-    private DataSource    targetDS;
-    private MappingConfig config;
-
-    public ClickHouseEtlService(DataSource targetDS, MappingConfig config){
-        super("CLICKHOUSE", config);
-        this.targetDS = targetDS;
-        this.config = config;
-    }
-
-    /**
-     * 导入数据
-     */
-    public EtlResult importData(List<String> params) {
-        DbMapping dbMapping = config.getDbMapping();
-        DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType());
-        return importData(sql, params);
-    }
-
-    /**
-     * 执行导入
-     */
-    protected boolean executeSqlImport(DataSource srcDS, String sql, List<Object> values,
-                                       AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
-        try {
-            DbMapping dbMapping = (DbMapping) mapping;
-            Map<String, String> columnsMap = new LinkedHashMap<>();
-            Map<String, Integer> columnType = new LinkedHashMap<>();
-            DruidDataSource dataSource = (DruidDataSource) srcDS;
-            String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
-
-            Util.sqlRS(targetDS,
-                "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " LIMIT 1 ",
-                rs -> {
-                    try {
-
-                        ResultSetMetaData rsd = rs.getMetaData();
-                        int columnCount = rsd.getColumnCount();
-                        List<String> columns = new ArrayList<>();
-                        for (int i = 1; i <= columnCount; i++) {
-                            columnType.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
-                            columns.add(rsd.getColumnName(i));
-                        }
-
-                        columnsMap.putAll(SyncUtil.getColumnsMap(dbMapping, columns));
-                        return true;
-                    } catch (Exception e) {
-                        logger.error(e.getMessage(), e);
-                        return false;
-                    }
-                });
-
-            Util.sqlRS(srcDS, sql, values, rs -> {
-                int idx = 1;
-
-                try {
-                    boolean completed = false;
-
-                    StringBuilder insertSql = new StringBuilder();
-                    insertSql.append("INSERT INTO ")
-                        .append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()))
-                        .append(" (");
-                    columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick)
-                        .append(targetColumnName)
-                        .append(backtick)
-                        .append(","));
-
-                    int len = insertSql.length();
-                    insertSql.delete(len - 1, len).append(") VALUES (");
-                    int mapLen = columnsMap.size();
-                    for (int i = 0; i < mapLen; i++) {
-                        insertSql.append("?,");
-                    }
-                    len = insertSql.length();
-                    insertSql.delete(len - 1, len).append(")");
-                    logger.info("executeSqlImport sql:{}", insertSql.toString());
-                    try (Connection connTarget = targetDS.getConnection();
-                            PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
-                        connTarget.setAutoCommit(false);
-
-                        while (rs.next()) {
-                            completed = false;
-
-                            pstmt.clearParameters();
-
-                            // 删除数据
-                            Map<String, Object> pkVal = new LinkedHashMap<>();
-                            StringBuilder deleteSql = new StringBuilder(
-                                "ALTER TABLE " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())
-                                                                        + " DELETE WHERE ");
-                            appendCondition(dbMapping, deleteSql, pkVal, rs, backtick);
-                            try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
-                                int k = 1;
-                                for (Object val : pkVal.values()) {
-                                    pstmt2.setObject(k++, val);
-                                }
-                                pstmt2.execute();
-                            }
-
-                            int i = 1;
-                            for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
-                                String targetClolumnName = entry.getKey();
-                                String srcColumnName = entry.getValue();
-                                if (srcColumnName == null) {
-                                    srcColumnName = targetClolumnName;
-                                }
-
-                                Integer type = columnType.get(targetClolumnName.toLowerCase());
-                                Object value = rs.getObject(srcColumnName);
-                                if (value != null) {
-                                    SyncUtil.setPStmt(type, pstmt, value, i);
-                                } else {
-                                    pstmt.setNull(i, type);
-                                }
-
-                                i++;
-                            }
-
-                            pstmt.execute();
-                            if (logger.isTraceEnabled()) {
-                                logger.trace("Insert into target table, sql: {}", insertSql);
-                            }
-
-                            if (idx % dbMapping.getCommitBatch() == 0) {
-                                connTarget.commit();
-                                completed = true;
-                            }
-                            idx++;
-                            impCount.incrementAndGet();
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("successful import count:" + impCount.get());
-                            }
-                        }
-                        if (!completed) {
-                            connTarget.commit();
-                        }
-                    }
-
-                } catch (Exception e) {
-                    logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
-                    errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage());
-                }
-                return idx;
-            });
-            return true;
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-            return false;
-        }
-    }
-
-    /**
-     * 拼接目标表主键where条件
-     */
-    private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
-                                        ResultSet rs, String backtick) throws SQLException {
-        // 拼接主键
-        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
-            String targetColumnName = entry.getKey();
-            String srcColumnName = entry.getValue();
-            if (srcColumnName == null) {
-                srcColumnName = targetColumnName;
-            }
-            sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND ");
-            values.put(targetColumnName, rs.getObject(srcColumnName));
-        }
-        int len = sql.length();
-        sql.delete(len - 4, len);
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseMirrorDbBatchSyncService.java

@@ -1,180 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.service;
-
-import java.sql.Connection;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.fastjson2.JSON;
-import com.alibaba.fastjson2.JSONWriter.Feature;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MirrorDbConfig;
-import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-/**
- * CLICKHOUSE镜像库同步操作业务
- *
- * @author: Xander
- * @date: Created in 2023/11/10 22:23
- * @email: zhrunxin33@gmail.com
- * @version 1.1.8
- */
-public class ClickHouseMirrorDbBatchSyncService {
-
-    private static final Logger         logger = LoggerFactory.getLogger(ClickHouseMirrorDbBatchSyncService.class);
-
-    // 镜像库配置
-    private Map<String, MirrorDbConfig> mirrorDbConfigCache;
-    private DruidDataSource             dataSource;
-    // clickhouseSyncService代理
-    private ClickHouseBatchSyncService  clickHouseBatchSyncService;
-
-    public ClickHouseMirrorDbBatchSyncService(Map<String, MirrorDbConfig> mirrorDbConfigCache,
-                                              DruidDataSource dataSource, Integer threads, Integer batchSize,
-                                              Long scheduleTime, Map<String, Map<String, Integer>> columnsTypeCache,
-                                              boolean skipDupException){
-        this.mirrorDbConfigCache = mirrorDbConfigCache;
-        this.dataSource = dataSource;
-        this.clickHouseBatchSyncService = new ClickHouseBatchSyncService(dataSource,
-            threads,
-            batchSize,
-            scheduleTime,
-            columnsTypeCache,
-            skipDupException);
-    }
-
-    /**
-     * 批量同步方法
-     *
-     * @param dmls 批量 DML，包含DDL
-     */
-    public void sync(List<Dml> dmls) {
-        List<Dml> dmlList = new ArrayList<>();
-        for (Dml dml : dmls) {
-            String destination = StringUtils.trimToEmpty(dml.getDestination());
-            String database = dml.getDatabase();
-            MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(destination + "." + database);
-            if (mirrorDbConfig == null) {
-                continue;
-            }
-            if (mirrorDbConfig.getMappingConfig() == null) {
-                continue;
-            }
-            if (dml.getGroupId() != null && StringUtils.isNotEmpty(mirrorDbConfig.getMappingConfig().getGroupId())) {
-                if (!mirrorDbConfig.getMappingConfig().getGroupId().equals(dml.getGroupId())) {
-                    continue; // 如果groupId不匹配则过滤
-                }
-            }
-
-            if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
-                // 确保执行DDL前DML已执行完
-                syncDml(dmlList);
-                dmlList.clear();
-
-                // DDL
-                if (logger.isDebugEnabled()) {
-                    logger.debug("DDL: {}", JSON.toJSONString(dml, Feature.WriteNulls));
-                }
-                executeDdl(mirrorDbConfig, dml);
-                clickHouseBatchSyncService.getColumnsTypeCache()
-                    .remove(destination + "." + database + "." + dml.getTable());
-                mirrorDbConfig.getTableConfig().remove(dml.getTable()); // 删除对应库表配置
-            } else {
-                // DML
-                initMappingConfig(dml.getTable(), mirrorDbConfig.getMappingConfig(), mirrorDbConfig, dml);
-                dmlList.add(dml);
-            }
-        }
-        syncDml(dmlList);
-    }
-
-    /**
-     * 批量同步Dml
-     *
-     * @param dmlList Dml列表，不包含DDL
-     */
-    private void syncDml(List<Dml> dmlList) {
-        if (dmlList == null || dmlList.isEmpty()) {
-            return;
-        }
-        clickHouseBatchSyncService.sync(dmlList, dml -> {
-            MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase());
-            if (mirrorDbConfig == null) {
-                return false;
-            }
-            String table = dml.getTable();
-            MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
-
-            if (config == null) {
-                return false;
-            }
-            clickHouseBatchSyncService.appendDmlBufferPartition(config, dml);
-            return true;
-        });
-    }
-
-    /**
-     * 初始化表配置
-     *
-     * @param key 配置key: destination.database.table
-     * @param baseConfigMap db sync config
-     * @param dml DML
-     */
-    private void initMappingConfig(String key, MappingConfig baseConfigMap, MirrorDbConfig mirrorDbConfig, Dml dml) {
-        MappingConfig mappingConfig = mirrorDbConfig.getTableConfig().get(key);
-        if (mappingConfig == null) {
-            // 构造表配置
-            mappingConfig = new MappingConfig();
-            mappingConfig.setDataSourceKey(baseConfigMap.getDataSourceKey());
-            mappingConfig.setDestination(baseConfigMap.getDestination());
-            mappingConfig.setGroupId(baseConfigMap.getGroupId());
-            mappingConfig.setOuterAdapterKey(baseConfigMap.getOuterAdapterKey());
-            mappingConfig.setConcurrent(baseConfigMap.getConcurrent());
-            MappingConfig.DbMapping dbMapping = new MappingConfig.DbMapping();
-            mappingConfig.setDbMapping(dbMapping);
-            dbMapping.setDatabase(dml.getDatabase());
-            dbMapping.setTable(dml.getTable());
-            dbMapping.setTargetDb(dml.getDatabase());
-            dbMapping.setTargetTable(dml.getTable());
-            dbMapping.setMapAll(true);
-            List<String> pkNames = dml.getPkNames();
-            Map<String, String> pkMapping = new LinkedHashMap<>();
-            pkNames.forEach(pkName -> pkMapping.put(pkName, pkName));
-            dbMapping.setTargetPk(pkMapping);
-
-            mirrorDbConfig.getTableConfig().put(key, mappingConfig);
-        }
-    }
-
-    /**
-     * DDL 操作
-     *
-     * @param ddl DDL
-     */
-    private void executeDdl(MirrorDbConfig mirrorDbConfig, Dml ddl) {
-        try (Connection conn = dataSource.getConnection(); Statement statement = conn.createStatement()) {
-            // 替换反引号
-            String sql = ddl.getSql();
-            String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
-            if (!"`".equals(backtick)) {
-                sql = sql.replaceAll("`", backtick);
-            }
-            statement.execute(sql);
-            // 移除对应配置
-            mirrorDbConfig.getTableConfig().remove(ddl.getTable());
-            if (logger.isTraceEnabled()) {
-                logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
-            }
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/support/BatchExecutor.java

@@ -1,113 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.support;
-
-import java.io.Closeable;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import javax.sql.DataSource;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * sql批量执行器
- *
- * @author rewerma 2018-11-7 下午06:45:49
- * @version 1.0.0
- */
-public class BatchExecutor implements Closeable {
-
-    private static final Logger logger = LoggerFactory.getLogger(BatchExecutor.class);
-
-    private DataSource          dataSource;
-    private Connection          conn;
-    private AtomicInteger       idx    = new AtomicInteger(0);
-
-    public BatchExecutor(DataSource dataSource){
-        this.dataSource = dataSource;
-    }
-
-    public Connection getConn() {
-        if (conn == null) {
-            try {
-                conn = dataSource.getConnection();
-                this.conn.setAutoCommit(false);
-            } catch (SQLException e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-        return conn;
-    }
-
-    public static void setValue(List<Map<String, ?>> values, int type, Object value) {
-        Map<String, Object> valueItem = new HashMap<>();
-        valueItem.put("type", type);
-        valueItem.put("value", value);
-        values.add(valueItem);
-    }
-
-    public void execute(String sql, List<Map<String, ?>> values) throws SQLException {
-        PreparedStatement pstmt = getConn().prepareStatement(sql);
-        int len = values.size();
-        for (int i = 0; i < len; i++) {
-            int type = (Integer) values.get(i).get("type");
-            Object value = values.get(i).get("value");
-            SyncUtil.setPStmt(type, pstmt, value, i + 1);
-        }
-
-        pstmt.execute();
-        idx.incrementAndGet();
-        pstmt.close();
-    }
-
-    public void batchExecute(String sql, List<List<Map<String, ?>>> batchValues) throws SQLException {
-        PreparedStatement pstmt = getConn().prepareStatement(sql);
-        for (int i = 0; i < batchValues.size(); i++) {
-            List<Map<String, ?>> values = batchValues.get(i);
-            for (int j = 0; j < values.size(); j++) {
-                int type = (Integer) values.get(j).get("type");
-                Object value = values.get(j).get("value");
-                SyncUtil.setPStmt(type, pstmt, value, j + 1);
-            }
-            pstmt.addBatch();
-        }
-
-        pstmt.executeBatch();
-        idx.addAndGet(batchValues.size());
-        pstmt.close();
-    }
-
-    public void commit() throws SQLException {
-        getConn().commit();
-        if (logger.isTraceEnabled()) {
-            logger.trace("Batch executor commit " + idx.get() + " rows");
-        }
-        idx.set(0);
-    }
-
-    public void rollback() throws SQLException {
-        getConn().rollback();
-        if (logger.isTraceEnabled()) {
-            logger.trace("Batch executor rollback " + idx.get() + " rows");
-        }
-        idx.set(0);
-    }
-
-    @Override
-    public void close() {
-        if (conn != null) {
-            try {
-                conn.close();
-            } catch (SQLException e) {
-                logger.error(e.getMessage(), e);
-            } finally {
-                conn = null;
-            }
-        }
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/support/SingleDml.java

@@ -1,112 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.support;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.util.LinkedCaseInsensitiveMap;
-
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-public class SingleDml {
-
-    private String              destination;
-    private String              database;
-    private String              table;
-    private String              type;
-    private Map<String, Object> data;
-    private Map<String, Object> old;
-
-    public String getDestination() {
-        return destination;
-    }
-
-    public void setDestination(String destination) {
-        this.destination = destination;
-    }
-
-    public String getDatabase() {
-        return database;
-    }
-
-    public void setDatabase(String database) {
-        this.database = database;
-    }
-
-    public String getTable() {
-        return table;
-    }
-
-    public void setTable(String table) {
-        this.table = table;
-    }
-
-    public String getType() {
-        return type;
-    }
-
-    public void setType(String type) {
-        this.type = type;
-    }
-
-    public Map<String, Object> getData() {
-        return data;
-    }
-
-    public void setData(Map<String, Object> data) {
-        this.data = data;
-    }
-
-    public Map<String, Object> getOld() {
-        return old;
-    }
-
-    public void setOld(Map<String, Object> old) {
-        this.old = old;
-    }
-
-    public static List<SingleDml> dml2SingleDmls(Dml dml, boolean caseInsensitive) {
-        List<SingleDml> singleDmls = new ArrayList<>();
-        if (dml.getData() != null) {
-            int size = dml.getData().size();
-            for (int i = 0; i < size; i++) {
-                SingleDml singleDml = new SingleDml();
-                singleDml.setDestination(dml.getDestination());
-                singleDml.setDatabase(dml.getDatabase());
-                singleDml.setTable(dml.getTable());
-                singleDml.setType(dml.getType());
-                Map<String, Object> data = dml.getData().get(i);
-                if (caseInsensitive) {
-                    data = toCaseInsensitiveMap(data);
-                }
-                singleDml.setData(data);
-                if (dml.getOld() != null) {
-                    Map<String, Object> oldData = dml.getOld().get(i);
-                    if (caseInsensitive) {
-                        oldData = toCaseInsensitiveMap(oldData);
-                    }
-                    singleDml.setOld(oldData);
-                }
-                singleDmls.add(singleDml);
-            }
-        } else if ("TRUNCATE".equalsIgnoreCase(dml.getType())) {
-            SingleDml singleDml = new SingleDml();
-            singleDml.setDestination(dml.getDestination());
-            singleDml.setDatabase(dml.getDatabase());
-            singleDml.setTable(dml.getTable());
-            singleDml.setType(dml.getType());
-            singleDmls.add(singleDml);
-        }
-        return singleDmls;
-    }
-
-    public static List<SingleDml> dml2SingleDmls(Dml dml) {
-        return dml2SingleDmls(dml, false);
-    }
-
-    private static <V> LinkedCaseInsensitiveMap<V> toCaseInsensitiveMap(Map<String, V> data) {
-        LinkedCaseInsensitiveMap map = new LinkedCaseInsensitiveMap();
-        map.putAll(data);
-        return map;
-    }
-}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/support/SyncUtil.java

@@ -1,293 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.support;
-
-import java.io.Reader;
-import java.io.StringReader;
-import java.math.BigDecimal;
-import java.nio.charset.StandardCharsets;
-import java.sql.*;
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.druid.DbType;
-import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-public class SyncUtil {
-
-    private static final Logger logger = LoggerFactory.getLogger(SyncUtil.class);
-
-    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Map<String, Object> data) {
-        return getColumnsMap(dbMapping, data.keySet());
-    }
-
-    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Collection<String> columns) {
-        Map<String, String> columnsMap;
-        if (dbMapping.getMapAll()) {
-            if (dbMapping.getAllMapColumns() != null) {
-                return dbMapping.getAllMapColumns();
-            }
-            columnsMap = new LinkedHashMap<>();
-            for (String srcColumn : columns) {
-                boolean flag = true;
-                if (dbMapping.getTargetColumns() != null) {
-                    for (Map.Entry<String, String> entry : dbMapping.getTargetColumns().entrySet()) {
-                        if (srcColumn.equals(entry.getValue())) {
-                            columnsMap.put(entry.getKey(), srcColumn);
-                            flag = false;
-                            break;
-                        }
-                    }
-                }
-                if (flag) {
-                    columnsMap.put(srcColumn, srcColumn);
-                }
-            }
-            dbMapping.setAllMapColumns(columnsMap);
-        } else {
-            columnsMap = dbMapping.getTargetColumns();
-        }
-        return columnsMap;
-    }
-
-    /**
-     * 设置 preparedStatement
-     *
-     * @param type sqlType
-     * @param pstmt 需要设置的preparedStatement
-     * @param value 值
-     * @param i 索引号
-     */
-    public static void setPStmt(int type, PreparedStatement pstmt, Object value, int i) throws SQLException {
-        switch (type) {
-            case Types.BIT:
-            case Types.BOOLEAN:
-                if (value instanceof Boolean) {
-                    pstmt.setBoolean(i, (Boolean) value);
-                } else if (value instanceof String) {
-                    boolean v = !value.equals("0");
-                    pstmt.setBoolean(i, v);
-                } else if (value instanceof Number) {
-                    boolean v = ((Number) value).intValue() != 0;
-                    pstmt.setBoolean(i, v);
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.CHAR:
-            case Types.NCHAR:
-            case Types.VARCHAR:
-            case Types.LONGVARCHAR:
-                if (value instanceof String) {
-                    pstmt.setString(i, (String) value);
-                } else if (value == null) {
-                    pstmt.setNull(i, type);
-                } else {
-                    pstmt.setString(i, value.toString());
-                }
-                break;
-            case Types.TINYINT:
-                // 向上提升一级，处理unsigned情况
-                if (value instanceof Number) {
-                    pstmt.setShort(i, ((Number) value).shortValue());
-                } else if (value instanceof String) {
-                    pstmt.setShort(i, Short.parseShort((String) value));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.SMALLINT:
-                if (value instanceof Number) {
-                    pstmt.setInt(i, ((Number) value).intValue());
-                } else if (value instanceof String) {
-                    pstmt.setInt(i, Integer.parseInt((String) value));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.INTEGER:
-                if (value instanceof Number) {
-                    pstmt.setLong(i, ((Number) value).longValue());
-                } else if (value instanceof String) {
-                    pstmt.setLong(i, Long.parseLong((String) value));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.BIGINT:
-                if (value instanceof Number) {
-                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
-                } else if (value instanceof String) {
-                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.DECIMAL:
-            case Types.NUMERIC:
-                if (value instanceof BigDecimal) {
-                    pstmt.setBigDecimal(i, (BigDecimal) value);
-                } else if (value instanceof Byte) {
-                    pstmt.setInt(i, ((Byte) value).intValue());
-                } else if (value instanceof Short) {
-                    pstmt.setInt(i, ((Short) value).intValue());
-                } else if (value instanceof Integer) {
-                    pstmt.setInt(i, (Integer) value);
-                } else if (value instanceof Long) {
-                    pstmt.setLong(i, (Long) value);
-                } else if (value instanceof Float) {
-                    pstmt.setBigDecimal(i, new BigDecimal((float) value));
-                } else if (value instanceof Double) {
-                    pstmt.setBigDecimal(i, new BigDecimal((double) value));
-                } else if (value != null) {
-                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.REAL:
-                if (value instanceof Number) {
-                    pstmt.setFloat(i, ((Number) value).floatValue());
-                } else if (value instanceof String) {
-                    pstmt.setFloat(i, Float.parseFloat((String) value));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.FLOAT:
-            case Types.DOUBLE:
-                if (value instanceof Number) {
-                    pstmt.setDouble(i, ((Number) value).doubleValue());
-                } else if (value instanceof String) {
-                    pstmt.setDouble(i, Double.parseDouble((String) value));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.BINARY:
-            case Types.VARBINARY:
-            case Types.LONGVARBINARY:
-            case Types.BLOB:
-                if (value instanceof Blob) {
-                    pstmt.setBlob(i, (Blob) value);
-                } else if (value instanceof byte[]) {
-                    pstmt.setBytes(i, (byte[]) value);
-                } else if (value instanceof String) {
-                    pstmt.setBytes(i, ((String) value).getBytes(StandardCharsets.ISO_8859_1));
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.CLOB:
-                if (value instanceof Clob) {
-                    pstmt.setClob(i, (Clob) value);
-                } else if (value instanceof byte[]) {
-                    pstmt.setBytes(i, (byte[]) value);
-                } else if (value instanceof String) {
-                    Reader clobReader = new StringReader((String) value);
-                    pstmt.setCharacterStream(i, clobReader);
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.DATE:
-                if (value instanceof Date) {
-                    pstmt.setDate(i, (Date) value);
-                } else if (value instanceof java.util.Date) {
-                    pstmt.setDate(i, new Date(((java.util.Date) value).getTime()));
-                } else if (value instanceof String) {
-                    String v = (String) value;
-                    if (!v.startsWith("0000-00-00")) {
-                        java.util.Date date = Util.parseDate(v);
-                        if (date != null) {
-                            pstmt.setDate(i, new Date(date.getTime()));
-                        } else {
-                            pstmt.setNull(i, type);
-                        }
-                    } else {
-                        pstmt.setObject(i, value);
-                    }
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.TIME:
-                if (value instanceof Time) {
-                    pstmt.setTime(i, (Time) value);
-                } else if (value instanceof java.util.Date) {
-                    pstmt.setTime(i, new Time(((java.util.Date) value).getTime()));
-                } else if (value instanceof String) {
-                    String v = (String) value;
-                    java.util.Date date = Util.parseDate(v);
-                    if (date != null) {
-                        pstmt.setTime(i, new Time(date.getTime()));
-                    } else {
-                        pstmt.setNull(i, type);
-                    }
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            case Types.TIMESTAMP:
-                if (value instanceof Timestamp) {
-                    pstmt.setTimestamp(i, (Timestamp) value);
-                } else if (value instanceof java.util.Date) {
-                    pstmt.setTimestamp(i, new Timestamp(((java.util.Date) value).getTime()));
-                } else if (value instanceof String) {
-                    String v = (String) value;
-                    if (!v.startsWith("0000-00-00")) {
-                        java.util.Date date = Util.parseDate(v);
-                        if (date != null) {
-                            pstmt.setTimestamp(i, new Timestamp(date.getTime()));
-                        } else {
-                            pstmt.setNull(i, type);
-                        }
-                    } else {
-                        pstmt.setObject(i, value);
-                    }
-                } else {
-                    pstmt.setNull(i, type);
-                }
-                break;
-            default:
-                pstmt.setObject(i, value, type);
-        }
-    }
-
-    public static String getDbTableName(MappingConfig.DbMapping dbMapping, String dbType) {
-        String result = "";
-        String backtick = getBacktickByDbType(dbType);
-        if (StringUtils.isNotEmpty(dbMapping.getTargetDb())) {
-            result += (backtick + dbMapping.getTargetDb() + backtick + ".");
-        }
-        result += (backtick + dbMapping.getTargetTable() + backtick);
-        return result;
-    }
-
-    /**
-     * 根据DbType返回反引号或空字符串
-     *
-     * @param dbTypeName DbType名称
-     * @return 反引号或空字符串
-     */
-    public static String getBacktickByDbType(String dbTypeName) {
-        DbType dbType = DbType.of(dbTypeName);
-        if (dbType == null) {
-            dbType = DbType.other;
-        }
-
-        // 只有当dbType为MySQL/MariaDB或OceanBase时返回反引号
-        switch (dbType) {
-            case mysql:
-            case mariadb:
-            case oceanbase:
-                return "`";
-            default:
-                return "";
-        }
-    }
-}

client-adapter/clickhouse/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -1 +0,0 @@
-clickhouse=com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter

client-adapter/clickhouse/src/main/resources/clickhouse/mytest_user.yml

@@ -1,31 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-outerAdapterKey: clickhouse1
-concurrent: true
-dbMapping:
-  database: mytest
-  table: user
-  targetTable: mytest.user
-  targetPk:
-    id: id
-#  mapAll: true
-  targetColumns:
-    id:
-    name:
-    role_id:
-    c_time:
-    test1:
-  etlCondition: "where c_time>={}"
-  commitBatch: 3000 # 批量提交的大小
-
-
-## Mirror schema synchronize config
-#dataSourceKey: defaultDS
-#destination: example
-#groupId: g1
-#outerAdapterKey: mysql1
-#concurrent: true
-#dbMapping:
-#  mirrorDb: true
-#  database: mytest

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseBatchSyncServiceTest.java

@@ -1,109 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse;
-
-import java.util.*;
-
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.clickhouse.sync.Common;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-@Ignore
-public class ClickHouseBatchSyncServiceTest {
-
-    private ClickHouseAdapter clickHouseAdapter;
-
-    @Before
-    public void init() {
-        clickHouseAdapter = Common.init();
-    }
-
-    @Test
-    public void insert() throws InterruptedException {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric");
-        data.put("role_id", 1L);
-        data.put("c_time", new Date());
-        data.put("test1", "sdfasdfawe中国asfwef");
-        dml.setData(dataList);
-
-        clickHouseAdapter.sync(Collections.singletonList(dml));
-        Thread.sleep(10000L);
-    }
-
-    @Test
-    public void update() {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric2");
-        dml.setData(dataList);
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("name", "Eric");
-        dml.setOld(oldList);
-
-        clickHouseAdapter.sync(Collections.singletonList(dml));
-    }
-
-    @Test
-    public void delete() {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("DELETE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric2");
-        dml.setData(dataList);
-
-        clickHouseAdapter.sync(Collections.singletonList(dml));
-    }
-
-    @Test
-    public void truncate() throws InterruptedException {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("TRUNCATE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric2");
-        dml.setData(dataList);
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("name", "Eric");
-        dml.setOld(oldList);
-
-        clickHouseAdapter.sync(Collections.singletonList(dml));
-        Thread.sleep(1000L);
-
-    }
-}

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseBatchSyncThreadSafeTest.java

@@ -1,81 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse;
-
-import java.util.*;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.clickhouse.sync.Common;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-import ch.qos.logback.classic.Level;
-
-/**
- * @author: Xander
- * @date: Created in 2023/11/13 22:27
- * @email: zhrunxin33@gmail.com
- * @description: Testing thread safe
- */
-
-@Ignore
-public class ClickHouseBatchSyncThreadSafeTest {
-
-    private ClickHouseAdapter clickHouseAdapter;
-
-    private ExecutorService   executorService;
-
-    private String[]          operations = new String[] { "INSERT", "UPDATE" };
-
-    private String[]          tables     = new String[] { "user", "customer" };
-
-    @Before
-    public void init() {
-        clickHouseAdapter = Common.init();
-        Common.setLogLevel(Level.INFO);
-        executorService = Executors.newFixedThreadPool(5);
-    }
-
-    @Test
-    public void test01() throws InterruptedException, ExecutionException {
-        ArrayList<Future> list = new ArrayList();
-        AtomicInteger count = new AtomicInteger();
-        for (int i = 0; i < 10; i++) {
-            list.add(executorService.submit(() -> {
-                for (int j = 0; j < 300; j++) {
-                    Random random = new Random();
-                    int cou = count.incrementAndGet();
-                    // test insert
-                    String dmlType = operations[random.nextInt(1)];
-                    Dml dml = new Dml();
-                    dml.setDestination("example");
-                    dml.setTs(new Date().getTime());
-                    dml.setType(dmlType);
-                    dml.setDatabase("mytest");
-                    dml.setTable(tables[(int) Math.round(Math.random())]);
-                    List<Map<String, Object>> dataList = new ArrayList<>();
-                    Map<String, Object> data = new LinkedHashMap<>();
-                    dataList.add(data);
-                    data.put("id", cou);
-                    data.put("name", "Eric" + cou);
-                    data.put("role_id", cou);
-                    data.put("c_time", new Date());
-                    data.put("test1", "sdfasdfawe中国asfwef");
-                    dml.setData(dataList);
-                    clickHouseAdapter.sync(Collections.singletonList(dml));
-                }
-            }));
-
-        }
-        for (Future future : list) {
-            future.get();
-        }
-        Thread.sleep(10000L); // waiting multiple threads execute successfully.
-    }
-
-}

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/TestConstant.java

@@ -1,37 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse;
-
-import java.sql.SQLException;
-
-import com.alibaba.druid.pool.DruidDataSource;
-
-public class TestConstant {
-
-    public final static String          jdbcUrl      = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
-    public final static String          jdbcUser     = "root";
-    public final static String          jdbcPassword = "121212";
-
-    public final static DruidDataSource dataSource;
-
-    static {
-        dataSource = new DruidDataSource();
-        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
-        dataSource.setUrl(jdbcUrl);
-        dataSource.setUsername(jdbcUser);
-        dataSource.setPassword(jdbcPassword);
-        dataSource.setInitialSize(1);
-        dataSource.setMinIdle(1);
-        dataSource.setMaxActive(1);
-        dataSource.setMaxWait(60000);
-        dataSource.setTimeBetweenEvictionRunsMillis(60000);
-        dataSource.setMinEvictableIdleTimeMillis(300000);
-        dataSource.setPoolPreparedStatements(false);
-        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
-        dataSource.setValidationQuery("select 1");
-        try {
-            dataSource.init();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-    }
-
-}

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/sync/Common.java

@@ -1,51 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.clickhouse.sync;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter;
-import com.alibaba.otter.canal.client.adapter.clickhouse.TestConstant;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.Logger;
-
-/**
- * @author: Xander
- * @date: Created in 2023/11/13 0:16
- * @email: zhrunxin33@gmail.com @description：
- */
-public class Common {
-
-    public static ClickHouseAdapter init() {
-        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
-
-        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
-        outerAdapterConfig.setName("clickhouse");
-        outerAdapterConfig.setKey("clickhouse1");
-        Map<String, String> properties = new HashMap<>();
-        properties.put("jdbc.driveClassName", "ru.yandex.clickhouse.ClickHouseDriver");
-        properties.put("jdbc.url", "jdbc:clickhouse://127.0.0.1:8123/default");
-        properties.put("jdbc.username", "default");
-        properties.put("jdbc.password", "123456");
-        outerAdapterConfig.setProperties(properties);
-
-        ClickHouseAdapter adapter = new ClickHouseAdapter();
-        adapter.init(outerAdapterConfig, null);
-        return adapter;
-    }
-
-    /**
-     * set global log level
-     *
-     * @param logLevel
-     */
-    public static void setLogLevel(Level logLevel) {
-        ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory
-            .getLogger(Logger.ROOT_LOGGER_NAME);
-        logger.setLevel(logLevel);
-    }
-}

client-adapter/clickhouse/src/test/resources/clickhouse/mytest_customer.yml

@@ -1,31 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-outerAdapterKey: clickhouse1
-concurrent: true
-dbMapping:
-  database: mytest
-  table: customer
-  targetTable: mytest.customer
-  targetPk:
-    id: id
-#  mapAll: true
-  targetColumns:
-    id:
-    name:
-    role_id:
-    c_time:
-    test1:
-  etlCondition: "where c_time>={}"
-  commitBatch: 3000 # 批量提交的大小
-
-
-## Mirror schema synchronize config
-#dataSourceKey: defaultDS
-#destination: example
-#groupId: g1
-#outerAdapterKey: mysql1
-#concurrent: true
-#dbMapping:
-#  mirrorDb: true
-#  database: mytest

client-adapter/clickhouse/src/test/resources/clickhouse/mytest_user.yml

@@ -1,31 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-outerAdapterKey: clickhouse1
-concurrent: true
-dbMapping:
-  database: mytest
-  table: user
-  targetTable: mytest.user
-  targetPk:
-    id: id
-#  mapAll: true
-  targetColumns:
-    id:
-    name:
-    role_id:
-    c_time:
-    test1:
-  etlCondition: "where c_time>={}"
-  commitBatch: 3000 # 批量提交的大小
-
-
-## Mirror schema synchronize config
-#dataSourceKey: defaultDS
-#destination: example
-#groupId: g1
-#outerAdapterKey: mysql1
-#concurrent: true
-#dbMapping:
-#  mirrorDb: true
-#  database: mytest

client-adapter/common/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -23,8 +23,29 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.springframework.boot</groupId>
+            <groupId>org.springframework</groupId>
-            <artifactId>spring-boot</artifactId>
+            <artifactId>spring-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-aop</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-orm</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-test</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>joda-time</groupId>
@@ -34,6 +55,10 @@
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.powermock</groupId>
             <artifactId>powermock-api-mockito</artifactId>
@@ -54,15 +79,11 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>com.diffblue</groupId>
             <artifactId>deeptestutils</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>
+
 </project>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/ProxyOuterAdapter.java

@@ -1,87 +0,0 @@
-package com.alibaba.otter.canal.client.adapter;
-
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-public class ProxyOuterAdapter implements OuterAdapter {
-
-    private OuterAdapter outerAdapter;
-
-    public ProxyOuterAdapter(OuterAdapter outerAdapter) {
-        this.outerAdapter = outerAdapter;
-    }
-
-    private ClassLoader changeCL() {
-        ClassLoader cl = Thread.currentThread().getContextClassLoader();
-        Thread.currentThread().setContextClassLoader(outerAdapter.getClass().getClassLoader());
-        return cl;
-    }
-
-    private void revertCL(ClassLoader cl) {
-        Thread.currentThread().setContextClassLoader(cl);
-    }
-
-    @Override
-    public void init(OuterAdapterConfig configuration, Properties envProperties) {
-        ClassLoader cl = changeCL();
-        try {
-            outerAdapter.init(configuration, envProperties);
-        } finally {
-            revertCL(cl);
-        }
-    }
-
-    @Override
-    public void sync(List<Dml> dmls) {
-        ClassLoader cl = changeCL();
-        try {
-            outerAdapter.sync(dmls);
-        } finally {
-            revertCL(cl);
-        }
-    }
-
-    @Override
-    public void destroy() {
-        ClassLoader cl = changeCL();
-        try {
-            outerAdapter.destroy();
-        } finally {
-            revertCL(cl);
-        }
-    }
-
-    @Override
-    public EtlResult etl(String task, List<String> params) {
-        ClassLoader cl = changeCL();
-        try {
-            return OuterAdapter.super.etl(task, params);
-        } finally {
-            revertCL(cl);
-        }
-    }
-
-    @Override
-    public Map<String, Object> count(String task) {
-        ClassLoader cl = changeCL();
-        try {
-            return OuterAdapter.super.count(task);
-        } finally {
-            revertCL(cl);
-        }
-    }
-
-    @Override
-    public String getDestination(String task) {
-        ClassLoader cl = changeCL();
-        try {
-            return OuterAdapter.super.getDestination(task);
-        } finally {
-            revertCL(cl);
-        }
-    }
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java

@@ -0,0 +1,155 @@
+package com.alibaba.otter.canal.client.adapter.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.springframework.util.PropertyPlaceholderHelper;
+import org.springframework.util.StringUtils;
+
+import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory;
+import com.alibaba.otter.canal.client.adapter.config.common.*;
+
+/**
+ * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范
+ *
+ * @author reweerma 2019-2-1 上午9:14:02
+ * @version 1.0.0
+ */
+public class YmlConfigBinder {
+
+    /**
+     * 将当前内容绑定到指定对象
+     *
+     * @param content yml内容
+     * @param clazz 指定对象类型
+     * @return 对象
+     */
+    public static <T> T bindYmlToObj(String content, Class<T> clazz) {
+        return bindYmlToObj(null, content, clazz, null);
+    }
+
+    /**
+     * 将当前内容绑定到指定对象并指定内容编码格式
+     *
+     * @param content yml内容
+     * @param clazz 指定对象类型
+     * @param charset yml内容编码格式
+     * @return 对象
+     */
+    public static <T> T bindYmlToObj(String content, Class<T> clazz, String charset) {
+        return bindYmlToObj(null, content, clazz, charset);
+    }
+
+    /**
+     * 将当前内容指定前缀部分绑定到指定对象
+     *
+     * @param prefix 指定前缀
+     * @param content yml内容
+     * @param clazz 指定对象类型
+     * @return 对象
+     */
+    public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz) {
+        return bindYmlToObj(prefix, content, clazz, null);
+    }
+
+    /**
+     * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式
+     *
+     * @param prefix 指定前缀
+     * @param content yml内容
+     * @param clazz 指定对象类型
+     * @param charset yml内容编码格式
+     * @return 对象
+     */
+    public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset) {
+        return bindYmlToObj(prefix, content, clazz, charset, null);
+    }
+
+    /**
+     * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers}
+     * 在envProperties中有属性 zookeeper.servers:
+     * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为
+     * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中
+     * zkServers 属性是 List<String> 对象, 则会自动映射成List
+     *
+     * @param prefix 指定前缀
+     * @param content yml内容
+     * @param clazz 指定对象类型
+     * @param charset yml内容编码格式
+     * @return 对象
+     */
+    public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset,
+                                     Properties baseProperties) {
+        try {
+            byte[] contentBytes;
+            if (charset == null) {
+                contentBytes = content.getBytes("UTF-8");
+            } else {
+                contentBytes = content.getBytes(charset);
+            }
+            YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
+            Resource configResource = new ByteArrayResource(contentBytes);
+            PropertySource<?> propertySource = propertySourceLoader.load("manualBindConfig", configResource, null);
+
+            if (propertySource == null) {
+                return null;
+            }
+
+            Properties properties = new Properties();
+            Map<String, Object> propertiesRes = new LinkedHashMap<>();
+            if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) {
+                prefix = prefix + ".";
+            }
+
+            properties.putAll((Map<?, ?>) propertySource.getSource());
+
+            if (baseProperties != null) {
+                baseProperties.putAll(properties);
+                properties = baseProperties;
+            }
+
+            for (Map.Entry<?, ?> entry : ((Map<?, ?>) propertySource.getSource()).entrySet()) {
+                String key = (String) entry.getKey();
+                Object value = entry.getValue();
+
+                if (prefix != null) {
+                    if (key != null && key.startsWith(prefix)) {
+                        key = key.substring(prefix.length());
+                    } else {
+                        continue;
+                    }
+                }
+
+                if (value instanceof String && ((String) value).contains("${") && ((String) value).contains("}")) {
+                    PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}");
+                    value = propertyPlaceholderHelper.replacePlaceholders((String) value, properties);
+                }
+
+                propertiesRes.put(key, value);
+            }
+
+            if (propertiesRes.isEmpty()) {
+                return null;
+            }
+
+            propertySource = new MapPropertySource(propertySource.getName(), propertiesRes);
+
+            T target = clazz.newInstance();
+
+            MutablePropertySources propertySources = new MutablePropertySources();
+            propertySources.addFirst(propertySource);
+
+            PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<>(target);
+            factory.setPropertySources(propertySources);
+            factory.setIgnoreInvalidFields(true);
+            factory.setIgnoreUnknownFields(true);
+
+            factory.bindPropertiesToTarget();
+
+            return target;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java

@@ -0,0 +1,97 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * {@link PropertyNamePatternsMatcher} that matches when a property name exactly
+ * matches one of the given names, or starts with one of the given names
+ * followed by a delimiter. This implementation is optimized for frequent calls.
+ *
+ * @author Phillip Webb
+ * @since 1.2.0
+ */
+class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher {
+
+    private final char[]   delimiters;
+
+    private final boolean  ignoreCase;
+
+    private final String[] names;
+
+    protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){
+        this(delimiters, false, names);
+    }
+
+    protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
+        this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names)));
+    }
+
+    DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){
+        this.delimiters = delimiters;
+        this.ignoreCase = ignoreCase;
+        this.names = names.toArray(new String[names.size()]);
+    }
+
+    @Override
+    public boolean matches(String propertyName) {
+        char[] propertyNameChars = propertyName.toCharArray();
+        boolean[] match = new boolean[this.names.length];
+        boolean noneMatched = true;
+        for (int i = 0; i < this.names.length; i++) {
+            if (this.names[i].length() <= propertyNameChars.length) {
+                match[i] = true;
+                noneMatched = false;
+            }
+        }
+        if (noneMatched) {
+            return false;
+        }
+        for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) {
+            for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) {
+                if (match[nameIndex]) {
+                    match[nameIndex] = false;
+                    if (charIndex < this.names[nameIndex].length()) {
+                        if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) {
+                            match[nameIndex] = true;
+                            noneMatched = false;
+                        }
+                    } else {
+                        char charAfter = propertyNameChars[this.names[nameIndex].length()];
+                        if (isDelimiter(charAfter)) {
+                            match[nameIndex] = true;
+                            noneMatched = false;
+                        }
+                    }
+                }
+            }
+            if (noneMatched) {
+                return false;
+            }
+        }
+        for (boolean b : match) {
+            if (b) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean isCharMatch(char c1, char c2) {
+        if (this.ignoreCase) {
+            return Character.toLowerCase(c1) == Character.toLowerCase(c2);
+        }
+        return c1 == c2;
+    }
+
+    private boolean isDelimiter(char c) {
+        for (char delimiter : this.delimiters) {
+            if (c == delimiter) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.beans.PropertyEditorSupport;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * {@link PropertyNamePatternsMatcher} that matches when a property name exactly
+ * matches one of the given names, or starts with one of the given names
+ * followed by a delimiter. This implementation is optimized for frequent calls.
+ *
+ * @author Phillip Webb
+ * @since 1.2.0
+ */
+class InetAddressEditor extends PropertyEditorSupport {
+
+    @Override
+    public String getAsText() {
+        return ((InetAddress) getValue()).getHostAddress();
+    }
+
+    @Override
+    public void setAsText(String text) throws IllegalArgumentException {
+        try {
+            setValue(InetAddress.getByName(text));
+        } catch (UnknownHostException ex) {
+            throw new IllegalArgumentException("Cannot locate host", ex);
+        }
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import org.springframework.beans.PropertyValue;
+
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
+
+/**
+ * A {@link PropertyValue} that can provide information about its origin.
+ *
+ * @author Andy Wilkinson
+ */
+class OriginCapablePropertyValue extends PropertyValue {
+
+    private static final String  ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin";
+
+    private final PropertyOrigin origin;
+
+    OriginCapablePropertyValue(PropertyValue propertyValue){
+        this(propertyValue.getName(),
+            propertyValue.getValue(),
+            (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN));
+    }
+
+    OriginCapablePropertyValue(String name, Object value, String originName, PropertySource<?> originSource){
+        this(name, value, new PropertyOrigin(originSource, originName));
+    }
+
+    OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){
+        super(name, value);
+        this.origin = origin;
+        setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin);
+    }
+
+    public PropertyOrigin getOrigin() {
+        return this.origin;
+    }
+
+    @Override
+    public String toString() {
+        String name = (this.origin != null ? this.origin.getName() : this.getName());
+        String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown");
+        return "'" + name + "' from '" + source + "'";
+    }
+
+    public static PropertyOrigin getOrigin(PropertyValue propertyValue) {
+        if (propertyValue instanceof OriginCapablePropertyValue) {
+            return ((OriginCapablePropertyValue) propertyValue).getOrigin();
+        }
+        return new OriginCapablePropertyValue(propertyValue).getOrigin();
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java

@@ -0,0 +1,27 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.util.Collection;
+
+import org.springframework.util.PatternMatchUtils;
+
+/**
+ * {@link PropertyNamePatternsMatcher} that delegates to
+ * {@link PatternMatchUtils#simpleMatch(String[], String)}.
+ *
+ * @author Phillip Webb
+ * @since 1.2.0
+ */
+class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher {
+
+    private final String[] patterns;
+
+    PatternPropertyNamePatternsMatcher(Collection<String> patterns){
+        this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {});
+    }
+
+    @Override
+    public boolean matches(String propertyName) {
+        return PatternMatchUtils.simpleMatch(this.patterns, propertyName);
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java

@@ -0,0 +1,356 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.beans.PropertyDescriptor;
+import java.util.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.PropertyValues;
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.support.ResourceEditorRegistrar;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.context.MessageSource;
+import org.springframework.context.MessageSourceAware;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.util.Assert;
+import org.springframework.util.StringUtils;
+import org.springframework.validation.*;
+
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
+
+/**
+ * Validate some {@link Properties} (or optionally
+ * {@link org.springframework.core.env.PropertySources}) by binding them to an
+ * object of a specified type and then optionally running a {@link Validator}
+ * over it.
+ *
+ * @param <T> the target type
+ * @author Dave Syer
+ */
+public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, ApplicationContextAware, MessageSourceAware, InitializingBean {
+
+    private static final char[] EXACT_DELIMITERS       = { '_', '.', '[' };
+
+    private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' };
+
+    private static final Log    logger                 = LogFactory.getLog(PropertiesConfigurationFactory.class);
+
+    private boolean             ignoreUnknownFields    = true;
+
+    private boolean             ignoreInvalidFields;
+
+    private boolean             exceptionIfInvalid     = true;
+
+    private PropertySources     propertySources;
+
+    private final T             target;
+
+    private Validator           validator;
+
+    private ApplicationContext  applicationContext;
+
+    private MessageSource       messageSource;
+
+    private boolean             hasBeenBound           = false;
+
+    private boolean             ignoreNestedProperties = false;
+
+    private String              targetName;
+
+    private ConversionService   conversionService;
+
+    private boolean             resolvePlaceholders    = true;
+
+    /**
+     * Create a new {@link PropertiesConfigurationFactory} instance.
+     *
+     * @param target the target object to bind too
+     * @see #PropertiesConfigurationFactory(Class)
+     */
+    public PropertiesConfigurationFactory(T target){
+        Assert.notNull(target, "target must not be null");
+        this.target = target;
+    }
+
+    /**
+     * Create a new {@link PropertiesConfigurationFactory} instance.
+     *
+     * @param type the target type
+     * @see #PropertiesConfigurationFactory(Class)
+     */
+    @SuppressWarnings("unchecked")
+    public PropertiesConfigurationFactory(Class<?> type){
+        Assert.notNull(type, "type must not be null");
+        this.target = (T) BeanUtils.instantiate(type);
+    }
+
+    /**
+     * Flag to disable binding of nested properties (i.e. those with period
+     * separators in their paths). Can be useful to disable this if the name prefix
+     * is empty and you don't want to ignore unknown fields.
+     *
+     * @param ignoreNestedProperties the flag to set (default false)
+     */
+    public void setIgnoreNestedProperties(boolean ignoreNestedProperties) {
+        this.ignoreNestedProperties = ignoreNestedProperties;
+    }
+
+    /**
+     * Set whether to ignore unknown fields, that is, whether to ignore bind
+     * parameters that do not have corresponding fields in the target object.
+     * <p>
+     * Default is "true". Turn this off to enforce that all bind parameters must
+     * have a matching field in the target object.
+     *
+     * @param ignoreUnknownFields if unknown fields should be ignored
+     */
+    public void setIgnoreUnknownFields(boolean ignoreUnknownFields) {
+        this.ignoreUnknownFields = ignoreUnknownFields;
+    }
+
+    /**
+     * Set whether to ignore invalid fields, that is, whether to ignore bind
+     * parameters that have corresponding fields in the target object which are not
+     * accessible (for example because of null values in the nested path).
+     * <p>
+     * Default is "false". Turn this on to ignore bind parameters for nested objects
+     * in non-existing parts of the target object graph.
+     *
+     * @param ignoreInvalidFields if invalid fields should be ignored
+     */
+    public void setIgnoreInvalidFields(boolean ignoreInvalidFields) {
+        this.ignoreInvalidFields = ignoreInvalidFields;
+    }
+
+    /**
+     * Set the target name.
+     *
+     * @param targetName the target name
+     */
+    public void setTargetName(String targetName) {
+        this.targetName = targetName;
+    }
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) {
+        this.applicationContext = applicationContext;
+    }
+
+    /**
+     * Set the message source.
+     *
+     * @param messageSource the message source
+     */
+    @Override
+    public void setMessageSource(MessageSource messageSource) {
+        this.messageSource = messageSource;
+    }
+
+    /**
+     * Set the property sources.
+     *
+     * @param propertySources the property sources
+     */
+    public void setPropertySources(PropertySources propertySources) {
+        this.propertySources = propertySources;
+    }
+
+    /**
+     * Set the conversion service.
+     *
+     * @param conversionService the conversion service
+     */
+    public void setConversionService(ConversionService conversionService) {
+        this.conversionService = conversionService;
+    }
+
+    /**
+     * Set the validator.
+     *
+     * @param validator the validator
+     */
+    public void setValidator(Validator validator) {
+        this.validator = validator;
+    }
+
+    /**
+     * Set a flag to indicate that an exception should be raised if a Validator is
+     * available and validation fails.
+     *
+     * @param exceptionIfInvalid the flag to set
+     * @deprecated as of 1.5, do not specify a {@link Validator} if validation
+     * should not occur
+     */
+    @Deprecated
+    public void setExceptionIfInvalid(boolean exceptionIfInvalid) {
+        this.exceptionIfInvalid = exceptionIfInvalid;
+    }
+
+    /**
+     * Flag to indicate that placeholders should be replaced during binding. Default
+     * is true.
+     *
+     * @param resolvePlaceholders flag value
+     */
+    public void setResolvePlaceholders(boolean resolvePlaceholders) {
+        this.resolvePlaceholders = resolvePlaceholders;
+    }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        bindPropertiesToTarget();
+    }
+
+    @Override
+    public Class<?> getObjectType() {
+        if (this.target == null) {
+            return Object.class;
+        }
+        return this.target.getClass();
+    }
+
+    @Override
+    public boolean isSingleton() {
+        return true;
+    }
+
+    @Override
+    public T getObject() throws Exception {
+        if (!this.hasBeenBound) {
+            bindPropertiesToTarget();
+        }
+        return this.target;
+    }
+
+    public void bindPropertiesToTarget() throws BindException {
+        Assert.state(this.propertySources != null, "PropertySources should not be null");
+        try {
+            if (logger.isTraceEnabled()) {
+                logger.trace("Property Sources: " + this.propertySources);
+
+            }
+            this.hasBeenBound = true;
+            doBindPropertiesToTarget();
+        } catch (BindException ex) {
+            if (this.exceptionIfInvalid) {
+                throw ex;
+            }
+            logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex);
+        }
+    }
+
+    private void doBindPropertiesToTarget() throws BindException {
+        RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target,
+            this.targetName) : new RelaxedDataBinder(this.target));
+        if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) {
+            dataBinder.setValidator(this.validator);
+        }
+        if (this.conversionService != null) {
+            dataBinder.setConversionService(this.conversionService);
+        }
+        dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE);
+        dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties);
+        dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields);
+        dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields);
+        customizeBinder(dataBinder);
+        if (this.applicationContext != null) {
+            ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext,
+                this.applicationContext.getEnvironment());
+            resourceEditorRegistrar.registerCustomEditors(dataBinder);
+        }
+        Iterable<String> relaxedTargetNames = getRelaxedTargetNames();
+        Set<String> names = getNames(relaxedTargetNames);
+        PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames);
+        dataBinder.bind(propertyValues);
+        if (this.validator != null) {
+            dataBinder.validate();
+        }
+        checkForBindingErrors(dataBinder);
+    }
+
+    private Iterable<String> getRelaxedTargetNames() {
+        return (this.target != null
+                && StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null);
+    }
+
+    private Set<String> getNames(Iterable<String> prefixes) {
+        Set<String> names = new LinkedHashSet<>();
+        if (this.target != null) {
+            PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
+            for (PropertyDescriptor descriptor : descriptors) {
+                String name = descriptor.getName();
+                if (!name.equals("class")) {
+                    RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name);
+                    if (prefixes == null) {
+                        for (String relaxedName : relaxedNames) {
+                            names.add(relaxedName);
+                        }
+                    } else {
+                        for (String prefix : prefixes) {
+                            for (String relaxedName : relaxedNames) {
+                                names.add(prefix + "." + relaxedName);
+                                names.add(prefix + "_" + relaxedName);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return names;
+    }
+
+    private PropertyValues getPropertySourcesPropertyValues(Set<String> names, Iterable<String> relaxedTargetNames) {
+        PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames);
+        return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders);
+    }
+
+    private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set<String> names,
+                                                                       Iterable<String> relaxedTargetNames) {
+        if (this.ignoreUnknownFields && !isMapTarget()) {
+            // Since unknown fields are ignored we can filter them out early to save
+            // unnecessary calls to the PropertySource.
+            return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names);
+        }
+        if (relaxedTargetNames != null) {
+            // We can filter properties to those starting with the target name, but
+            // we can't do a complete filter since we need to trigger the
+            // unknown fields check
+            Set<String> relaxedNames = new HashSet<>();
+            for (String relaxedTargetName : relaxedTargetNames) {
+                relaxedNames.add(relaxedTargetName);
+            }
+            return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames);
+        }
+        // Not ideal, we basically can't filter anything
+        return PropertyNamePatternsMatcher.ALL;
+    }
+
+    private boolean isMapTarget() {
+        return this.target != null && Map.class.isAssignableFrom(this.target.getClass());
+    }
+
+    private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException {
+        BindingResult errors = dataBinder.getBindingResult();
+        if (errors.hasErrors()) {
+            logger.error("Properties configuration failed validation");
+            for (ObjectError error : errors.getAllErrors()) {
+                logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault())
+                                                          + " (" + error + ")" : error);
+            }
+            if (this.exceptionIfInvalid) {
+                throw new BindException(errors);
+            }
+        }
+    }
+
+    /**
+     * Customize the data binder.
+     *
+     * @param dataBinder the data binder that will be used to bind and validate
+     */
+    protected void customizeBinder(DataBinder dataBinder) {
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java

@@ -0,0 +1,24 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+/**
+ * Strategy interface used to check if a property name matches specific
+ * criteria.
+ *
+ * @author Phillip Webb
+ * @since 1.2.0
+ */
+interface PropertyNamePatternsMatcher {
+
+    PropertyNamePatternsMatcher ALL  = propertyName -> true;
+
+    PropertyNamePatternsMatcher NONE = propertyName -> false;
+
+    /**
+     * Return {@code true} of the property name matches.
+     *
+     * @param propertyName the property name
+     * @return {@code true} if the property name matches
+     */
+    boolean matches(String propertyName);
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java

@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
+
+/**
+ * The origin of a property, specifically its source and its name before any
+ * prefix was removed.
+ *
+ * @author Andy Wilkinson
+ * @since 1.3.0
+ */
+public class PropertyOrigin {
+
+    private final PropertySource<?> source;
+
+    private final String            name;
+
+    PropertyOrigin(PropertySource<?> source, String name){
+        this.name = name;
+        this.source = source;
+    }
+
+    public PropertySource<?> getSource() {
+        return this.source;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java

@@ -0,0 +1,164 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import org.springframework.core.convert.ConversionException;
+import org.springframework.core.env.AbstractEnvironment;
+import org.springframework.core.env.AbstractPropertyResolver;
+import org.springframework.core.env.PropertyResolver;
+import org.springframework.util.ClassUtils;
+
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
+
+/**
+ * {@link PropertyResolver} implementation that resolves property values against
+ * an underlying set of {@link PropertySources}.
+ *
+ * @author Chris Beams
+ * @author Juergen Hoeller
+ * @see PropertySource
+ * @see PropertySources
+ * @see AbstractEnvironment
+ * @since 3.1
+ */
+public class PropertySourcesPropertyResolver extends AbstractPropertyResolver {
+
+    private final PropertySources propertySources;
+
+    /**
+     * Create a new resolver against the given property sources.
+     *
+     * @param propertySources the set of {@link PropertySource} objects to use
+     */
+    public PropertySourcesPropertyResolver(PropertySources propertySources){
+        this.propertySources = propertySources;
+    }
+
+    @Override
+    public boolean containsProperty(String key) {
+        if (this.propertySources != null) {
+            for (PropertySource<?> propertySource : this.propertySources) {
+                if (propertySource.containsProperty(key)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public String getProperty(String key) {
+        return getProperty(key, String.class, true);
+    }
+
+    @Override
+    public <T> T getProperty(String key, Class<T> targetValueType) {
+        return getProperty(key, targetValueType, true);
+    }
+
+    @Override
+    protected String getPropertyAsRawString(String key) {
+        return getProperty(key, String.class, false);
+    }
+
+    protected <T> T getProperty(String key, Class<T> targetValueType, boolean resolveNestedPlaceholders) {
+        if (this.propertySources != null) {
+            for (PropertySource<?> propertySource : this.propertySources) {
+                if (logger.isTraceEnabled()) {
+                    logger
+                        .trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'");
+                }
+                Object value = propertySource.getProperty(key);
+                if (value != null) {
+                    if (resolveNestedPlaceholders && value instanceof String) {
+                        value = resolveNestedPlaceholders((String) value);
+                    }
+                    logKeyFound(key, propertySource, value);
+                    return convertValueIfNecessary(value, targetValueType);
+                }
+            }
+        }
+        if (logger.isDebugEnabled()) {
+            logger.debug("Could not find key '" + key + "' in any property source");
+        }
+        return null;
+    }
+
+    @Deprecated
+    public <T> Class<T> getPropertyAsClass(String key, Class<T> targetValueType) {
+        if (this.propertySources != null) {
+            for (PropertySource<?> propertySource : this.propertySources) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName()));
+                }
+                Object value = propertySource.getProperty(key);
+                if (value != null) {
+                    logKeyFound(key, propertySource, value);
+                    Class<?> clazz;
+                    if (value instanceof String) {
+                        try {
+                            clazz = ClassUtils.forName((String) value, null);
+                        } catch (Exception ex) {
+                            throw new PropertySourcesPropertyResolver.ClassConversionException((String) value,
+                                targetValueType,
+                                ex);
+                        }
+                    } else if (value instanceof Class) {
+                        clazz = (Class<?>) value;
+                    } else {
+                        clazz = value.getClass();
+                    }
+                    if (!targetValueType.isAssignableFrom(clazz)) {
+                        throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType);
+                    }
+                    @SuppressWarnings("unchecked")
+                    Class<T> targetClass = (Class<T>) clazz;
+                    return targetClass;
+                }
+            }
+        }
+        if (logger.isDebugEnabled()) {
+            logger.debug(String.format("Could not find key '%s' in any property source", key));
+        }
+        return null;
+    }
+
+    /**
+     * Log the given key as found in the given {@link PropertySource}, resulting in
+     * the given value.
+     * <p>
+     * The default implementation writes a debug log message with key and source. As
+     * of 4.3.3, this does not log the value anymore in order to avoid accidental
+     * logging of sensitive settings. Subclasses may override this method to change
+     * the log level and/or log message, including the property's value if desired.
+     *
+     * @param key the key found
+     * @param propertySource the {@code PropertySource} that the key has been found
+     *     in
+     * @param value the corresponding value
+     * @since 4.3.1
+     */
+    protected void logKeyFound(String key, PropertySource<?> propertySource, Object value) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName()
+                         + "' with value of type " + value.getClass().getSimpleName());
+        }
+    }
+
+    @SuppressWarnings("serial")
+    @Deprecated
+    private static class ClassConversionException extends ConversionException {
+
+        public ClassConversionException(Class<?> actual, Class<?> expected){
+            super(String
+                .format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName()));
+        }
+
+        public ClassConversionException(String actual, Class<?> expected, Exception ex){
+            super(
+                String
+                    .format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()),
+                ex);
+        }
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java

@@ -0,0 +1,233 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+
+import org.springframework.beans.MutablePropertyValues;
+import org.springframework.beans.PropertyValue;
+import org.springframework.beans.PropertyValues;
+import org.springframework.util.Assert;
+import org.springframework.validation.DataBinder;
+
+import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource;
+import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource;
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
+import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
+
+/**
+ * A {@link PropertyValues} implementation backed by a {@link PropertySources},
+ * bridging the two abstractions and allowing (for instance) a regular
+ * {@link DataBinder} to be used with the latter.
+ *
+ * @author Dave Syer
+ * @author Phillip Webb
+ */
+public class PropertySourcesPropertyValues implements PropertyValues {
+
+    private static final Pattern                               COLLECTION_PROPERTY = Pattern
+        .compile("\\[(\\d+)\\](\\.\\S+)?");
+
+    private final PropertySources                              propertySources;
+
+    private final Collection<String>                           nonEnumerableFallbackNames;
+
+    private final PropertyNamePatternsMatcher                  includes;
+
+    private final Map<String, PropertyValue>                   propertyValues      = new LinkedHashMap<>();
+
+    private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners    = new ConcurrentHashMap<>();
+
+    private final boolean                                      resolvePlaceholders;
+
+    /**
+     * Create a new PropertyValues from the given PropertySources.
+     *
+     * @param propertySources a PropertySources instance
+     */
+    public PropertySourcesPropertyValues(PropertySources propertySources){
+        this(propertySources, true);
+    }
+
+    /**
+     * Create a new PropertyValues from the given PropertySources that will
+     * optionally resolve placeholders.
+     *
+     * @param propertySources a PropertySources instance
+     * @param resolvePlaceholders {@code true} if placeholders should be resolved.
+     * @since 1.5.2
+     */
+    public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){
+        this(propertySources, (Collection<String>) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders);
+    }
+
+    /**
+     * Create a new PropertyValues from the given PropertySources.
+     *
+     * @param propertySources a PropertySources instance
+     * @param includePatterns property name patterns to include from system
+     *     properties and environment variables
+     * @param nonEnumerableFallbackNames the property names to try in lieu of an
+     *     {@link EnumerablePropertySource}.
+     */
+    public PropertySourcesPropertyValues(PropertySources propertySources, Collection<String> includePatterns,
+                                         Collection<String> nonEnumerableFallbackNames){
+        this(propertySources,
+            nonEnumerableFallbackNames,
+            new PatternPropertyNamePatternsMatcher(includePatterns),
+            true);
+    }
+
+    /**
+     * Create a new PropertyValues from the given PropertySources.
+     *
+     * @param propertySources a PropertySources instance
+     * @param nonEnumerableFallbackNames the property names to try in lieu of an
+     *     {@link EnumerablePropertySource}.
+     * @param includes the property name patterns to include
+     * @param resolvePlaceholders flag to indicate the placeholders should be
+     *     resolved
+     */
+    PropertySourcesPropertyValues(PropertySources propertySources, Collection<String> nonEnumerableFallbackNames,
+                                  PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){
+        Assert.notNull(propertySources, "PropertySources must not be null");
+        Assert.notNull(includes, "Includes must not be null");
+        this.propertySources = propertySources;
+        this.nonEnumerableFallbackNames = nonEnumerableFallbackNames;
+        this.includes = includes;
+        this.resolvePlaceholders = resolvePlaceholders;
+        PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources);
+        for (PropertySource<?> source : propertySources) {
+            processPropertySource(source, resolver);
+        }
+    }
+
+    private void processPropertySource(PropertySource<?> source, PropertySourcesPropertyResolver resolver) {
+        if (source instanceof CompositePropertySource) {
+            processCompositePropertySource((CompositePropertySource) source, resolver);
+        } else if (source instanceof EnumerablePropertySource) {
+            processEnumerablePropertySource((EnumerablePropertySource<?>) source, resolver, this.includes);
+        } else {
+            processNonEnumerablePropertySource(source, resolver);
+        }
+    }
+
+    private void processCompositePropertySource(CompositePropertySource source,
+                                                PropertySourcesPropertyResolver resolver) {
+        for (PropertySource<?> nested : source.getPropertySources()) {
+            processPropertySource(nested, resolver);
+        }
+    }
+
+    private void processEnumerablePropertySource(EnumerablePropertySource<?> source,
+                                                 PropertySourcesPropertyResolver resolver,
+                                                 PropertyNamePatternsMatcher includes) {
+        if (source.getPropertyNames().length > 0) {
+            for (String propertyName : source.getPropertyNames()) {
+                if (includes.matches(propertyName)) {
+                    Object value = getEnumerableProperty(source, resolver, propertyName);
+                    putIfAbsent(propertyName, value, source);
+                }
+            }
+        }
+    }
+
+    private Object getEnumerableProperty(EnumerablePropertySource<?> source, PropertySourcesPropertyResolver resolver,
+                                         String propertyName) {
+        try {
+            if (this.resolvePlaceholders) {
+                return resolver.getProperty(propertyName, Object.class);
+            }
+        } catch (RuntimeException ex) {
+            // Probably could not resolve placeholders, ignore it here
+        }
+        return source.getProperty(propertyName);
+    }
+
+    private void processNonEnumerablePropertySource(PropertySource<?> source,
+                                                    PropertySourcesPropertyResolver resolver) {
+        // We can only do exact matches for non-enumerable property names, but
+        // that's better than nothing...
+        if (this.nonEnumerableFallbackNames == null) {
+            return;
+        }
+        for (String propertyName : this.nonEnumerableFallbackNames) {
+            if (!source.containsProperty(propertyName)) {
+                continue;
+            }
+            Object value = null;
+            try {
+                value = resolver.getProperty(propertyName, Object.class);
+            } catch (RuntimeException ex) {
+                // Probably could not convert to Object, weird, but ignorable
+            }
+            if (value == null) {
+                value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH));
+            }
+            putIfAbsent(propertyName, value, source);
+        }
+    }
+
+    @Override
+    public PropertyValue[] getPropertyValues() {
+        Collection<PropertyValue> values = this.propertyValues.values();
+        return values.toArray(new PropertyValue[values.size()]);
+    }
+
+    @Override
+    public PropertyValue getPropertyValue(String propertyName) {
+        PropertyValue propertyValue = this.propertyValues.get(propertyName);
+        if (propertyValue != null) {
+            return propertyValue;
+        }
+        for (PropertySource<?> source : this.propertySources) {
+            Object value = source.getProperty(propertyName);
+            propertyValue = putIfAbsent(propertyName, value, source);
+            if (propertyValue != null) {
+                return propertyValue;
+            }
+        }
+        return null;
+    }
+
+    private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource<?> source) {
+        if (value != null && !this.propertyValues.containsKey(propertyName)) {
+            PropertySource<?> collectionOwner = this.collectionOwners
+                .putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source);
+            if (collectionOwner == null || collectionOwner == source) {
+                PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source);
+                this.propertyValues.put(propertyName, propertyValue);
+                return propertyValue;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public PropertyValues changesSince(PropertyValues old) {
+        MutablePropertyValues changes = new MutablePropertyValues();
+        // for each property value in the new set
+        for (PropertyValue newValue : getPropertyValues()) {
+            // if there wasn't an old one, add it
+            PropertyValue oldValue = old.getPropertyValue(newValue.getName());
+            if (oldValue == null || !oldValue.equals(newValue)) {
+                changes.addPropertyValue(newValue);
+            }
+        }
+        return changes;
+    }
+
+    @Override
+    public boolean contains(String propertyName) {
+        return getPropertyValue(propertyName) != null;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return this.propertyValues.isEmpty();
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java

@@ -0,0 +1,127 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.util.EnumSet;
+import java.util.Locale;
+import java.util.Set;
+
+import org.springframework.core.convert.ConversionFailedException;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.core.convert.TypeDescriptor;
+import org.springframework.core.convert.converter.Converter;
+import org.springframework.core.convert.converter.ConverterFactory;
+import org.springframework.core.convert.support.DefaultConversionService;
+import org.springframework.core.convert.support.GenericConversionService;
+import org.springframework.util.Assert;
+
+/**
+ * Internal {@link ConversionService} used by {@link RelaxedDataBinder} to
+ * support additional relaxed conversion.
+ *
+ * @author Phillip Webb
+ * @author Stephane Nicoll
+ * @since 1.1.0
+ */
+class RelaxedConversionService implements ConversionService {
+
+    private final ConversionService        conversionService;
+
+    private final GenericConversionService additionalConverters;
+
+    /**
+     * Create a new {@link RelaxedConversionService} instance.
+     *
+     * @param conversionService and option root conversion service
+     */
+    RelaxedConversionService(ConversionService conversionService){
+        this.conversionService = conversionService;
+        this.additionalConverters = new GenericConversionService();
+        DefaultConversionService.addCollectionConverters(this.additionalConverters);
+        this.additionalConverters
+            .addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory());
+        this.additionalConverters.addConverter(new StringToCharArrayConverter());
+    }
+
+    @Override
+    public boolean canConvert(Class<?> sourceType, Class<?> targetType) {
+        return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType))
+               || this.additionalConverters.canConvert(sourceType, targetType);
+    }
+
+    @Override
+    public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) {
+        return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType))
+               || this.additionalConverters.canConvert(sourceType, targetType);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public <T> T convert(Object source, Class<T> targetType) {
+        Assert.notNull(targetType, "The targetType to convert to cannot be null");
+        return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType));
+    }
+
+    @Override
+    public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) {
+        if (this.conversionService != null) {
+            try {
+                return this.conversionService.convert(source, sourceType, targetType);
+            } catch (ConversionFailedException ex) {
+                // Ignore and try the additional converters
+            }
+        }
+        return this.additionalConverters.convert(source, sourceType, targetType);
+    }
+
+    /**
+     * Clone of Spring's package private StringToEnumConverterFactory, but ignoring
+     * the case of the source.
+     */
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory<String, Enum> {
+
+        @Override
+        public <T extends Enum> Converter<String, T> getConverter(Class<T> targetType) {
+            Class<?> enumType = targetType;
+            while (enumType != null && !enumType.isEnum()) {
+                enumType = enumType.getSuperclass();
+            }
+            Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum");
+            return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType);
+        }
+
+        private static class StringToEnum<T extends Enum> implements Converter<String, T> {
+
+            private final Class<T> enumType;
+
+            StringToEnum(Class<T> enumType){
+                this.enumType = enumType;
+            }
+
+            @Override
+            public T convert(String source) {
+                if (source.isEmpty()) {
+                    // It's an empty enum identifier: reset the enum value to null.
+                    return null;
+                }
+                source = source.trim();
+                for (T candidate : (Set<T>) EnumSet.allOf(this.enumType)) {
+                    RelaxedNames names = new RelaxedNames(
+                        candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH));
+                    for (String name : names) {
+                        if (name.equals(source)) {
+                            return candidate;
+                        }
+                    }
+                    if (candidate.name().equalsIgnoreCase(source)) {
+                        return candidate;
+                    }
+                }
+                throw new IllegalArgumentException(
+                    "No enum constant " + this.enumType.getCanonicalName() + "." + source);
+            }
+
+        }
+
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java

@@ -0,0 +1,729 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.beans.PropertyEditor;
+import java.net.InetAddress;
+import java.util.*;
+
+import org.springframework.beans.*;
+import org.springframework.beans.propertyeditors.FileEditor;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.core.convert.TypeDescriptor;
+import org.springframework.core.env.StandardEnvironment;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.util.StringUtils;
+import org.springframework.validation.AbstractPropertyBindingResult;
+import org.springframework.validation.BeanPropertyBindingResult;
+import org.springframework.validation.DataBinder;
+
+/**
+ * Binder implementation that allows caller to bind to maps and also allows
+ * property names to match a bit loosely (if underscores or dashes are removed
+ * and replaced with camel case for example).
+ *
+ * @author Dave Syer
+ * @author Phillip Webb
+ * @author Stephane Nicoll
+ * @author Andy Wilkinson
+ * @see RelaxedNames
+ */
+public class RelaxedDataBinder extends DataBinder {
+
+    private static final Set<Class<?>> EXCLUDED_EDITORS;
+
+    static {
+        Set<Class<?>> excluded = new HashSet<>();
+        excluded.add(FileEditor.class);
+        EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
+    }
+
+    private static final Object           BLANK       = new Object();
+
+    private String                        namePrefix;
+
+    private boolean                       ignoreNestedProperties;
+
+    private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<>();
+
+    /**
+     * Create a new {@link RelaxedDataBinder} instance.
+     *
+     * @param target the target into which properties are bound
+     */
+    public RelaxedDataBinder(Object target){
+        super(wrapTarget(target));
+    }
+
+    /**
+     * Create a new {@link RelaxedDataBinder} instance.
+     *
+     * @param target the target into which properties are bound
+     * @param namePrefix An optional prefix to be used when reading properties
+     */
+    public RelaxedDataBinder(Object target, String namePrefix){
+        super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME));
+        this.namePrefix = cleanNamePrefix(namePrefix);
+    }
+
+    private String cleanNamePrefix(String namePrefix) {
+        if (!StringUtils.hasLength(namePrefix)) {
+            return null;
+        }
+        return (namePrefix.endsWith(".") ? namePrefix : namePrefix + ".");
+    }
+
+    /**
+     * Flag to disable binding of nested properties (i.e. those with period
+     * separators in their paths). Can be useful to disable this if the name prefix
+     * is empty and you don't want to ignore unknown fields.
+     *
+     * @param ignoreNestedProperties the flag to set (default false)
+     */
+    public void setIgnoreNestedProperties(boolean ignoreNestedProperties) {
+        this.ignoreNestedProperties = ignoreNestedProperties;
+    }
+
+    /**
+     * Set name aliases.
+     *
+     * @param aliases a map of property name to aliases
+     */
+    public void setNameAliases(Map<String, List<String>> aliases) {
+        this.nameAliases = new LinkedMultiValueMap<>(aliases);
+    }
+
+    /**
+     * Add aliases to the {@link DataBinder}.
+     *
+     * @param name the property name to alias
+     * @param alias aliases for the property names
+     * @return this instance
+     */
+    public RelaxedDataBinder withAlias(String name, String... alias) {
+        for (String value : alias) {
+            this.nameAliases.add(name, value);
+        }
+        return this;
+    }
+
+    @Override
+    protected void doBind(MutablePropertyValues propertyValues) {
+        super.doBind(modifyProperties(propertyValues, getTarget()));
+    }
+
+    /**
+     * Modify the property values so that period separated property paths are valid
+     * for map keys. Also creates new maps for properties of map type that are null
+     * (assuming all maps are potentially nested). The standard bracket {@code[...]}
+     * dereferencing is also accepted.
+     *
+     * @param propertyValues the property values
+     * @param target the target object
+     * @return modified property values
+     */
+    private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) {
+        propertyValues = getPropertyValuesForNamePrefix(propertyValues);
+        if (target instanceof RelaxedDataBinder.MapHolder) {
+            propertyValues = addMapPrefix(propertyValues);
+        }
+        BeanWrapper wrapper = new BeanWrapperImpl(target);
+        wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
+        wrapper.setAutoGrowNestedPaths(true);
+        List<PropertyValue> sortedValues = new ArrayList<>();
+        Set<String> modifiedNames = new HashSet<>();
+        List<String> sortedNames = getSortedPropertyNames(propertyValues);
+        for (String name : sortedNames) {
+            PropertyValue propertyValue = propertyValues.getPropertyValue(name);
+            PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue);
+            if (modifiedNames.add(modifiedProperty.getName())) {
+                sortedValues.add(modifiedProperty);
+            }
+        }
+        return new MutablePropertyValues(sortedValues);
+    }
+
+    private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
+        List<String> names = new LinkedList<>();
+        for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
+            names.add(propertyValue.getName());
+        }
+        sortPropertyNames(names);
+        return names;
+    }
+
+    /**
+     * Sort by name so that parent properties get processed first (e.g. 'foo.bar'
+     * before 'foo.bar.spam'). Don't use Collections.sort() because the order might
+     * be significant for other property names (it shouldn't be but who knows what
+     * people might be relying on, e.g. HSQL has a JDBCXADataSource where
+     * "databaseName" is a synonym for "url").
+     *
+     * @param names the names to sort
+     */
+    private void sortPropertyNames(List<String> names) {
+        for (String name : new ArrayList<>(names)) {
+            int propertyIndex = names.indexOf(name);
+            RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
+            for (String prefix : path.prefixes()) {
+                int prefixIndex = names.indexOf(prefix);
+                if (prefixIndex >= propertyIndex) {
+                    // The child property has a parent in the list in the wrong order
+                    names.remove(name);
+                    names.add(prefixIndex, name);
+                }
+            }
+        }
+    }
+
+    private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) {
+        MutablePropertyValues rtn = new MutablePropertyValues();
+        for (PropertyValue pv : propertyValues.getPropertyValues()) {
+            rtn.add("map." + pv.getName(), pv.getValue());
+        }
+        return rtn;
+    }
+
+    private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) {
+        if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) {
+            return propertyValues;
+        }
+        MutablePropertyValues rtn = new MutablePropertyValues();
+        for (PropertyValue value : propertyValues.getPropertyValues()) {
+            String name = value.getName();
+            for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) {
+                for (String separator : new String[] { ".", "_" }) {
+                    String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix);
+                    if (name.startsWith(candidate)) {
+                        name = name.substring(candidate.length());
+                        if (!(this.ignoreNestedProperties && name.contains("."))) {
+                            PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value);
+                            rtn.addPropertyValue(
+                                new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin));
+                        }
+                    }
+                }
+            }
+        }
+        return rtn;
+    }
+
+    private String stripLastDot(String string) {
+        if (StringUtils.hasLength(string) && string.endsWith(".")) {
+            string = string.substring(0, string.length() - 1);
+        }
+        return string;
+    }
+
+    private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) {
+        String name = propertyValue.getName();
+        String normalizedName = normalizePath(target, name);
+        if (!normalizedName.equals(name)) {
+            return new PropertyValue(normalizedName, propertyValue.getValue());
+        }
+        return propertyValue;
+    }
+
+    /**
+     * Normalize a bean property path to a format understood by a BeanWrapper. This
+     * is used so that
+     * <ul>
+     * <li>Fuzzy matching can be employed for bean property names</li>
+     * <li>Period separators can be used instead of indexing ([...]) for map
+     * keys</li>
+     * </ul>
+     *
+     * @param wrapper a bean wrapper for the object to bind
+     * @param path the bean path to bind
+     * @return a transformed path with correct bean wrapper syntax
+     */
+    protected String normalizePath(BeanWrapper wrapper, String path) {
+        return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0);
+    }
+
+    @Override
+    protected AbstractPropertyBindingResult createBeanPropertyBindingResult() {
+        return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(),
+            getObjectName(),
+            isAutoGrowNestedPaths(),
+            getAutoGrowCollectionLimit(),
+            getConversionService());
+    }
+
+    private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
+        String prefix = path.prefix(index);
+        String key = path.name(index);
+        if (path.isProperty(index)) {
+            key = getActualPropertyName(wrapper, prefix, key);
+            path.rename(index, key);
+        }
+        if (path.name(++index) == null) {
+            return path.toString();
+        }
+        String name = path.prefix(index);
+        TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name);
+        if (descriptor == null || descriptor.isMap()) {
+            if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) {
+                path.collapseKeys(index);
+            }
+            path.mapIndex(index);
+            extendMapIfNecessary(wrapper, path, index);
+        } else if (descriptor.isCollection()) {
+            extendCollectionIfNecessary(wrapper, path, index);
+        } else if (descriptor.getType().equals(Object.class)) {
+            if (isBlanked(wrapper, name, path.name(index))) {
+                path.collapseKeys(index);
+            }
+            path.mapIndex(index);
+            if (path.isLastNode(index)) {
+                wrapper.setPropertyValue(path.toString(), BLANK);
+            } else {
+                String next = path.prefix(index + 1);
+                if (wrapper.getPropertyValue(next) == null) {
+                    wrapper.setPropertyValue(next, new LinkedHashMap<String, Object>());
+                }
+            }
+        }
+        return initializePath(wrapper, path, index);
+    }
+
+    private boolean isMapValueStringType(TypeDescriptor descriptor) {
+        if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) {
+            return false;
+        }
+        if (Properties.class.isAssignableFrom(descriptor.getObjectType())) {
+            // Properties is declared as Map<Object,Object> but we know it's really
+            // Map<String,String>
+            return true;
+        }
+        Class<?> valueType = descriptor.getMapValueTypeDescriptor().getObjectType();
+        return (valueType != null && CharSequence.class.isAssignableFrom(valueType));
+    }
+
+    @SuppressWarnings("rawtypes")
+    private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) {
+        Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null);
+        if (value instanceof Map) {
+            if (((Map) value).get(key) == BLANK) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
+        String name = path.prefix(index);
+        TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor();
+        if (!elementDescriptor.isMap() && !elementDescriptor.isCollection()
+            && !elementDescriptor.getType().equals(Object.class)) {
+            return;
+        }
+        Object extend = new LinkedHashMap<String, Object>();
+        if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
+            extend = new ArrayList<>();
+        }
+        wrapper.setPropertyValue(path.prefix(index + 1), extend);
+    }
+
+    private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
+        String name = path.prefix(index);
+        TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name);
+        if (parent == null) {
+            return;
+        }
+        TypeDescriptor descriptor = parent.getMapValueTypeDescriptor();
+        if (descriptor == null) {
+            descriptor = TypeDescriptor.valueOf(Object.class);
+        }
+        if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) {
+            return;
+        }
+        String extensionName = path.prefix(index + 1);
+        if (wrapper.isReadableProperty(extensionName)) {
+            Object currentValue = wrapper.getPropertyValue(extensionName);
+            if ((descriptor.isCollection() && currentValue instanceof Collection)
+                || (!descriptor.isCollection() && currentValue instanceof Map)) {
+                return;
+            }
+        }
+        Object extend = new LinkedHashMap<String, Object>();
+        if (descriptor.isCollection()) {
+            extend = new ArrayList<>();
+        }
+        if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
+            extend = BLANK;
+        }
+        wrapper.setPropertyValue(extensionName, extend);
+    }
+
+    private String getActualPropertyName(BeanWrapper target, String prefix, String name) {
+        String propertyName = resolvePropertyName(target, prefix, name);
+        if (propertyName == null) {
+            propertyName = resolveNestedPropertyName(target, prefix, name);
+        }
+        return (propertyName != null ? propertyName : name);
+    }
+
+    private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) {
+        StringBuilder candidate = new StringBuilder();
+        for (String field : name.split("[_\\-\\.]")) {
+            candidate.append(candidate.length() > 0 ? "." : "");
+            candidate.append(field);
+            String nested = resolvePropertyName(target, prefix, candidate.toString());
+            if (nested != null) {
+                Class<?> type = target.getPropertyType(nested);
+                if ((type != null) && Map.class.isAssignableFrom(type)) {
+                    // Special case for map property (gh-3836).
+                    return nested + "[" + name.substring(candidate.length() + 1) + "]";
+                }
+                String propertyName = resolvePropertyName(target,
+                    joinString(prefix, nested),
+                    name.substring(candidate.length() + 1));
+                if (propertyName != null) {
+                    return joinString(nested, propertyName);
+                }
+            }
+        }
+        return null;
+    }
+
+    private String resolvePropertyName(BeanWrapper target, String prefix, String name) {
+        Iterable<String> names = getNameAndAliases(name);
+        for (String nameOrAlias : names) {
+            for (String candidate : new RelaxedNames(nameOrAlias)) {
+                try {
+                    if (target.getPropertyType(joinString(prefix, candidate)) != null) {
+                        return candidate;
+                    }
+                } catch (InvalidPropertyException ex) {
+                    // swallow and continue
+                }
+            }
+        }
+        return null;
+    }
+
+    private String joinString(String prefix, String name) {
+        return (StringUtils.hasLength(prefix) ? prefix + "." + name : name);
+    }
+
+    private Iterable<String> getNameAndAliases(String name) {
+        List<String> aliases = this.nameAliases.get(name);
+        if (aliases == null) {
+            return Collections.singleton(name);
+        }
+        List<String> nameAndAliases = new ArrayList<>(aliases.size() + 1);
+        nameAndAliases.add(name);
+        nameAndAliases.addAll(aliases);
+        return nameAndAliases;
+    }
+
+    private static Object wrapTarget(Object target) {
+        if (target instanceof Map) {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> map = (Map<String, Object>) target;
+            target = new RelaxedDataBinder.MapHolder(map);
+        }
+        return target;
+    }
+
+    @Override
+    public void registerCustomEditor(Class<?> requiredType, PropertyEditor propertyEditor) {
+        if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) {
+            super.registerCustomEditor(requiredType, propertyEditor);
+        }
+    }
+
+    @Override
+    public void registerCustomEditor(Class<?> requiredType, String field, PropertyEditor propertyEditor) {
+        if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) {
+            super.registerCustomEditor(requiredType, field, propertyEditor);
+        }
+    }
+
+    /**
+     * Holder to allow Map targets to be bound.
+     */
+    static class MapHolder {
+
+        private Map<String, Object> map;
+
+        MapHolder(Map<String, Object> map){
+            this.map = map;
+        }
+
+        public void setMap(Map<String, Object> map) {
+            this.map = map;
+        }
+
+        public Map<String, Object> getMap() {
+            return this.map;
+        }
+
+    }
+
+    /**
+     * A path though properties of a bean.
+     */
+    private static class BeanPath {
+
+        private List<PathNode> nodes;
+
+        BeanPath(String path){
+            this.nodes = splitPath(path);
+        }
+
+        public List<String> prefixes() {
+            List<String> prefixes = new ArrayList<>();
+            for (int index = 1; index < this.nodes.size(); index++) {
+                prefixes.add(prefix(index));
+            }
+            return prefixes;
+        }
+
+        public boolean isLastNode(int index) {
+            return index >= this.nodes.size() - 1;
+        }
+
+        private List<PathNode> splitPath(String path) {
+            List<PathNode> nodes = new ArrayList<>();
+            String current = extractIndexedPaths(path, nodes);
+            for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
+                if (StringUtils.hasText(name)) {
+                    nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name));
+                }
+            }
+            return nodes;
+        }
+
+        private String extractIndexedPaths(String path, List<PathNode> nodes) {
+            int startRef = path.indexOf("[");
+            String current = path;
+            while (startRef >= 0) {
+                if (startRef > 0) {
+                    nodes.addAll(splitPath(current.substring(0, startRef)));
+                }
+                int endRef = current.indexOf("]", startRef);
+                if (endRef > 0) {
+                    String sub = current.substring(startRef + 1, endRef);
+                    if (sub.matches("[0-9]+")) {
+                        nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub));
+                    } else {
+                        nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub));
+                    }
+                }
+                current = current.substring(endRef + 1);
+                startRef = current.indexOf("[");
+            }
+            return current;
+        }
+
+        public void collapseKeys(int index) {
+            List<PathNode> revised = new ArrayList<>();
+            for (int i = 0; i < index; i++) {
+                revised.add(this.nodes.get(i));
+            }
+            StringBuilder builder = new StringBuilder();
+            for (int i = index; i < this.nodes.size(); i++) {
+                if (i > index) {
+                    builder.append(".");
+                }
+                builder.append(this.nodes.get(i).name);
+            }
+            revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString()));
+            this.nodes = revised;
+        }
+
+        public void mapIndex(int index) {
+            RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index);
+            if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) {
+                node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex();
+            }
+            this.nodes.set(index, node);
+        }
+
+        public String prefix(int index) {
+            return range(0, index);
+        }
+
+        public void rename(int index, String name) {
+            this.nodes.get(index).name = name;
+        }
+
+        public String name(int index) {
+            if (index < this.nodes.size()) {
+                return this.nodes.get(index).name;
+            }
+            return null;
+        }
+
+        private String range(int start, int end) {
+            StringBuilder builder = new StringBuilder();
+            for (int i = start; i < end; i++) {
+                RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i);
+                builder.append(node);
+            }
+            if (builder.toString().startsWith(("."))) {
+                builder.replace(0, 1, "");
+            }
+            return builder.toString();
+        }
+
+        public boolean isArrayIndex(int index) {
+            return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode;
+        }
+
+        public boolean isProperty(int index) {
+            return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode;
+        }
+
+        @Override
+        public String toString() {
+            return prefix(this.nodes.size());
+        }
+
+        private static class PathNode {
+
+            protected String name;
+
+            PathNode(String name){
+                this.name = name;
+            }
+
+        }
+
+        private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode {
+
+            ArrayIndexNode(String name){
+                super(name);
+            }
+
+            @Override
+            public String toString() {
+                return "[" + this.name + "]";
+            }
+
+        }
+
+        private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode {
+
+            MapIndexNode(String name){
+                super(name);
+            }
+
+            @Override
+            public String toString() {
+                return "[" + this.name + "]";
+            }
+
+        }
+
+        private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode {
+
+            PropertyNode(String name){
+                super(name);
+            }
+
+            public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() {
+                return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name);
+            }
+
+            @Override
+            public String toString() {
+                return "." + this.name;
+            }
+
+        }
+
+    }
+
+    /**
+     * Extended version of {@link BeanPropertyBindingResult} to support relaxed
+     * binding.
+     */
+    private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult {
+
+        private RelaxedConversionService conversionService;
+
+        RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths,
+                                         int autoGrowCollectionLimit, ConversionService conversionService){
+            super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit);
+            this.conversionService = new RelaxedConversionService(conversionService);
+        }
+
+        @Override
+        protected BeanWrapper createBeanWrapper() {
+            BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget());
+            beanWrapper.setConversionService(this.conversionService);
+            beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor());
+            return beanWrapper;
+        }
+
+    }
+
+    /**
+     * Extended version of {@link BeanWrapperImpl} to support relaxed binding.
+     */
+    private static class RelaxedBeanWrapper extends BeanWrapperImpl {
+
+        private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
+
+        static {
+            Set<String> names = new HashSet<>();
+            names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
+            names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
+            BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);
+        }
+
+        RelaxedBeanWrapper(Object target){
+            super(target);
+        }
+
+        @Override
+        public void setPropertyValue(PropertyValue pv) throws BeansException {
+            try {
+                super.setPropertyValue(pv);
+            } catch (NotWritablePropertyException ex) {
+                PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv);
+                if (isBenign(origin)) {
+                    logger.debug("Ignoring benign property binding failure", ex);
+                    return;
+                }
+                if (origin == null) {
+                    throw ex;
+                }
+                throw new RelaxedBindingNotWritablePropertyException(ex, origin);
+            }
+        }
+
+        private boolean isBenign(PropertyOrigin origin) {
+            String name = (origin != null ? origin.getSource().getName() : null);
+            return BENIGN_PROPERTY_SOURCE_NAMES.contains(name);
+        }
+
+    }
+
+    public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException {
+
+        private final String         message;
+
+        private final PropertyOrigin propertyOrigin;
+
+        RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){
+            super(ex.getBeanClass(), ex.getPropertyName());
+            this.propertyOrigin = propertyOrigin;
+            this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '"
+                           + propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '"
+                           + ex.getBeanClass().getName() + "'";
+        }
+
+        @Override
+        public String getMessage() {
+            return this.message;
+        }
+
+        public PropertyOrigin getPropertyOrigin() {
+            return this.propertyOrigin;
+        }
+
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java

@@ -0,0 +1,241 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Locale;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.springframework.util.StringUtils;
+
+/**
+ * Generates relaxed name variations from a given source.
+ *
+ * @author Phillip Webb
+ * @author Dave Syer
+ * @see RelaxedDataBinder
+ */
+public final class RelaxedNames implements Iterable<String> {
+
+    private static final Pattern CAMEL_CASE_PATTERN              = Pattern.compile("([^A-Z-])([A-Z])");
+
+    private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]");
+
+    private final String         name;
+
+    private final Set<String>    values                          = new LinkedHashSet<>();
+
+    /**
+     * Create a new {@link RelaxedNames} instance.
+     *
+     * @param name the source name. For the maximum number of variations specify the
+     *     name using dashed notation (e.g. {@literal my-property-name}
+     */
+    public RelaxedNames(String name){
+        this.name = (name != null ? name : "");
+        initialize(RelaxedNames.this.name, this.values);
+    }
+
+    @Override
+    public Iterator<String> iterator() {
+        return this.values.iterator();
+    }
+
+    private void initialize(String name, Set<String> values) {
+        if (values.contains(name)) {
+            return;
+        }
+        for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) {
+            for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) {
+                String result = name;
+                result = manipulation.apply(result);
+                result = variation.apply(result);
+                values.add(result);
+                initialize(result, values);
+            }
+        }
+    }
+
+    /**
+     * Name variations.
+     */
+    enum Variation {
+
+                    NONE {
+
+                        @Override
+                        public String apply(String value) {
+                            return value;
+                        }
+
+                    },
+
+                    LOWERCASE {
+
+                        @Override
+                        public String apply(String value) {
+                            return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH));
+                        }
+
+                    },
+
+                    UPPERCASE {
+
+                        @Override
+                        public String apply(String value) {
+                            return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH));
+                        }
+
+                    };
+
+        public abstract String apply(String value);
+
+    }
+
+    /**
+     * Name manipulations.
+     */
+    enum Manipulation {
+
+                       NONE {
+
+                           @Override
+                           public String apply(String value) {
+                               return value;
+                           }
+
+                       },
+
+                       HYPHEN_TO_UNDERSCORE {
+
+                           @Override
+                           public String apply(String value) {
+                               return (value.indexOf('-') != -1 ? value.replace('-', '_') : value);
+                           }
+
+                       },
+
+                       UNDERSCORE_TO_PERIOD {
+
+                           @Override
+                           public String apply(String value) {
+                               return (value.indexOf('_') != -1 ? value.replace('_', '.') : value);
+                           }
+
+                       },
+
+                       PERIOD_TO_UNDERSCORE {
+
+                           @Override
+                           public String apply(String value) {
+                               return (value.indexOf('.') != -1 ? value.replace('.', '_') : value);
+                           }
+
+                       },
+
+                       CAMELCASE_TO_UNDERSCORE {
+
+                           @Override
+                           public String apply(String value) {
+                               if (value.isEmpty()) {
+                                   return value;
+                               }
+                               Matcher matcher = CAMEL_CASE_PATTERN.matcher(value);
+                               if (!matcher.find()) {
+                                   return value;
+                               }
+                               matcher = matcher.reset();
+                               StringBuffer result = new StringBuffer();
+                               while (matcher.find()) {
+                                   matcher.appendReplacement(result,
+                                       matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2)));
+                               }
+                               matcher.appendTail(result);
+                               return result.toString();
+                           }
+
+                       },
+
+                       CAMELCASE_TO_HYPHEN {
+
+                           @Override
+                           public String apply(String value) {
+                               if (value.isEmpty()) {
+                                   return value;
+                               }
+                               Matcher matcher = CAMEL_CASE_PATTERN.matcher(value);
+                               if (!matcher.find()) {
+                                   return value;
+                               }
+                               matcher = matcher.reset();
+                               StringBuffer result = new StringBuffer();
+                               while (matcher.find()) {
+                                   matcher.appendReplacement(result,
+                                       matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2)));
+                               }
+                               matcher.appendTail(result);
+                               return result.toString();
+                           }
+
+                       },
+
+                       SEPARATED_TO_CAMELCASE {
+
+                           @Override
+                           public String apply(String value) {
+                               return separatedToCamelCase(value, false);
+                           }
+
+                       },
+
+                       CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE {
+
+                           @Override
+                           public String apply(String value) {
+                               return separatedToCamelCase(value, true);
+                           }
+
+                       };
+
+        private static final char[] SUFFIXES = new char[] { '_', '-', '.' };
+
+        public abstract String apply(String value);
+
+        private static String separatedToCamelCase(String value, boolean caseInsensitive) {
+            if (value.isEmpty()) {
+                return value;
+            }
+            StringBuilder builder = new StringBuilder();
+            for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) {
+                field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field);
+                builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field);
+            }
+            char lastChar = value.charAt(value.length() - 1);
+            for (char suffix : SUFFIXES) {
+                if (lastChar == suffix) {
+                    builder.append(suffix);
+                    break;
+                }
+            }
+            return builder.toString();
+        }
+
+    }
+
+    /**
+     * Return a {@link RelaxedNames} for the given source camelCase source name.
+     *
+     * @param name the source name in camelCase
+     * @return the relaxed names
+     */
+    public static RelaxedNames forCamelCase(String name) {
+        StringBuilder result = new StringBuilder();
+        for (char c : name.toCharArray()) {
+            result.append(Character.isUpperCase(c) && result.length() > 0
+                          && result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c);
+        }
+        return new RelaxedNames(result.toString());
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java

@@ -0,0 +1,17 @@
+package com.alibaba.otter.canal.client.adapter.config.bind;
+
+import org.springframework.core.convert.converter.Converter;
+
+/**
+ * Converts a String to a Char Array.
+ *
+ * @author Phillip Webb
+ */
+class StringToCharArrayConverter implements Converter<String, char[]> {
+
+    @Override
+    public char[] convert(String source) {
+        return source.toCharArray();
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java

@@ -0,0 +1,203 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.springframework.util.Assert;
+import org.springframework.util.ResourceUtils;
+
+/**
+ * Convenience base class for {@link Resource} implementations, pre-implementing
+ * typical behavior.
+ * <p>
+ * The "exists" method will check whether a File or InputStream can be opened;
+ * "isOpen" will always return false; "getURL" and "getFile" throw an exception;
+ * and "toString" will return the description.
+ *
+ * @author Juergen Hoeller
+ * @since 28.12.2003
+ */
+public abstract class AbstractResource implements Resource {
+
+    /**
+     * This implementation checks whether a File can be opened, falling back to
+     * whether an InputStream can be opened. This will cover both directories and
+     * content resources.
+     */
+    @Override
+    public boolean exists() {
+        // Try file existence: can we find the file in the file system?
+        try {
+            return getFile().exists();
+        } catch (IOException ex) {
+            // Fall back to stream existence: can we open the stream?
+            try {
+                InputStream is = getInputStream();
+                is.close();
+                return true;
+            } catch (Throwable isEx) {
+                return false;
+            }
+        }
+    }
+
+    /**
+     * This implementation always returns {@code true}.
+     */
+    @Override
+    public boolean isReadable() {
+        return true;
+    }
+
+    /**
+     * This implementation always returns {@code false}.
+     */
+    @Override
+    public boolean isOpen() {
+        return false;
+    }
+
+    /**
+     * This implementation throws a FileNotFoundException, assuming that the
+     * resource cannot be resolved to a URL.
+     */
+    @Override
+    public URL getURL() throws IOException {
+        throw new FileNotFoundException(getDescription() + " cannot be resolved to URL");
+    }
+
+    /**
+     * This implementation builds a URI based on the URL returned by
+     * {@link #getURL()}.
+     */
+    @Override
+    public URI getURI() throws IOException {
+        URL url = getURL();
+        try {
+            return ResourceUtils.toURI(url);
+        } catch (URISyntaxException ex) {
+            throw new RuntimeException("Invalid URI [" + url + "]", ex);
+        }
+    }
+
+    /**
+     * This implementation throws a FileNotFoundException, assuming that the
+     * resource cannot be resolved to an absolute file path.
+     */
+    @Override
+    public File getFile() throws IOException {
+        throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path");
+    }
+
+    /**
+     * This implementation reads the entire InputStream to calculate the content
+     * length. Subclasses will almost always be able to provide a more optimal
+     * version of this, e.g. checking a File length.
+     *
+     * @see #getInputStream()
+     */
+    @Override
+    public long contentLength() throws IOException {
+        InputStream is = getInputStream();
+        Assert.state(is != null, "Resource InputStream must not be null");
+        try {
+            long size = 0;
+            byte[] buf = new byte[255];
+            int read;
+            while ((read = is.read(buf)) != -1) {
+                size += read;
+            }
+            return size;
+        } finally {
+            try {
+                is.close();
+            } catch (IOException ex) {
+            }
+        }
+    }
+
+    /**
+     * This implementation checks the timestamp of the underlying File, if
+     * available.
+     *
+     * @see #getFileForLastModifiedCheck()
+     */
+    @Override
+    public long lastModified() throws IOException {
+        long lastModified = getFileForLastModifiedCheck().lastModified();
+        if (lastModified == 0L) {
+            throw new FileNotFoundException(
+                getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp");
+        }
+        return lastModified;
+    }
+
+    /**
+     * Determine the File to use for timestamp checking.
+     * <p>
+     * The default implementation delegates to {@link #getFile()}.
+     *
+     * @return the File to use for timestamp checking (never {@code null})
+     * @throws FileNotFoundException if the resource cannot be resolved as an
+     *     absolute file path, i.e. is not available in a file system
+     * @throws IOException in case of general resolution/reading failures
+     */
+    protected File getFileForLastModifiedCheck() throws IOException {
+        return getFile();
+    }
+
+    /**
+     * This implementation throws a FileNotFoundException, assuming that relative
+     * resources cannot be created for this resource.
+     */
+    @Override
+    public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException {
+        throw new FileNotFoundException("Cannot create a relative resource for " + getDescription());
+    }
+
+    /**
+     * This implementation always returns {@code null}, assuming that this resource
+     * type does not have a filename.
+     */
+    @Override
+    public String getFilename() {
+        return null;
+    }
+
+    /**
+     * This implementation returns the description of this resource.
+     *
+     * @see #getDescription()
+     */
+    @Override
+    public String toString() {
+        return getDescription();
+    }
+
+    /**
+     * This implementation compares description strings.
+     *
+     * @see #getDescription()
+     */
+    @Override
+    public boolean equals(Object obj) {
+        return (obj == this
+                || (obj instanceof org.springframework.core.io.Resource
+                    && ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription())));
+    }
+
+    /**
+     * This implementation returns the description's hash code.
+     *
+     * @see #getDescription()
+     */
+    @Override
+    public int hashCode() {
+        return getDescription().hashCode();
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java

@@ -0,0 +1,117 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.core.io.Resource;
+import org.springframework.util.Assert;
+
+/**
+ * {@link Resource} implementation for a given byte array.
+ * <p>
+ * Creates a {@link ByteArrayInputStream} for the given byte array.
+ * <p>
+ * Useful for loading content from any given byte array, without having to
+ * resort to a single-use {@link InputStreamResource}. Particularly useful for
+ * creating mail attachments from local content, where JavaMail needs to be able
+ * to read the stream multiple times.
+ *
+ * @author Juergen Hoeller
+ * @author Sam Brannen
+ * @see ByteArrayInputStream
+ * @see InputStreamResource
+ * @since 1.2.3
+ */
+public class ByteArrayResource extends AbstractResource {
+
+    private final byte[] byteArray;
+
+    private final String description;
+
+    /**
+     * Create a new {@code ByteArrayResource}.
+     *
+     * @param byteArray the byte array to wrap
+     */
+    public ByteArrayResource(byte[] byteArray){
+        this(byteArray, "resource loaded from byte array");
+    }
+
+    /**
+     * Create a new {@code ByteArrayResource} with a description.
+     *
+     * @param byteArray the byte array to wrap
+     * @param description where the byte array comes from
+     */
+    public ByteArrayResource(byte[] byteArray, String description){
+        Assert.notNull(byteArray, "Byte array must not be null");
+        this.byteArray = byteArray;
+        this.description = (description != null ? description : "");
+    }
+
+    /**
+     * Return the underlying byte array.
+     */
+    public final byte[] getByteArray() {
+        return this.byteArray;
+    }
+
+    /**
+     * This implementation always returns {@code true}.
+     */
+    @Override
+    public boolean exists() {
+        return true;
+    }
+
+    /**
+     * This implementation returns the length of the underlying byte array.
+     */
+    @Override
+    public long contentLength() {
+        return this.byteArray.length;
+    }
+
+    /**
+     * This implementation returns a ByteArrayInputStream for the underlying byte
+     * array.
+     *
+     * @see ByteArrayInputStream
+     */
+    @Override
+    public InputStream getInputStream() throws IOException {
+        return new ByteArrayInputStream(this.byteArray);
+    }
+
+    /**
+     * This implementation returns a description that includes the passed-in
+     * {@code description}, if any.
+     */
+    @Override
+    public String getDescription() {
+        return "Byte array resource [" + this.description + "]";
+    }
+
+    /**
+     * This implementation compares the underlying byte array.
+     *
+     * @see Arrays#equals(byte[], byte[])
+     */
+    @Override
+    public boolean equals(Object obj) {
+        return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource
+                                && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray)));
+    }
+
+    /**
+     * This implementation returns the hash code based on the underlying byte array.
+     */
+    @Override
+    public int hashCode() {
+        return (byte[].class.hashCode() * 29 * this.byteArray.length);
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java

@@ -0,0 +1,107 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.util.*;
+
+import org.springframework.util.StringUtils;
+
+/**
+ * Composite {@link PropertySource} implementation that iterates over a set of
+ * {@link PropertySource} instances. Necessary in cases where multiple property
+ * sources share the same name, e.g. when multiple values are supplied to
+ * {@code @PropertySource}.
+ * <p>
+ * As of Spring 4.1.2, this class extends {@link EnumerablePropertySource}
+ * instead of plain {@link PropertySource}, exposing {@link #getPropertyNames()}
+ * based on the accumulated property names from all contained sources (as far as
+ * possible).
+ *
+ * @author Chris Beams
+ * @author Juergen Hoeller
+ * @author Phillip Webb
+ * @since 3.1.1
+ */
+public class CompositePropertySource extends EnumerablePropertySource<Object> {
+
+    private final Set<PropertySource<?>> propertySources = new LinkedHashSet<>();
+
+    /**
+     * Create a new {@code CompositePropertySource}.
+     *
+     * @param name the name of the property source
+     */
+    public CompositePropertySource(String name){
+        super(name);
+    }
+
+    @Override
+    public Object getProperty(String name) {
+        for (PropertySource<?> propertySource : this.propertySources) {
+            Object candidate = propertySource.getProperty(name);
+            if (candidate != null) {
+                return candidate;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public boolean containsProperty(String name) {
+        for (PropertySource<?> propertySource : this.propertySources) {
+            if (propertySource.containsProperty(name)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public String[] getPropertyNames() {
+        Set<String> names = new LinkedHashSet<>();
+        for (PropertySource<?> propertySource : this.propertySources) {
+            if (!(propertySource instanceof EnumerablePropertySource)) {
+                throw new IllegalStateException(
+                    "Failed to enumerate property names due to non-enumerable property source: " + propertySource);
+            }
+            names.addAll(Arrays.asList(((EnumerablePropertySource<?>) propertySource).getPropertyNames()));
+        }
+        return StringUtils.toStringArray(names);
+    }
+
+    /**
+     * Add the given {@link PropertySource} to the end of the chain.
+     *
+     * @param propertySource the PropertySource to add
+     */
+    public void addPropertySource(PropertySource<?> propertySource) {
+        this.propertySources.add(propertySource);
+    }
+
+    /**
+     * Add the given {@link PropertySource} to the start of the chain.
+     *
+     * @param propertySource the PropertySource to add
+     * @since 4.1
+     */
+    public void addFirstPropertySource(PropertySource<?> propertySource) {
+        List<PropertySource<?>> existing = new ArrayList<>(this.propertySources);
+        this.propertySources.clear();
+        this.propertySources.add(propertySource);
+        this.propertySources.addAll(existing);
+    }
+
+    /**
+     * Return all property sources that this composite source holds.
+     *
+     * @since 4.1.1
+     */
+    public Collection<PropertySource<?>> getPropertySources() {
+        return this.propertySources;
+    }
+
+    @Override
+    public String toString() {
+        return String
+            .format("%s [name='%s', propertySources=%s]", getClass().getSimpleName(), this.name, this.propertySources);
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java

@@ -0,0 +1,58 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import org.springframework.util.ObjectUtils;
+
+/**
+ * A {@link PropertySource} implementation capable of interrogating its
+ * underlying source object to enumerate all possible property name/value pairs.
+ * Exposes the {@link #getPropertyNames()} method to allow callers to introspect
+ * available properties without having to access the underlying source object.
+ * This also facilitates a more efficient implementation of
+ * {@link #containsProperty(String)}, in that it can call
+ * {@link #getPropertyNames()} and iterate through the returned array rather
+ * than attempting a call to {@link #getProperty(String)} which may be more
+ * expensive. Implementations may consider caching the result of
+ * {@link #getPropertyNames()} to fully exploit this performance opportunity.
+ * <p>
+ * Most framework-provided {@code PropertySource} implementations are
+ * enumerable; a counter-example would be {@code JndiPropertySource} where, due
+ * to the nature of JNDI it is not possible to determine all possible property
+ * names at any given time; rather it is only possible to try to access a
+ * property (via {@link #getProperty(String)}) in order to evaluate whether it
+ * is present or not.
+ *
+ * @author Chris Beams
+ * @author Juergen Hoeller
+ * @since 3.1
+ */
+public abstract class EnumerablePropertySource<T> extends PropertySource<T> {
+
+    public EnumerablePropertySource(String name, T source){
+        super(name, source);
+    }
+
+    protected EnumerablePropertySource(String name){
+        super(name);
+    }
+
+    /**
+     * Return whether this {@code PropertySource} contains a property with the given
+     * name.
+     * <p>
+     * This implementation checks for the presence of the given name within the
+     * {@link #getPropertyNames()} array.
+     *
+     * @param name the name of the property to find
+     */
+    @Override
+    public boolean containsProperty(String name) {
+        return ObjectUtils.containsElement(getPropertyNames(), name);
+    }
+
+    /**
+     * Return the names of all properties contained by the {@linkplain #getSource()
+     * source} object (never {@code null}).
+     */
+    public abstract String[] getPropertyNames();
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java

@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.util.Map;
+
+import org.springframework.core.env.PropertiesPropertySource;
+import org.springframework.core.env.PropertySource;
+import org.springframework.util.StringUtils;
+
+/**
+ * {@link PropertySource} that reads keys and values from a {@code Map} object.
+ *
+ * @author Chris Beams
+ * @author Juergen Hoeller
+ * @since 3.1
+ * @see PropertiesPropertySource
+ */
+public class MapPropertySource extends EnumerablePropertySource<Map<String, Object>> {
+
+    public MapPropertySource(String name, Map<String, Object> source){
+        super(name, source);
+    }
+
+    @Override
+    public Object getProperty(String name) {
+        return this.source.get(name);
+    }
+
+    @Override
+    public boolean containsProperty(String name) {
+        return this.source.containsKey(name);
+    }
+
+    @Override
+    public String[] getPropertyNames() {
+        return StringUtils.toStringArray(this.source.keySet());
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java

@@ -0,0 +1,221 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.core.env.PropertyResolver;
+import org.springframework.core.env.PropertySourcesPropertyResolver;
+
+/**
+ * Default implementation of the {@link PropertySources} interface. Allows
+ * manipulation of contained property sources and provides a constructor for
+ * copying an existing {@code PropertySources} instance.
+ * <p>
+ * Where <em>precedence</em> is mentioned in methods such as {@link #addFirst}
+ * and {@link #addLast}, this is with regard to the order in which property
+ * sources will be searched when resolving a given property with a
+ * {@link PropertyResolver}.
+ *
+ * @author Chris Beams
+ * @author Juergen Hoeller
+ * @see PropertySourcesPropertyResolver
+ * @since 3.1
+ */
+public class MutablePropertySources implements PropertySources {
+
+    private final Log                     logger;
+
+    private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<>();
+
+    /**
+     * Create a new {@link MutablePropertySources}
+     * object.
+     */
+    public MutablePropertySources(){
+        this.logger = LogFactory.getLog(getClass());
+    }
+
+    /**
+     * Create a new {@code MutablePropertySources} from the given propertySources
+     * object, preserving the original order of contained {@code PropertySource}
+     * objects.
+     */
+    public MutablePropertySources(PropertySources propertySources){
+        this();
+        for (PropertySource<?> propertySource : propertySources) {
+            addLast(propertySource);
+        }
+    }
+
+    /**
+     * Create a new {@link MutablePropertySources}
+     * object and inherit the given logger, usually from an enclosing
+     * {@link Environment}.
+     */
+    MutablePropertySources(Log logger){
+        this.logger = logger;
+    }
+
+    @Override
+    public boolean contains(String name) {
+        return this.propertySourceList.contains(PropertySource.named(name));
+    }
+
+    @Override
+    public PropertySource<?> get(String name) {
+        int index = this.propertySourceList.indexOf(PropertySource.named(name));
+        return (index != -1 ? this.propertySourceList.get(index) : null);
+    }
+
+    @Override
+    public Iterator<PropertySource<?>> iterator() {
+        return this.propertySourceList.iterator();
+    }
+
+    /**
+     * Add the given property source object with highest precedence.
+     */
+    public void addFirst(PropertySource<?> propertySource) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Adding PropertySource '" + propertySource.getName() + "' with highest search precedence");
+        }
+        removeIfPresent(propertySource);
+        this.propertySourceList.add(0, propertySource);
+    }
+
+    /**
+     * Add the given property source object with lowest precedence.
+     */
+    public void addLast(PropertySource<?> propertySource) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Adding PropertySource '" + propertySource.getName() + "' with lowest search precedence");
+        }
+        removeIfPresent(propertySource);
+        this.propertySourceList.add(propertySource);
+    }
+
+    /**
+     * Add the given property source object with precedence immediately higher than
+     * the named relative property source.
+     */
+    public void addBefore(String relativePropertySourceName, PropertySource<?> propertySource) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Adding PropertySource '" + propertySource.getName()
+                         + "' with search precedence immediately higher than '" + relativePropertySourceName + "'");
+        }
+        assertLegalRelativeAddition(relativePropertySourceName, propertySource);
+        removeIfPresent(propertySource);
+        int index = assertPresentAndGetIndex(relativePropertySourceName);
+        addAtIndex(index, propertySource);
+    }
+
+    /**
+     * Add the given property source object with precedence immediately lower than
+     * the named relative property source.
+     */
+    public void addAfter(String relativePropertySourceName, PropertySource<?> propertySource) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Adding PropertySource '" + propertySource.getName()
+                         + "' with search precedence immediately lower than '" + relativePropertySourceName + "'");
+        }
+        assertLegalRelativeAddition(relativePropertySourceName, propertySource);
+        removeIfPresent(propertySource);
+        int index = assertPresentAndGetIndex(relativePropertySourceName);
+        addAtIndex(index + 1, propertySource);
+    }
+
+    /**
+     * Return the precedence of the given property source, {@code -1} if not found.
+     */
+    public int precedenceOf(PropertySource<?> propertySource) {
+        return this.propertySourceList.indexOf(propertySource);
+    }
+
+    /**
+     * Remove and return the property source with the given name, {@code null} if
+     * not found.
+     *
+     * @param name the name of the property source to find and remove
+     */
+    public PropertySource<?> remove(String name) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Removing PropertySource '" + name + "'");
+        }
+        int index = this.propertySourceList.indexOf(PropertySource.named(name));
+        return (index != -1 ? this.propertySourceList.remove(index) : null);
+    }
+
+    /**
+     * Replace the property source with the given name with the given property
+     * source object.
+     *
+     * @param name the name of the property source to find and replace
+     * @param propertySource the replacement property source
+     * @throws IllegalArgumentException if no property source with the given name is
+     *     present
+     * @see #contains
+     */
+    public void replace(String name, PropertySource<?> propertySource) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Replacing PropertySource '" + name + "' with '" + propertySource.getName() + "'");
+        }
+        int index = assertPresentAndGetIndex(name);
+        this.propertySourceList.set(index, propertySource);
+    }
+
+    /**
+     * Return the number of {@link PropertySource} objects contained.
+     */
+    public int size() {
+        return this.propertySourceList.size();
+    }
+
+    @Override
+    public String toString() {
+        return this.propertySourceList.toString();
+    }
+
+    /**
+     * Ensure that the given property source is not being added relative to itself.
+     */
+    protected void assertLegalRelativeAddition(String relativePropertySourceName, PropertySource<?> propertySource) {
+        String newPropertySourceName = propertySource.getName();
+        if (relativePropertySourceName.equals(newPropertySourceName)) {
+            throw new IllegalArgumentException(
+                "PropertySource named '" + newPropertySourceName + "' cannot be added relative to itself");
+        }
+    }
+
+    /**
+     * Remove the given property source if it is present.
+     */
+    protected void removeIfPresent(PropertySource<?> propertySource) {
+        this.propertySourceList.remove(propertySource);
+    }
+
+    /**
+     * Add the given property source at a particular index in the list.
+     */
+    private void addAtIndex(int index, PropertySource<?> propertySource) {
+        removeIfPresent(propertySource);
+        this.propertySourceList.add(index, propertySource);
+    }
+
+    /**
+     * Assert that the named property source is present and return its index.
+     *
+     * @param name {@linkplain PropertySource#getName() name of the property source}
+     *     to find
+     * @throws IllegalArgumentException if the named property source is not present
+     */
+    private int assertPresentAndGetIndex(String name) {
+        int index = this.propertySourceList.indexOf(PropertySource.named(name));
+        if (index == -1) {
+            throw new IllegalArgumentException("PropertySource named '" + name + "' does not exist");
+        }
+        return index;
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java

@@ -0,0 +1,34 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.springframework.core.env.PropertySource;
+
+/**
+ * {@link PropertySource} implementation that extracts properties from a
+ * {@link Properties} object.
+ * <p>
+ * Note that because a {@code Properties} object is technically an
+ * {@code <Object, Object>} {@link java.util.Hashtable Hashtable}, one may
+ * contain non-{@code String} keys or values. This implementation, however is
+ * restricted to accessing only {@code String}-based keys and values, in the
+ * same fashion as {@link Properties#getProperty} and
+ * {@link Properties#setProperty}.
+ *
+ * @author Chris Beams
+ * @author Juergen Hoeller
+ * @since 3.1
+ */
+public class PropertiesPropertySource extends MapPropertySource {
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public PropertiesPropertySource(String name, Properties source){
+        super(name, (Map) source);
+    }
+
+    protected PropertiesPropertySource(String name, Map<String, Object> source){
+        super(name, source);
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java

@@ -0,0 +1,239 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.util.Assert;
+import org.springframework.util.ObjectUtils;
+
+/**
+ * Abstract base class representing a source of name/value property pairs. The
+ * underlying {@linkplain #getSource() source object} may be of any type
+ * {@code T} that encapsulates properties. Examples include
+ * {@link java.util.Properties} objects, {@link java.util.Map} objects,
+ * {@code ServletContext} and {@code ServletConfig} objects (for access to init
+ * parameters). Explore the {@code PropertySource} type hierarchy to see
+ * provided implementations.
+ * <p>
+ * {@code PropertySource} objects are not typically used in isolation, but
+ * rather through a {@link PropertySources} object, which aggregates property
+ * sources and in conjunction with a {@link PropertyResolver} implementation
+ * that can perform precedence-based searches across the set of
+ * {@code PropertySources}.
+ * <p>
+ * {@code PropertySource} identity is determined not based on the content of
+ * encapsulated properties, but rather based on the {@link #getName() name} of
+ * the {@code PropertySource} alone. This is useful for manipulating
+ * {@code PropertySource} objects when in collection contexts. See operations in
+ * {@link MutablePropertySources} as well as the {@link #named(String)} and
+ * {@link #toString()} methods for details.
+ * <p>
+ * Note that when working
+ * with @{@link org.springframework.context.annotation.Configuration
+ * Configuration} classes that the @{@link PropertySource PropertySource}
+ * annotation provides a convenient and declarative way of adding property
+ * sources to the enclosing {@code Environment}.
+ *
+ * @author Chris Beams
+ * @since 3.1
+ * @see PropertySources
+ * @see MutablePropertySources
+ * @see PropertySource
+ */
+public abstract class PropertySource<T> {
+
+    protected final Log    logger = LogFactory.getLog(getClass());
+
+    protected final String name;
+
+    protected final T      source;
+
+    /**
+     * Create a new {@code PropertySource} with the given name and source object.
+     */
+    public PropertySource(String name, T source){
+        Assert.hasText(name, "Property source name must contain at least one character");
+        Assert.notNull(source, "Property source must not be null");
+        this.name = name;
+        this.source = source;
+    }
+
+    /**
+     * Create a new {@code PropertySource} with the given name and with a new
+     * {@code Object} instance as the underlying source.
+     * <p>
+     * Often useful in testing scenarios when creating anonymous implementations
+     * that never query an actual source but rather return hard-coded values.
+     */
+    @SuppressWarnings("unchecked")
+    public PropertySource(String name){
+        this(name, (T) new Object());
+    }
+
+    /**
+     * Return the name of this {@code PropertySource}
+     */
+    public String getName() {
+        return this.name;
+    }
+
+    /**
+     * Return the underlying source object for this {@code PropertySource}.
+     */
+    public T getSource() {
+        return this.source;
+    }
+
+    /**
+     * Return whether this {@code PropertySource} contains the given name.
+     * <p>
+     * This implementation simply checks for a {@code null} return value from
+     * {@link #getProperty(String)}. Subclasses may wish to implement a more
+     * efficient algorithm if possible.
+     *
+     * @param name the property name to find
+     */
+    public boolean containsProperty(String name) {
+        return (getProperty(name) != null);
+    }
+
+    /**
+     * Return the value associated with the given name, or {@code null} if not
+     * found.
+     *
+     * @param name the property to find
+     */
+    public abstract Object getProperty(String name);
+
+    /**
+     * This {@code PropertySource} object is equal to the given object if:
+     * <ul>
+     * <li>they are the same instance
+     * <li>the {@code name} properties for both objects are equal
+     * </ul>
+     * <p>
+     * No properties other than {@code name} are evaluated.
+     */
+    @Override
+    public boolean equals(Object obj) {
+        return (this == obj || (obj instanceof PropertySource
+                                && ObjectUtils.nullSafeEquals(this.name, ((PropertySource<?>) obj).name)));
+    }
+
+    /**
+     * Return a hash code derived from the {@code name} property of this
+     * {@code PropertySource} object.
+     */
+    @Override
+    public int hashCode() {
+        return ObjectUtils.nullSafeHashCode(this.name);
+    }
+
+    /**
+     * Produce concise output (type and name) if the current log level does not
+     * include debug. If debug is enabled, produce verbose output including the hash
+     * code of the PropertySource instance and every name/value property pair.
+     * <p>
+     * This variable verbosity is useful as a property source such as system
+     * properties or environment variables may contain an arbitrary number of
+     * property pairs, potentially leading to difficult to read exception and log
+     * messages.
+     *
+     * @see Log#isDebugEnabled()
+     */
+    @Override
+    public String toString() {
+        if (logger.isDebugEnabled()) {
+            return getClass().getSimpleName() + "@" + System.identityHashCode(this) + " {name='" + this.name
+                   + "', properties=" + this.source + "}";
+        } else {
+            return getClass().getSimpleName() + " {name='" + this.name + "'}";
+        }
+    }
+
+    /**
+     * Return a {@code PropertySource} implementation intended for collection
+     * comparison purposes only.
+     * <p>
+     * Primarily for internal use, but given a collection of {@code PropertySource}
+     * objects, may be used as follows:
+     *
+     * <pre class="code">
+     *
+     * {
+     *     &#64;code
+     *     List<PropertySource<?>> sources = new ArrayList<PropertySource<?>>();
+     *     sources.add(new MapPropertySource("sourceA", mapA));
+     *     sources.add(new MapPropertySource("sourceB", mapB));
+     *     assert sources.contains(PropertySource.named("sourceA"));
+     *     assert sources.contains(PropertySource.named("sourceB"));
+     *     assert !sources.contains(PropertySource.named("sourceC"));
+     * }
+     * </pre>
+     *
+     * The returned {@code PropertySource} will throw
+     * {@code UnsupportedOperationException} if any methods other than
+     * {@code equals(Object)}, {@code hashCode()}, and {@code toString()} are
+     * called.
+     *
+     * @param name the name of the comparison {@code PropertySource} to be created
+     *     and returned.
+     */
+    public static PropertySource<?> named(String name) {
+        return new ComparisonPropertySource(name);
+    }
+
+    /**
+     * {@code PropertySource} to be used as a placeholder in cases where an actual
+     * property source cannot be eagerly initialized at application context creation
+     * time. For example, a {@code ServletContext}-based property source must wait
+     * until the {@code ServletContext} object is available to its enclosing
+     * {@code ApplicationContext}. In such cases, a stub should be used to hold the
+     * intended default position/order of the property source, then be replaced
+     * during context refresh.
+     *
+     * @see org.springframework.web.context.support.StandardServletEnvironment
+     * @see org.springframework.web.context.support.ServletContextPropertySource
+     */
+    public static class StubPropertySource extends PropertySource<Object> {
+
+        public StubPropertySource(String name){
+            super(name, new Object());
+        }
+
+        /**
+         * Always returns {@code null}.
+         */
+        @Override
+        public String getProperty(String name) {
+            return null;
+        }
+    }
+
+    /**
+     * @see PropertySource#named(String)
+     */
+    static class ComparisonPropertySource extends StubPropertySource {
+
+        private static final String USAGE_ERROR = "ComparisonPropertySource instances are for use with collection comparison only";
+
+        public ComparisonPropertySource(String name){
+            super(name);
+        }
+
+        @Override
+        public Object getSource() {
+            throw new UnsupportedOperationException(USAGE_ERROR);
+        }
+
+        @Override
+        public boolean containsProperty(String name) {
+            throw new UnsupportedOperationException(USAGE_ERROR);
+        }
+
+        @Override
+        public String getProperty(String name) {
+            throw new UnsupportedOperationException(USAGE_ERROR);
+        }
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.io.IOException;
+
+import org.springframework.core.io.support.SpringFactoriesLoader;
+
+/**
+ * Strategy interface located via {@link SpringFactoriesLoader} and used to load
+ * a {@link PropertySource}.
+ *
+ * @author Dave Syer
+ * @author Phillip Webb
+ */
+public interface PropertySourceLoader {
+
+    /**
+     * Returns the file extensions that the loader supports (excluding the '.').
+     *
+     * @return the file extensions
+     */
+    String[] getFileExtensions();
+
+    /**
+     * Load the resource into a property source.
+     *
+     * @param name the name of the property source
+     * @param resource the resource to load
+     * @param profile the name of the profile to load or {@code null}. The profile
+     *     can be used to load multi-document files (such as YAML). Simple property
+     *     formats should {@code null} when asked to load a profile.
+     * @return a property source or {@code null}
+     * @throws IOException if the source cannot be loaded
+     */
+    PropertySource<?> load(String name, Resource resource, String profile) throws IOException;
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java

@@ -0,0 +1,25 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+/**
+ * Holder containing one or more {@link PropertySource} objects.
+ *
+ * @author Chris Beams
+ * @since 3.1
+ */
+public interface PropertySources extends Iterable<PropertySource<?>> {
+
+    /**
+     * Return whether a property source with the given name is contained.
+     *
+     * @param name the {@linkplain PropertySource#getName() name of the property source} to find
+     */
+    boolean contains(String name);
+
+    /**
+     * Return the property source with the given name, {@code null} if not found.
+     *
+     * @param name the {@linkplain PropertySource#getName() name of the property source} to find
+     */
+    PropertySource<?> get(String name);
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java

@@ -0,0 +1,57 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+
+import org.springframework.core.io.*;
+import org.springframework.core.io.ByteArrayResource;
+
+/**
+ * Interface for a resource descriptor that abstracts from the actual type of
+ * underlying resource, such as a file or class path resource.
+ * <p>
+ * An InputStream can be opened for every resource if it exists in physical
+ * form, but a URL or File handle can just be returned for certain resources.
+ * The actual behavior is implementation-specific.
+ *
+ * @author Juergen Hoeller
+ * @since 28.12.2003
+ * @see #getInputStream()
+ * @see #getURL()
+ * @see #getURI()
+ * @see #getFile()
+ * @see WritableResource
+ * @see ContextResource
+ * @see UrlResource
+ * @see ClassPathResource
+ * @see FileSystemResource
+ * @see PathResource
+ * @see ByteArrayResource
+ * @see InputStreamResource
+ */
+public interface Resource extends InputStreamSource {
+
+    boolean exists();
+
+    boolean isReadable();
+
+    boolean isOpen();
+
+    URL getURL() throws IOException;
+
+    URI getURI() throws IOException;
+
+    File getFile() throws IOException;
+
+    long contentLength() throws IOException;
+
+    long lastModified() throws IOException;
+
+    org.springframework.core.io.Resource createRelative(String var1) throws IOException;
+
+    String getFilename();
+
+    String getDescription();
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java

@@ -0,0 +1,182 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.util.*;
+
+import org.springframework.core.env.Environment;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+
+/**
+ * {@link YamlProcessor.DocumentMatcher} backed by
+ * {@link Environment#getActiveProfiles()}. A YAML document may define a
+ * "spring.profiles" element as a comma-separated list of Spring profile names,
+ * optionally negated using the {@code !} character. If both negated and
+ * non-negated profiles are specified for a single document, at least one
+ * non-negated profile must match and no negated profiles may match.
+ *
+ * @author Dave Syer
+ * @author Matt Benson
+ * @author Phillip Webb
+ * @author Andy Wilkinson
+ */
+public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatcher {
+
+    private String[] activeProfiles = new String[0];
+
+    public SpringProfileDocumentMatcher(){
+    }
+
+    public SpringProfileDocumentMatcher(String... profiles){
+        addActiveProfiles(profiles);
+    }
+
+    public void addActiveProfiles(String... profiles) {
+        LinkedHashSet<String> set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles));
+        Collections.addAll(set, profiles);
+        this.activeProfiles = set.toArray(new String[set.size()]);
+    }
+
+    @Override
+    public YamlProcessor.MatchStatus matches(Properties properties) {
+        List<String> profiles = extractSpringProfiles(properties);
+        ProfilesMatcher profilesMatcher = getProfilesMatcher();
+        Set<String> negative = extractProfiles(profiles, ProfileType.NEGATIVE);
+        Set<String> positive = extractProfiles(profiles, ProfileType.POSITIVE);
+        if (!CollectionUtils.isEmpty(negative)) {
+            if (profilesMatcher.matches(negative) == YamlProcessor.MatchStatus.FOUND) {
+                return YamlProcessor.MatchStatus.NOT_FOUND;
+            }
+            if (CollectionUtils.isEmpty(positive)) {
+                return YamlProcessor.MatchStatus.FOUND;
+            }
+        }
+        return profilesMatcher.matches(positive);
+    }
+
+    private List<String> extractSpringProfiles(Properties properties) {
+        SpringProperties springProperties = new SpringProperties();
+        MutablePropertySources propertySources = new MutablePropertySources();
+        propertySources.addFirst(new PropertiesPropertySource("profiles", properties));
+        // PropertyValues propertyValues = new PropertySourcesPropertyValues(
+        // propertySources);
+        // new RelaxedDataBinder(springProperties, "spring").bind(propertyValues);
+        // TODO
+        List<String> profiles = springProperties.getProfiles();
+        return profiles;
+    }
+
+    private ProfilesMatcher getProfilesMatcher() {
+        return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher(
+                new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
+    }
+
+    private Set<String> extractProfiles(List<String> profiles, ProfileType type) {
+        if (CollectionUtils.isEmpty(profiles)) {
+            return null;
+        }
+        Set<String> extractedProfiles = new HashSet<>();
+        for (String candidate : profiles) {
+            ProfileType candidateType = ProfileType.POSITIVE;
+            if (candidate.startsWith("!")) {
+                candidateType = ProfileType.NEGATIVE;
+            }
+            if (candidateType == type) {
+                extractedProfiles.add(type != ProfileType.POSITIVE ? candidate.substring(1) : candidate);
+            }
+        }
+        return extractedProfiles;
+    }
+
+    /**
+     * Profile match types.
+     */
+    enum ProfileType {
+
+                      POSITIVE, NEGATIVE
+
+    }
+
+    /**
+     * Base class for profile matchers.
+     */
+    private abstract static class ProfilesMatcher {
+
+        public final YamlProcessor.MatchStatus matches(Set<String> profiles) {
+            if (CollectionUtils.isEmpty(profiles)) {
+                return YamlProcessor.MatchStatus.ABSTAIN;
+            }
+            return doMatches(profiles);
+        }
+
+        protected abstract YamlProcessor.MatchStatus doMatches(Set<String> profiles);
+
+    }
+
+    /**
+     * {@link ProfilesMatcher} that matches when a value in {@code spring.profiles}
+     * is also in {@code spring.profiles.active}.
+     */
+    private static class ActiveProfilesMatcher extends ProfilesMatcher {
+
+        private final Set<String> activeProfiles;
+
+        ActiveProfilesMatcher(Set<String> activeProfiles){
+            this.activeProfiles = activeProfiles;
+        }
+
+        @Override
+        protected YamlProcessor.MatchStatus doMatches(Set<String> profiles) {
+            if (profiles.isEmpty()) {
+                return YamlProcessor.MatchStatus.NOT_FOUND;
+            }
+            for (String activeProfile : this.activeProfiles) {
+                if (profiles.contains(activeProfile)) {
+                    return YamlProcessor.MatchStatus.FOUND;
+                }
+            }
+            return YamlProcessor.MatchStatus.NOT_FOUND;
+        }
+
+    }
+
+    /**
+     * {@link ProfilesMatcher} that matches when {@code
+     * spring.profiles} is empty or contains a value with no text.
+     *
+     * @see StringUtils#hasText(String)
+     */
+    private static class EmptyProfilesMatcher extends ProfilesMatcher {
+
+        @Override
+        public YamlProcessor.MatchStatus doMatches(Set<String> springProfiles) {
+            if (springProfiles.isEmpty()) {
+                return YamlProcessor.MatchStatus.FOUND;
+            }
+            for (String profile : springProfiles) {
+                if (!StringUtils.hasText(profile)) {
+                    return YamlProcessor.MatchStatus.FOUND;
+                }
+            }
+            return YamlProcessor.MatchStatus.NOT_FOUND;
+        }
+
+    }
+
+    /**
+     * Class for binding {@code spring.profiles} property.
+     */
+    static class SpringProperties {
+
+        private List<String> profiles = new ArrayList<>();
+
+        public List<String> getProfiles() {
+            return this.profiles;
+        }
+
+        public void setProfiles(List<String> profiles) {
+            this.profiles = profiles;
+        }
+
+    }
+
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java

@@ -0,0 +1,417 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.util.Assert;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+import org.yaml.snakeyaml.nodes.MappingNode;
+import org.yaml.snakeyaml.parser.ParserException;
+import org.yaml.snakeyaml.reader.UnicodeReader;
+
+/**
+ * Base class for YAML factories.
+ *
+ * @author Dave Syer
+ * @author Juergen Hoeller
+ * @since 4.1
+ */
+public abstract class YamlProcessor {
+
+    private final Log             logger           = LogFactory.getLog(getClass());
+
+    private ResolutionMethod      resolutionMethod = ResolutionMethod.OVERRIDE;
+
+    private Resource[]            resources        = new Resource[0];
+
+    private List<DocumentMatcher> documentMatchers = Collections.emptyList();
+
+    private boolean               matchDefault     = true;
+
+    /**
+     * A map of document matchers allowing callers to selectively use only some of
+     * the documents in a YAML resource. In YAML documents are separated by
+     * <code>---<code> lines, and each document is converted to properties before
+     * the match is made. E.g.
+     *
+     * <pre class="code">
+     * environment: dev
+     * url: http://dev.bar.com
+     * name: Developer Setup
+     * ---
+     * environment: prod
+     * url:http://foo.bar.com
+     * name: My Cool App
+     * </pre>
+     *
+     * when mapped with
+     *
+     * <pre class="code">
+     * setDocumentMatchers(properties -> ("prod"
+     *     .equals(properties.getProperty("environment")) ? MatchStatus.FOUND : MatchStatus.NOT_FOUND));
+     * </pre>
+     *
+     * would end up as
+     *
+     * <pre class="code">
+     * environment=prod
+     * url=http://foo.bar.com
+     * name=My Cool App
+     * </pre>
+     */
+    public void setDocumentMatchers(DocumentMatcher... matchers) {
+        this.documentMatchers = Arrays.asList(matchers);
+    }
+
+    /**
+     * Flag indicating that a document for which all the
+     * {@link #setDocumentMatchers(DocumentMatcher...) document matchers} abstain
+     * will nevertheless match. Default is {@code true}.
+     */
+    public void setMatchDefault(boolean matchDefault) {
+        this.matchDefault = matchDefault;
+    }
+
+    /**
+     * Method to use for resolving resources. Each resource will be converted to a
+     * Map, so this property is used to decide which map entries to keep in the
+     * final output from this factory. Default is {@link ResolutionMethod#OVERRIDE}.
+     */
+    public void setResolutionMethod(ResolutionMethod resolutionMethod) {
+        Assert.notNull(resolutionMethod, "ResolutionMethod must not be null");
+        this.resolutionMethod = resolutionMethod;
+    }
+
+    /**
+     * Set locations of YAML {@link Resource resources} to be loaded.
+     *
+     * @see ResolutionMethod
+     */
+    public void setResources(Resource... resources) {
+        this.resources = resources;
+    }
+
+    /**
+     * Provide an opportunity for subclasses to process the Yaml parsed from the
+     * supplied resources. Each resource is parsed in turn and the documents inside
+     * checked against the {@link #setDocumentMatchers(DocumentMatcher...)
+     * matchers}. If a document matches it is passed into the callback, along with
+     * its representation as Properties. Depending on the
+     * {@link #setResolutionMethod(ResolutionMethod)} not all of the documents will
+     * be parsed.
+     *
+     * @param callback a callback to delegate to once matching documents are found
+     * @see #createYaml()
+     */
+    protected void process(MatchCallback callback) {
+        Yaml yaml = createYaml();
+        for (Resource resource : this.resources) {
+            boolean found = process(callback, yaml, resource);
+            if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND && found) {
+                return;
+            }
+        }
+    }
+
+    /**
+     * Create the {@link Yaml} instance to use.
+     */
+    protected Yaml createYaml() {
+        return new Yaml(new StrictMapAppenderConstructor());
+    }
+
+    private boolean process(MatchCallback callback, Yaml yaml, Resource resource) {
+        int count = 0;
+        try {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Loading from YAML: " + resource);
+            }
+            try (Reader reader = new UnicodeReader(resource.getInputStream())) {
+                for (Object object : yaml.loadAll(reader)) {
+                    if (object != null && process(asMap(object), callback)) {
+                        count++;
+                        if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND) {
+                            break;
+                        }
+                    }
+                }
+                if (logger.isDebugEnabled()) {
+                    logger.debug(
+                            "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
+                }
+            }
+        } catch (IOException ex) {
+            handleProcessError(resource, ex);
+        }
+        return (count > 0);
+    }
+
+    private void handleProcessError(Resource resource, IOException ex) {
+        if (this.resolutionMethod != ResolutionMethod.FIRST_FOUND
+            && this.resolutionMethod != ResolutionMethod.OVERRIDE_AND_IGNORE) {
+            throw new IllegalStateException(ex);
+        }
+        if (logger.isWarnEnabled()) {
+            logger.warn("Could not load map from " + resource + ": " + ex.getMessage());
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> asMap(Object object) {
+        // YAML can have numbers as keys
+        Map<String, Object> result = new LinkedHashMap<>();
+        if (!(object instanceof Map)) {
+            // A document can be a text literal
+            result.put("document", object);
+            return result;
+        }
+
+        Map<Object, Object> map = (Map<Object, Object>) object;
+        for (Map.Entry<Object, Object> entry : map.entrySet()) {
+            Object value = entry.getValue();
+            if (value instanceof Map) {
+                value = asMap(value);
+            }
+            Object key = entry.getKey();
+            if (key instanceof CharSequence) {
+                result.put(key.toString(), value);
+            } else {
+                // It has to be a map key in this case
+                result.put("[" + key.toString() + "]", value);
+            }
+        }
+        return result;
+    }
+
+    private boolean process(Map<String, Object> map, MatchCallback callback) {
+        @SuppressWarnings("serial")
+        Properties properties = new Properties() {
+
+            @Override
+            public String getProperty(String key) {
+                Object value = get(key);
+                return (value != null ? value.toString() : null);
+            }
+        };
+        properties.putAll(getFlattenedMap(map));
+
+        if (this.documentMatchers.isEmpty()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Merging document (no matchers set): " + map);
+            }
+            callback.process(properties, map);
+            return true;
+        }
+
+        MatchStatus result = MatchStatus.ABSTAIN;
+        for (DocumentMatcher matcher : this.documentMatchers) {
+            MatchStatus match = matcher.matches(properties);
+            result = MatchStatus.getMostSpecific(match, result);
+            if (match == MatchStatus.FOUND) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Matched document with document matcher: " + properties);
+                }
+                callback.process(properties, map);
+                return true;
+            }
+        }
+
+        if (result == MatchStatus.ABSTAIN && this.matchDefault) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Matched document with default matcher: " + map);
+            }
+            callback.process(properties, map);
+            return true;
+        }
+
+        if (logger.isDebugEnabled()) {
+            logger.debug("Unmatched document: " + map);
+        }
+        return false;
+    }
+
+    /**
+     * Return a flattened version of the given map, recursively following any nested
+     * Map or Collection values. Entries from the resulting map retain the same
+     * order as the source. When called with the Map from a {@link MatchCallback}
+     * the result will contain the same values as the {@link MatchCallback}
+     * Properties.
+     *
+     * @param source the source map
+     * @return a flattened map
+     * @since 4.1.3
+     */
+    protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) {
+        Map<String, Object> result = new LinkedHashMap<>();
+        buildFlattenedMap(result, source, null);
+        return result;
+    }
+
+    private static boolean containsText(CharSequence str) {
+        int strLen = str.length();
+        for (int i = 0; i < strLen; i++) {
+            if (!Character.isWhitespace(str.charAt(i))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private void buildFlattenedMap(Map<String, Object> result, Map<String, Object> source, String path) {
+        for (Map.Entry<String, Object> entry : source.entrySet()) {
+            String key = entry.getKey();
+            if (path != null && !path.isEmpty() && containsText(path)) {
+                if (key.startsWith("[")) {
+                    key = path + key;
+                } else {
+                    key = path + '.' + key;
+                }
+            }
+            Object value = entry.getValue();
+            if (value instanceof String) {
+                result.put(key, value);
+            } else if (value instanceof Map) {
+                // Need a compound key
+                @SuppressWarnings("unchecked")
+                Map<String, Object> map = (Map<String, Object>) value;
+                buildFlattenedMap(result, map, key);
+            } else if (value instanceof Collection) {
+                // Need a compound key
+                @SuppressWarnings("unchecked")
+                Collection<Object> collection = (Collection<Object>) value;
+                int count = 0;
+                for (Object object : collection) {
+                    buildFlattenedMap(result, Collections.singletonMap("[" + (count++) + "]", object), key);
+                }
+            } else {
+                result.put(key, (value != null ? value : ""));
+            }
+        }
+    }
+
+    /**
+     * Callback interface used to process the YAML parsing results.
+     */
+    public interface MatchCallback {
+
+        /**
+         * Process the given representation of the parsing results.
+         *
+         * @param properties the properties to process (as a flattened representation
+         *     with indexed keys in case of a collection or map)
+         * @param map the result map (preserving the original value structure in the
+         *     YAML document)
+         */
+        void process(Properties properties, Map<String, Object> map);
+    }
+
+    /**
+     * Strategy interface used to test if properties match.
+     */
+    public interface DocumentMatcher {
+
+        /**
+         * Test if the given properties match.
+         *
+         * @param properties the properties to test
+         * @return the status of the match
+         */
+        MatchStatus matches(Properties properties);
+    }
+
+    /**
+     * Status returned from {@link DocumentMatcher#matches(Properties)}
+     */
+    public enum MatchStatus {
+
+                             /**
+                              * A match was found.
+                              */
+                             FOUND,
+
+                             /**
+                              * No match was found.
+                              */
+                             NOT_FOUND,
+
+                             /**
+                              * The matcher should not be considered.
+                              */
+                             ABSTAIN;
+
+        /**
+         * Compare two {@link MatchStatus} items, returning the most specific status.
+         */
+        public static MatchStatus getMostSpecific(MatchStatus a, MatchStatus b) {
+            return (a.ordinal() < b.ordinal() ? a : b);
+        }
+    }
+
+    /**
+     * Method to use for resolving resources.
+     */
+    public enum ResolutionMethod {
+
+                                  /**
+                                   * Replace values from earlier in the list.
+                                   */
+                                  OVERRIDE,
+
+                                  /**
+                                   * Replace values from earlier in the list, ignoring any failures.
+                                   */
+                                  OVERRIDE_AND_IGNORE,
+
+                                  /**
+                                   * Take the first resource in the list that exists and use just that.
+                                   */
+                                  FIRST_FOUND
+    }
+
+    /**
+     * A specialized {@link Constructor} that checks for duplicate keys.
+     */
+    protected static class StrictMapAppenderConstructor extends Constructor {
+
+        // Declared as public for use in subclasses
+        public StrictMapAppenderConstructor(){
+            super();
+        }
+
+        @Override
+        protected Map<Object, Object> constructMapping(MappingNode node) {
+            try {
+                return super.constructMapping(node);
+            } catch (IllegalStateException ex) {
+                throw new ParserException("while parsing MappingNode",
+                    node.getStartMark(),
+                    ex.getMessage(),
+                    node.getEndMark());
+            }
+        }
+
+        @Override
+        protected Map<Object, Object> createDefaultMap() {
+            final Map<Object, Object> delegate = super.createDefaultMap();
+            return new AbstractMap<Object, Object>() {
+
+                @Override
+                public Object put(Object key, Object value) {
+                    if (delegate.containsKey(key)) {
+                        throw new IllegalStateException("Duplicate key: " + key);
+                    }
+                    return delegate.put(key, value);
+                }
+
+                @Override
+                public Set<Entry<Object, Object>> entrySet() {
+                    return delegate.entrySet();
+                }
+            };
+        }
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java

@@ -0,0 +1,80 @@
+package com.alibaba.otter.canal.client.adapter.config.common;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.springframework.util.ClassUtils;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.nodes.Tag;
+import org.yaml.snakeyaml.representer.Representer;
+import org.yaml.snakeyaml.resolver.Resolver;
+
+/**
+ * Strategy to load '.yml' (or '.yaml') files into a {@link PropertySource}.
+ *
+ * @author Dave Syer
+ * @author Phillip Webb
+ * @author Andy Wilkinson
+ */
+public class YamlPropertySourceLoader implements PropertySourceLoader {
+
+    @Override
+    public String[] getFileExtensions() {
+        return new String[] { "yml", "yaml" };
+    }
+
+    @Override
+    public PropertySource<?> load(String name, Resource resource, String profile) throws IOException {
+        if (ClassUtils.isPresent("org.yaml.snakeyaml.Yaml", null)) {
+            Processor processor = new Processor(resource, profile);
+            Map<String, Object> source = processor.process();
+            if (!source.isEmpty()) {
+                return new MapPropertySource(name, source);
+            }
+        }
+        return null;
+    }
+
+    /**
+     * {@link YamlProcessor} to create a {@link Map} containing the property values.
+     * Similar to {@link YamlPropertiesFactoryBean} but retains the order of
+     * entries.
+     */
+    private static class Processor extends YamlProcessor {
+
+        Processor(Resource resource, String profile){
+            if (profile == null) {
+                setMatchDefault(true);
+                setDocumentMatchers(new SpringProfileDocumentMatcher());
+            } else {
+                setMatchDefault(false);
+                setDocumentMatchers(new SpringProfileDocumentMatcher(profile));
+            }
+            setResources(resource);
+        }
+
+        @Override
+        protected Yaml createYaml() {
+            return new Yaml(new StrictMapAppenderConstructor(), new Representer(), new DumperOptions(), new Resolver() {
+
+                @Override
+                public void addImplicitResolver(Tag tag, Pattern regexp, String first) {
+                    if (tag == Tag.TIMESTAMP) {
+                        return;
+                    }
+                    super.addImplicitResolver(tag, regexp, first);
+                }
+            });
+        }
+
+        public Map<String, Object> process() {
+            final Map<String, Object> result = new LinkedHashMap<>();
+            process((properties, map) -> result.putAll(getFlattenedMap(map)));
+            return result;
+        }
+
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java

@@ -28,9 +28,9 @@ import com.google.common.cache.LoadingCache;
 
 public class Util {
 
-    private static final Logger logger                = LoggerFactory.getLogger(Util.class);
+    private static final Logger logger = LoggerFactory.getLogger(Util.class);
 
-    public static final String  AUTO_GENERATED_PREFIX = "AUTO_GENERATED_";
+    public static final String AUTO_GENERATED_PREFIX = "AUTO_GENERATED_";
 
     /**
      * 通过DS执行sql
@@ -165,38 +165,39 @@ public class Util {
 
     public static ThreadPoolExecutor newFixedDaemonThreadPool(int nThreads, long keepAliveTime) {
         return new ThreadPoolExecutor(nThreads,
-            nThreads,
+                nThreads,
-            keepAliveTime,
+                keepAliveTime,
-            TimeUnit.MILLISECONDS,
+                TimeUnit.MILLISECONDS,
-            new SynchronousQueue<>(),
+                new SynchronousQueue<>(),
-            DaemonThreadFactory.daemonThreadFactory,
+                DaemonThreadFactory.daemonThreadFactory,
-            (r, exe) -> {
+                (r, exe) -> {
-                if (!exe.isShutdown()) {
+                    if (!exe.isShutdown()) {
-                    try {
+                        try {
-                        exe.getQueue().put(r);
+                            exe.getQueue().put(r);
-                    } catch (InterruptedException e) {
+                        } catch (InterruptedException e) {
-                        // ignore
+                            // ignore
+                        }
                     }
                 }
-            });
+        );
     }
 
     public static ThreadPoolExecutor newSingleDaemonThreadExecutor(long keepAliveTime) {
         return new ThreadPoolExecutor(1,
-            1,
+                1,
-            keepAliveTime,
+                keepAliveTime,
-            TimeUnit.MILLISECONDS,
+                TimeUnit.MILLISECONDS,
-            new SynchronousQueue<>(),
+                new SynchronousQueue<>(),
-            DaemonThreadFactory.daemonThreadFactory,
+                DaemonThreadFactory.daemonThreadFactory,
-            (r, exe) -> {
+                (r, exe) -> {
-                if (!exe.isShutdown()) {
+                    if (!exe.isShutdown()) {
-                    try {
+                        try {
-                        exe.getQueue().put(r);
+                            exe.getQueue().put(r);
-                    } catch (InterruptedException e) {
+                        } catch (InterruptedException e) {
-                        // ignore
+                            // ignore
+                        }
                     }
-                }
+                });
-            });
     }
 
     public final static String  timeZone;    // 当前时区
@@ -326,56 +327,4 @@ public class Util {
 
         return null;
     }
-
-    /**
-     * Check if the time string has millisecond or microsecond units
-     * 
-     * @param timeStr time string
-     * @return boolean
-     */
-    public static boolean isAccuracyOverSecond(String timeStr) {
-        if (StringUtils.isEmpty(timeStr)) {
-            return false;
-        }
-        String[] times = StringUtils.split(timeStr, ".");
-        return times.length > 1 && !times[times.length - 1].isEmpty();
-    }
-
-    /**
-     * Check if the datetime string has microsecond or nanosecond units
-     * 
-     * @param datetimeStr datetime string
-     * @return boolean
-     */
-    public static boolean isAccuracyOverMillisecond(String datetimeStr) {
-        if (StringUtils.isEmpty(datetimeStr)) {
-            return false;
-        }
-        String[] times = StringUtils.split(datetimeStr, ".");
-        return times.length > 1 && times[times.length - 1].length() > 3;
-    }
-
-    /**
-     * MySQL has fractional seconds support for TIME, DATETIME, and TIMESTAMP
-     * values, with up to microseconds (6 digits) precision. ISO-8601 standard
-     * format is with up to nanoseconds (9 digits) precision, which is sufficient
-     * for storing MySQL time-related data.
-     * 
-     * @param datetimeStr datetime string
-     * @return LocalDateTime
-     */
-    public static LocalDateTime parseISOLocalDateTime(String datetimeStr) {
-        if (StringUtils.isEmpty(datetimeStr)) {
-            return null;
-        }
-        datetimeStr = datetimeStr.trim();
-        try {
-            // Replace SPACE in middle with `T` to comply with ISO-8601 standard format
-            return LocalDateTime.parse(datetimeStr.replace(" ", "T"));
-        } catch (Exception e) {
-            logger.error("Convert datetime string to ISOLocalDateTime fail:{}", datetimeStr, e);
-        }
-        return null;
-    }
-
 }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/YamlUtils.java

@@ -1,105 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.support;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.lang.StringUtils;
-import org.springframework.boot.context.properties.bind.Bindable;
-import org.springframework.boot.context.properties.bind.Binder;
-import org.springframework.boot.context.properties.source.ConfigurationPropertySource;
-import org.springframework.boot.context.properties.source.MapConfigurationPropertySource;
-import org.springframework.boot.env.YamlPropertySourceLoader;
-import org.springframework.boot.origin.OriginTrackedValue;
-import org.springframework.core.env.PropertySource;
-import org.springframework.core.io.ByteArrayResource;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.Resource;
-import org.springframework.util.PropertyPlaceholderHelper;
-
-public class YamlUtils {
-
-    public static <T> T resourceYmlToObj(String resource, String prefix, Class<T> clazz) {
-        ClassPathResource classPathResource = new ClassPathResource(resource);
-
-        String content;
-        try (InputStream inputStream = classPathResource.getInputStream()) {
-
-            ByteArrayOutputStream result = new ByteArrayOutputStream();
-            byte[] buffer = new byte[1024];
-            int length;
-            while ((length = inputStream.read(buffer)) != -1) {
-                result.write(buffer, 0, length);
-            }
-            content = result.toString("UTF-8");
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return ymlToObj(prefix, content, clazz);
-    }
-
-    public static <T> T ymlToObj(String content, Class<T> clazz) {
-        return ymlToObj("", content, clazz, null, null);
-    }
-
-    public static <T> T ymlToObj(String prefix, String content, Class<T> clazz) {
-        return ymlToObj(prefix, content, clazz, null, null);
-    }
-
-    public static <T> T ymlToObj(String prefix, String content, Class<T> clazz, String charset) {
-        return ymlToObj(prefix, content, clazz, charset, null);
-    }
-
-    public static <T> T ymlToObj(String prefix, String content, Class<T> clazz, String charset,
-                                 Properties baseProperties) {
-        try {
-            prefix = StringUtils.trimToEmpty(prefix);
-            byte[] contentBytes;
-            if (charset == null) {
-                contentBytes = content.getBytes(StandardCharsets.UTF_8);
-            } else {
-                contentBytes = content.getBytes(charset);
-            }
-            YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
-            Resource configResource = new ByteArrayResource(contentBytes);
-            List<PropertySource<?>> propertySources = propertySourceLoader.load("manualBindConfig", configResource);
-
-            if (propertySources == null || propertySources.isEmpty()) {
-                return null;
-            }
-
-            PropertySource<?> propertySource = propertySources.get(0);
-
-            Properties properties = new Properties();
-            if (baseProperties != null) {
-                properties.putAll(baseProperties);
-            }
-
-            properties.putAll((Map<?, ?>) propertySource.getSource());
-
-            for (Map.Entry<Object, Object> entry : properties.entrySet()) {
-                Object value = entry.getValue();
-                if (value instanceof OriginTrackedValue) {
-                    entry.setValue(((OriginTrackedValue) value).getValue());
-                }
-            }
-
-            ConfigurationPropertySource sources = new MapConfigurationPropertySource(properties);
-            PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}");
-            Binder binder = new Binder(Arrays.asList(sources), value -> {
-                if (value instanceof String) {
-                    return propertyPlaceholderHelper.replacePlaceholders((String) value, baseProperties);
-                }
-                return value;
-            });
-            return binder.bind(prefix, Bindable.of(clazz)).get();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-}

client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/YamlUtilsTest.java

@@ -1,245 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.support;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Value;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-
-public class YamlUtilsTest {
-
-
-
-    @Test
-    public void testLoadConfigToYml() {
-        String configStr="dataSourceKey: defaultDS\n"
-            + "destination: example\n"
-            + "groupId: g1\n"
-            + "outerAdapterKey: mysql1\n"
-            + "concurrent: true\n"
-            + "dbMapping:\n"
-            + "  _id: _id\n"
-            + "  database: mytest\n"
-            + "  table: user\n"
-            + "  targetTable: mytest2.user\n"
-            + "  targetPk:\n"
-            + "    id: id\n"
-            + "#  mapAll: true\n"
-            + "  targetColumns:\n"
-            + "    id:\n"
-            + "    name:\n"
-            + "    role_id:\n"
-            + "    c_time:\n"
-            + "    test1:\n"
-            + "  etlCondition: \"where c_time>={}\"\n"
-            + "  commitBatch: 3000 # 批量提交的大小";
-
-        MappingConfig config = YamlUtils.ymlToObj(null, configStr, MappingConfig.class, null, new Properties());
-
-        Assert.assertNotNull(config);
-        Assert.assertEquals(config.getDbMapping().getId(), "_id");
-        Assert.assertEquals(config.getDestination(), "example");
-        Assert.assertEquals(config.getOuterAdapterKey(), "mysql1");
-        Assert.assertEquals(config.getDbMapping().getDatabase(), "mytest");
-        Assert.assertEquals(config.getDbMapping().getTargetColumns().size(), 5);
-    }
-
-    private static class MappingConfig {
-        private String    dataSourceKey;
-
-        private String    destination;
-
-        private String    groupId;
-
-        private String    outerAdapterKey;
-
-        private boolean   concurrent = false;
-
-        private DbMapping dbMapping;
-
-        public String getDataSourceKey() {
-            return dataSourceKey;
-        }
-
-        public void setDataSourceKey(String dataSourceKey) {
-            this.dataSourceKey = dataSourceKey;
-        }
-
-        public String getDestination() {
-            return destination;
-        }
-
-        public void setDestination(String destination) {
-            this.destination = destination;
-        }
-
-        public String getGroupId() {
-            return groupId;
-        }
-
-        public void setGroupId(String groupId) {
-            this.groupId = groupId;
-        }
-
-        public String getOuterAdapterKey() {
-            return outerAdapterKey;
-        }
-
-        public void setOuterAdapterKey(String outerAdapterKey) {
-            this.outerAdapterKey = outerAdapterKey;
-        }
-
-        public boolean isConcurrent() {
-            return concurrent;
-        }
-
-        public void setConcurrent(boolean concurrent) {
-            this.concurrent = concurrent;
-        }
-
-        public DbMapping getDbMapping() {
-            return dbMapping;
-        }
-
-        public void setDbMapping(DbMapping dbMapping) {
-            this.dbMapping = dbMapping;
-        }
-    }
-
-    private static class DbMapping {
-
-        @Value("${_id}")
-        private String id ;
-        private boolean             mirrorDb        = false;                 // 是否镜像库
-        private String              database;                                // 数据库名或schema名
-        private String              table;                                   // 表名
-        private Map<String, String> targetPk        = new LinkedHashMap<>(); // 目标表主键字段
-        private boolean             mapAll          = false;                 // 映射所有字段
-        private String              targetDb;                                // 目标库名
-        private String              targetTable;                             // 目标表名
-        private Map<String, String> targetColumns;                           // 目标表字段映射
-
-        private boolean             caseInsensitive = false;                 // 目标表不区分大小写，默认是否
-
-        private String              etlCondition;                            // etl条件sql
-
-        private int                 readBatch       = 5000;
-        private int                 commitBatch     = 5000;                  // etl等批量提交大小
-
-        private Map<String, String> allMapColumns;
-
-        public boolean isMirrorDb() {
-            return mirrorDb;
-        }
-
-        public void setMirrorDb(boolean mirrorDb) {
-            this.mirrorDb = mirrorDb;
-        }
-
-        public String getDatabase() {
-            return database;
-        }
-
-        public void setDatabase(String database) {
-            this.database = database;
-        }
-
-        public String getTable() {
-            return table;
-        }
-
-        public void setTable(String table) {
-            this.table = table;
-        }
-
-        public Map<String, String> getTargetPk() {
-            return targetPk;
-        }
-
-        public void setTargetPk(Map<String, String> targetPk) {
-            this.targetPk = targetPk;
-        }
-
-        public boolean isMapAll() {
-            return mapAll;
-        }
-
-        public void setMapAll(boolean mapAll) {
-            this.mapAll = mapAll;
-        }
-
-        public String getTargetDb() {
-            return targetDb;
-        }
-
-        public void setTargetDb(String targetDb) {
-            this.targetDb = targetDb;
-        }
-
-        public String getTargetTable() {
-            return targetTable;
-        }
-
-        public void setTargetTable(String targetTable) {
-            this.targetTable = targetTable;
-        }
-
-        public Map<String, String> getTargetColumns() {
-            return targetColumns;
-        }
-
-        public void setTargetColumns(Map<String, String> targetColumns) {
-            this.targetColumns = targetColumns;
-        }
-
-        public boolean isCaseInsensitive() {
-            return caseInsensitive;
-        }
-
-        public void setCaseInsensitive(boolean caseInsensitive) {
-            this.caseInsensitive = caseInsensitive;
-        }
-
-        public String getEtlCondition() {
-            return etlCondition;
-        }
-
-        public void setEtlCondition(String etlCondition) {
-            this.etlCondition = etlCondition;
-        }
-
-        public int getReadBatch() {
-            return readBatch;
-        }
-
-        public void setReadBatch(int readBatch) {
-            this.readBatch = readBatch;
-        }
-
-        public int getCommitBatch() {
-            return commitBatch;
-        }
-
-        public void setCommitBatch(int commitBatch) {
-            this.commitBatch = commitBatch;
-        }
-
-        public Map<String, String> getAllMapColumns() {
-            return allMapColumns;
-        }
-
-        public void setAllMapColumns(Map<String, String> allMapColumns) {
-            this.allMapColumns = allMapColumns;
-        }
-
-        public String getId() {
-            return id;
-        }
-
-        public void setId(String id) {
-            this.id = id;
-        }
-    }
-}

client-adapter/es6x/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java

@@ -59,12 +59,12 @@ public class ES6xAdapter extends ESAdapter {
     public Map<String, Object> count(String task) {
         ESSyncConfig config = esSyncConfig.get(task);
         ESSyncConfig.ESMapping mapping = config.getEsMapping();
-        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.getIndex(), mapping.getType()).size(0)
+        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).size(0)
             .getResponse();
 
         long rowCount = response.getHits().getTotalHits();
         Map<String, Object> res = new LinkedHashMap<>();
-        res.put("esIndex", mapping.getIndex());
+        res.put("esIndex", mapping.get_index());
         res.put("count", rowCount);
         return res;
     }

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java

@@ -51,7 +51,7 @@ public class ESEtlService extends AbstractEtlService {
 
     public EtlResult importData(List<String> params) {
         ESMapping mapping = config.getEsMapping();
-        logger.info("start etl to import data to index: {}", mapping.getIndex());
+        logger.info("start etl to import data to index: {}", mapping.get_index());
         String sql = mapping.getSql();
         return importData(sql, params);
     }
@@ -78,7 +78,7 @@ public class ESEtlService extends AbstractEtlService {
                             }
 
                             // 如果是主键字段则不插入
-                if (fieldItem.getFieldName().equals(mapping.getId())) {
+                if (fieldItem.getFieldName().equals(mapping.get_id())) {
                     idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
                 } else {
                     Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
@@ -117,8 +117,8 @@ public class ESEtlService extends AbstractEtlService {
             if (idVal != null) {
                 String parentVal = (String) esFieldData.remove("$parent_routing");
                 if (mapping.isUpsert()) {
-                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                        mapping.getType(),
+                        mapping.get_type(),
                         idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
 
                     if (StringUtils.isNotEmpty(parentVal)) {
@@ -127,8 +127,8 @@ public class ESEtlService extends AbstractEtlService {
 
                     esBulkRequest.add(esUpdateRequest);
                 } else {
-                    ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.getIndex(),
+                    ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(),
-                        mapping.getType(),
+                        mapping.get_type(),
                         idVal.toString()).setSource(esFieldData);
                     if (StringUtils.isNotEmpty(parentVal)) {
                         esIndexRequest.setRouting(parentVal);
@@ -137,12 +137,12 @@ public class ESEtlService extends AbstractEtlService {
                 }
             } else {
                 idVal = esFieldData.get(mapping.getPk());
-                ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
+                ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                    mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
+                    mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
                 SearchResponse response = esSearchRequest.getResponse();
                 for (SearchHit hit : response.getHits()) {
-                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                        mapping.getType(),
+                        mapping.get_type(),
                         hit.getId()).setDoc(esFieldData);
                     esBulkRequest.add(esUpdateRequest);
                 }
@@ -160,7 +160,7 @@ public class ESEtlService extends AbstractEtlService {
                         (System.currentTimeMillis() - batchBegin),
                         (System.currentTimeMillis() - esBatchBegin),
                         esBulkRequest.numberOfActions(),
-                        mapping.getIndex());
+                        mapping.get_index());
                 }
                 batchBegin = System.currentTimeMillis();
                 esBulkRequest.resetBulk();
@@ -180,12 +180,12 @@ public class ESEtlService extends AbstractEtlService {
                     (System.currentTimeMillis() - batchBegin),
                     (System.currentTimeMillis() - esBatchBegin),
                     esBulkRequest.numberOfActions(),
-                    mapping.getIndex());
+                    mapping.get_index());
             }
         }
     } catch (Exception e) {
         logger.error(e.getMessage(), e);
-        errMsg.add(mapping.getIndex() + " etl failed! ==>" + e.getMessage());
+        errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
         throw new RuntimeException(e);
     }
     return count;

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java

@@ -66,19 +66,19 @@ public class ES6xTemplate implements ESTemplate {
 
     @Override
     public void insert(ESSyncConfig.ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                ESUpdateRequest updateRequest = esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest updateRequest = esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     updateRequest.setRouting(parentVal);
                 }
                 getBulk().add(updateRequest);
             } else {
-                ESIndexRequest indexRequest = esConnection.new ES6xIndexRequest(mapping.getIndex(),
+                ESIndexRequest indexRequest = esConnection.new ES6xIndexRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     pkVal.toString()).setSource(esFieldData);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     indexRequest.setRouting(parentVal);
@@ -87,13 +87,13 @@ public class ES6xTemplate implements ESTemplate {
             }
             commitBulk();
         } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
+                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
             SearchResponse response = esSearchRequest.getResponse();
 
             for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     hit.getId()).setDoc(esFieldData);
                 getBulk().add(esUpdateRequest);
                 commitBulk();
@@ -150,19 +150,19 @@ public class ES6xTemplate implements ESTemplate {
 
     @Override
     public void delete(ESSyncConfig.ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
-            ESDeleteRequest esDeleteRequest = this.esConnection.new ES6xDeleteRequest(mapping.getIndex(),
+            ESDeleteRequest esDeleteRequest = this.esConnection.new ES6xDeleteRequest(mapping.get_index(),
-                mapping.getType(),
+                mapping.get_type(),
                 pkVal.toString());
             getBulk().add(esDeleteRequest);
             commitBulk();
         } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
+                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
             SearchResponse response = esSearchRequest.getResponse();
             for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     hit.getId()).setDoc(esFieldData);
                 getBulk().add(esUpdateRequest);
                 commitBulk();
@@ -207,7 +207,7 @@ public class ES6xTemplate implements ESTemplate {
     public Object getESDataFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet,
                                   Map<String, Object> esFieldData) throws SQLException {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
@@ -216,7 +216,7 @@ public class ES6xTemplate implements ESTemplate {
                 resultIdVal = value;
             }
 
-            if (!fieldItem.getFieldName().equals(mapping.getId())
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
                     && !mapping.getSkips().contains(fieldItem.getFieldName())) {
                 esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
             }
@@ -231,7 +231,7 @@ public class ES6xTemplate implements ESTemplate {
     @Override
     public Object getIdValFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet) throws SQLException {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
@@ -248,7 +248,7 @@ public class ES6xTemplate implements ESTemplate {
     public Object getESDataFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
                                   Map<String, Object> esFieldData) throws SQLException {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             if (fieldItem.getFieldName().equals(idFieldName)) {
@@ -294,7 +294,7 @@ public class ES6xTemplate implements ESTemplate {
     public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping, Map<String, Object> dmlData,
                                        Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
@@ -304,7 +304,7 @@ public class ES6xTemplate implements ESTemplate {
                 resultIdVal = value;
             }
 
-            if (!fieldItem.getFieldName().equals(mapping.getId())
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
                     && !mapping.getSkips().contains(fieldItem.getFieldName())) {
                 esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
             }
@@ -319,7 +319,7 @@ public class ES6xTemplate implements ESTemplate {
     public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping,String owner, Map<String, Object> dmlData,
                                        Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
@@ -353,19 +353,19 @@ public class ES6xTemplate implements ESTemplate {
     }
 
     private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     esUpdateRequest.setRouting(parentVal);
                 }
                 getBulk().add(esUpdateRequest);
             } else {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     pkVal.toString()).setDoc(esFieldData);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     esUpdateRequest.setRouting(parentVal);
@@ -373,12 +373,12 @@ public class ES6xTemplate implements ESTemplate {
                 getBulk().add(esUpdateRequest);
             }
         } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex(),
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                mapping.getType()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
+                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
             SearchResponse response = esSearchRequest.getResponse();
             for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                    mapping.getType(),
+                    mapping.get_type(),
                     hit.getId()).setDoc(esFieldData);
                 getBulk().add(esUpdateRequest);
             }
@@ -394,15 +394,15 @@ public class ES6xTemplate implements ESTemplate {
      */
     @SuppressWarnings("unchecked")
     private String getEsType(ESMapping mapping, String fieldName) {
-        String key = mapping.getIndex() + "-" + mapping.getType();
+        String key = mapping.get_index() + "-" + mapping.get_type();
         Map<String, String> fieldType = esFieldTypes.get(key);
         if (fieldType != null) {
             return fieldType.get(fieldName);
         } else {
-            MappingMetaData mappingMetaData = esConnection.getMapping(mapping.getIndex(), mapping.getType());
+            MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index(), mapping.get_type());
 
             if (mappingMetaData == null) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.getIndex());
+                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
             }
 
             fieldType = new LinkedHashMap<>();

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/ConfigLoadTest.java

@@ -29,9 +29,9 @@ public class ConfigLoadTest {
         Assert.assertNotNull(config);
         Assert.assertEquals("defaultDS", config.getDataSourceKey());
         ESSyncConfig.ESMapping esMapping = config.getEsMapping();
-        Assert.assertEquals("mytest_user", esMapping.getIndex());
+        Assert.assertEquals("mytest_user", esMapping.get_index());
-        Assert.assertEquals("_doc", esMapping.getType());
+        Assert.assertEquals("_doc", esMapping.get_type());
-        Assert.assertEquals("id", esMapping.getId());
+        Assert.assertEquals("id", esMapping.get_id());
         Assert.assertNotNull(esMapping.getSql());
 
         // Map<String, List<ESSyncConfig>> dbTableEsSyncConfig =

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/Common.java

@@ -5,7 +5,6 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 
 import javax.sql.DataSource;
 
@@ -27,7 +26,7 @@ public class Common {
         outerAdapterConfig.setProperties(properties);
 
         ES6xAdapter esAdapter = new ES6xAdapter();
-        esAdapter.init(outerAdapterConfig, new Properties());
+        esAdapter.init(outerAdapterConfig, null);
         return esAdapter;
     }
 

client-adapter/es7x/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java

@@ -59,11 +59,11 @@ public class ES7xAdapter extends ESAdapter {
     public Map<String, Object> count(String task) {
         ESSyncConfig config = esSyncConfig.get(task);
         ESSyncConfig.ESMapping mapping = config.getEsMapping();
-        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.getIndex()).size(0).getResponse();
+        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index()).size(0).getResponse();
 
         long rowCount = response.getHits().getTotalHits().value;
         Map<String, Object> res = new LinkedHashMap<>();
-        res.put("esIndex", mapping.getIndex());
+        res.put("esIndex", mapping.get_index());
         res.put("count", rowCount);
         return res;
     }

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/etl/ESEtlService.java

@@ -51,7 +51,7 @@ public class ESEtlService extends AbstractEtlService {
 
     public EtlResult importData(List<String> params) {
         ESMapping mapping = config.getEsMapping();
-        logger.info("start etl to import data to index: {}", mapping.getIndex());
+        logger.info("start etl to import data to index: {}", mapping.get_index());
         String sql = mapping.getSql();
         return importData(sql, params);
     }
@@ -78,7 +78,7 @@ public class ESEtlService extends AbstractEtlService {
                             }
 
                             // 如果是主键字段则不插入
-                            if (fieldItem.getFieldName().equals(mapping.getId())) {
+                            if (fieldItem.getFieldName().equals(mapping.get_id())) {
                                 idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
                             } else {
                                 Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
@@ -118,7 +118,7 @@ public class ESEtlService extends AbstractEtlService {
                             String parentVal = (String) esFieldData.remove("$parent_routing");
                             if (mapping.isUpsert()) {
                                 ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(
-                                    mapping.getIndex(),
+                                    mapping.get_index(),
                                     idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
 
                                 if (StringUtils.isNotEmpty(parentVal)) {
@@ -128,7 +128,7 @@ public class ESEtlService extends AbstractEtlService {
                                 esBulkRequest.add(esUpdateRequest);
                             } else {
                                 ESIndexRequest esIndexRequest = this.esConnection.new ES7xIndexRequest(
-                                    mapping.getIndex(),
+                                    mapping.get_index(),
                                     idVal.toString()).setSource(esFieldData);
                                 if (StringUtils.isNotEmpty(parentVal)) {
                                     esIndexRequest.setRouting(parentVal);
@@ -137,13 +137,13 @@ public class ESEtlService extends AbstractEtlService {
                             }
                         } else {
                             idVal = esFieldData.get(mapping.getPk());
-                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
+                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
                                 .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
                                 .size(10000);
                             SearchResponse response = esSearchRequest.getResponse();
                             for (SearchHit hit : response.getHits()) {
                                 ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(
-                                    mapping.getIndex(),
+                                    mapping.get_index(),
                                     hit.getId()).setDoc(esFieldData);
                                 esBulkRequest.add(esUpdateRequest);
                             }
@@ -162,7 +162,7 @@ public class ESEtlService extends AbstractEtlService {
                                     (System.currentTimeMillis() - batchBegin),
                                     (System.currentTimeMillis() - esBatchBegin),
                                     esBulkRequest.numberOfActions(),
-                                    mapping.getIndex());
+                                    mapping.get_index());
                             }
                             batchBegin = System.currentTimeMillis();
                             esBulkRequest.resetBulk();
@@ -182,12 +182,12 @@ public class ESEtlService extends AbstractEtlService {
                                 (System.currentTimeMillis() - batchBegin),
                                 (System.currentTimeMillis() - esBatchBegin),
                                 esBulkRequest.numberOfActions(),
-                                mapping.getIndex());
+                                mapping.get_index());
                         }
                     }
                 } catch (Exception e) {
                     logger.error(e.getMessage(), e);
-                    errMsg.add(mapping.getIndex() + " etl failed! ==>" + e.getMessage());
+                    errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
                     throw new RuntimeException(e);
                 }
                 return count;

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -62,17 +66,17 @@ public class ES7xTemplate implements ESTemplate {
 
     @Override
     public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                ESUpdateRequest updateRequest = esConnection.new ES7xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest updateRequest = esConnection.new ES7xUpdateRequest(mapping.get_index(),
                     pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     updateRequest.setRouting(parentVal);
                 }
                 getBulk().add(updateRequest);
             } else {
-                ESIndexRequest indexRequest = esConnection.new ES7xIndexRequest(mapping.getIndex(), pkVal.toString())
+                ESIndexRequest indexRequest = esConnection.new ES7xIndexRequest(mapping.get_index(), pkVal.toString())
                     .setSource(esFieldData);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     indexRequest.setRouting(parentVal);
@@ -81,13 +85,13 @@ public class ES7xTemplate implements ESTemplate {
             }
             commitBulk();
         } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
                 .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
                 .size(10000);
             SearchResponse response = esSearchRequest.getResponse();
 
             for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(),
                     hit.getId()).setDoc(esFieldData);
                 getBulk().add(esUpdateRequest);
                 commitBulk();
@@ -144,18 +148,18 @@ public class ES7xTemplate implements ESTemplate {
 
     @Override
     public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
-            ESDeleteRequest esDeleteRequest = this.esConnection.new ES7xDeleteRequest(mapping.getIndex(),
+            ESDeleteRequest esDeleteRequest = this.esConnection.new ES7xDeleteRequest(mapping.get_index(),
                 pkVal.toString());
             getBulk().add(esDeleteRequest);
             commitBulk();
         } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
                 .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
                 .size(10000);
             SearchResponse response = esSearchRequest.getResponse();
             for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(),
                     hit.getId()).setDoc(esFieldData);
                 getBulk().add(esUpdateRequest);
                 commitBulk();
@@ -200,7 +204,7 @@ public class ES7xTemplate implements ESTemplate {
     public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,
                                   Map<String, Object> esFieldData) throws SQLException {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
@@ -209,7 +213,7 @@ public class ES7xTemplate implements ESTemplate {
                 resultIdVal = value;
             }
 
-            if (!fieldItem.getFieldName().equals(mapping.getId())
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
                 && !mapping.getSkips().contains(fieldItem.getFieldName())) {
                 esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
             }
@@ -224,7 +228,7 @@ public class ES7xTemplate implements ESTemplate {
     @Override
     public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
@@ -241,7 +245,7 @@ public class ES7xTemplate implements ESTemplate {
     public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
                                   Map<String, Object> esFieldData) throws SQLException {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             if (fieldItem.getFieldName().equals(idFieldName)) {
@@ -285,7 +289,7 @@ public class ES7xTemplate implements ESTemplate {
     @Override
     public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
@@ -295,7 +299,7 @@ public class ES7xTemplate implements ESTemplate {
                 resultIdVal = value;
             }
 
-            if (!fieldItem.getFieldName().equals(mapping.getId())
+            if (!fieldItem.getFieldName().equals(mapping.get_id())
                 && !mapping.getSkips().contains(fieldItem.getFieldName())) {
                 esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
             }
@@ -310,19 +314,15 @@ public class ES7xTemplate implements ESTemplate {
     public Object getESDataFromDmlData(ESMapping mapping,String owner, Map<String, Object> dmlData, Map<String, Object> dmlOld,
                                        Map<String, Object> esFieldData) {
         SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
         Object resultIdVal = null;
         for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
             ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
-            if (columnItem.getOwner() == null || columnItem.getColumnName() == null) {
-                continue;
-            }
-
             if (!columnItem.getOwner().equals(owner)) {
                 continue;
             }
-
             String columnName = columnItem.getColumnName();
+
             if (fieldItem.getFieldName().equals(idFieldName)) {
                 resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
             }
@@ -348,17 +348,17 @@ public class ES7xTemplate implements ESTemplate {
     }
 
     private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
             String parentVal = (String) esFieldData.remove("$parent_routing");
             if (mapping.isUpsert()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(),
                     pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     esUpdateRequest.setRouting(parentVal);
                 }
                 getBulk().add(esUpdateRequest);
             } else {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(),
                     pkVal.toString()).setDoc(esFieldData);
                 if (StringUtils.isNotEmpty(parentVal)) {
                     esUpdateRequest.setRouting(parentVal);
@@ -366,12 +366,12 @@ public class ES7xTemplate implements ESTemplate {
                 getBulk().add(esUpdateRequest);
             }
         } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
+            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index())
                 .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
                 .size(10000);
             SearchResponse response = esSearchRequest.getResponse();
             for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.getIndex(),
+                ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(),
                     hit.getId()).setDoc(esFieldData);
                 getBulk().add(esUpdateRequest);
             }
@@ -387,15 +387,15 @@ public class ES7xTemplate implements ESTemplate {
      */
     @SuppressWarnings("unchecked")
     private String getEsType(ESMapping mapping, String fieldName) {
-        String key = mapping.getIndex() + "-" + mapping.getType();
+        String key = mapping.get_index() + "-" + mapping.get_type();
         Map<String, String> fieldType = esFieldTypes.get(key);
         if (fieldType != null) {
             return fieldType.get(fieldName);
         } else {
-            MappingMetaData mappingMetaData = esConnection.getMapping(mapping.getIndex());
+            MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index());
 
             if (mappingMetaData == null) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.getIndex());
+                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
             }
 
             fieldType = new LinkedHashMap<>();

client-adapter/es8x/pom.xml

@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>canal.client-adapter</artifactId>
-        <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>com.alibaba.otter</groupId>
-    <artifactId>client-adapter.es8x</artifactId>
-    <packaging>jar</packaging>
-    <name>canal client adapter es v8x module for otter ${project.version}</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>client-adapter.common</artifactId>
-            <version>${project.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>client-adapter.escore</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>co.elastic.clients</groupId>
-            <artifactId>elasticsearch-java</artifactId>
-            <version>8.6.2</version>
-        </dependency>
-        <!--<dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>elasticsearch-rest-client</artifactId>
-            <version>8.6.2</version>
-        </dependency>-->
-        <dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>elasticsearch-rest-high-level-client</artifactId>
-            <version>7.17.9</version>
-        </dependency>
-
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <configuration>
-                            <tasks>
-                                <copy todir="${project.basedir}/../launcher/target/classes/es8" overwrite="true">
-                                    <fileset dir="${project.basedir}/target/classes/es8" erroronmissingdir="true">
-                                        <include name="*.yml" />
-                                    </fileset>
-                                </copy>
-                            </tasks>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/ES8xAdapter.java

@@ -1,117 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es8x;
-
-import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es8x.etl.ESEtlService;
-import com.alibaba.otter.canal.client.adapter.es8x.support.ES8xTemplate;
-import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-import org.elasticsearch.action.search.SearchResponse;
-
-import javax.sql.DataSource;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * ES 8.x 外部适配器
- *
- * @author ymz 2013-02-23
- * @version 1.0.0
- */
-@SPI("es8")
-public class ES8xAdapter extends ESAdapter {
-
-    private ESConnection esConnection;
-
-    public ESConnection getEsConnection() {
-        return esConnection;
-    }
-
-    @Override
-    public void init(OuterAdapterConfig configuration, Properties envProperties) {
-        try {
-            Map<String, String> properties = configuration.getProperties();
-
-            String[] hostArray = configuration.getHosts().split(",");
-            esConnection = new ESConnection(hostArray, properties);
-
-            this.esTemplate = new ES8xTemplate(esConnection);
-
-            envProperties.put("es.version", "es8");
-            super.init(configuration, envProperties);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public Map<String, Object> count(String task) {
-        ESSyncConfig config = esSyncConfig.get(task);
-        ESSyncConfig.ESMapping mapping = config.getEsMapping();
-        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.getIndex()).size(0).getResponse();
-
-        long rowCount = response.getHits().getTotalHits().value;
-        Map<String, Object> res = new LinkedHashMap<>();
-        res.put("esIndex", mapping.getIndex());
-        res.put("count", rowCount);
-        return res;
-    }
-
-    @Override
-    public EtlResult etl(String task, List<String> params) {
-        EtlResult etlResult = new EtlResult();
-        ESSyncConfig config = esSyncConfig.get(task);
-        if (config != null) {
-            DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-            ESEtlService esEtlService = new ESEtlService(esConnection, config);
-            if (dataSource != null) {
-                return esEtlService.importData(params);
-            } else {
-                etlResult.setSucceeded(false);
-                etlResult.setErrorMessage("DataSource not found");
-                return etlResult;
-            }
-        } else {
-            StringBuilder resultMsg = new StringBuilder();
-            boolean resSuccess = true;
-            for (ESSyncConfig configTmp : esSyncConfig.values()) {
-                // 取所有的destination为task的配置
-                if (configTmp.getDestination().equals(task)) {
-                    ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
-                    EtlResult etlRes = esEtlService.importData(params);
-                    if (!etlRes.getSucceeded()) {
-                        resSuccess = false;
-                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
-                    } else {
-                        resultMsg.append(etlRes.getResultMessage()).append("\n");
-                    }
-                }
-            }
-            if (resultMsg.length() > 0) {
-                etlResult.setSucceeded(resSuccess);
-                if (resSuccess) {
-                    etlResult.setResultMessage(resultMsg.toString());
-                } else {
-                    etlResult.setErrorMessage(resultMsg.toString());
-                }
-                return etlResult;
-            }
-        }
-        etlResult.setSucceeded(false);
-        etlResult.setErrorMessage("Task not found");
-        return etlResult;
-    }
-
-    @Override
-    public void destroy() {
-        super.destroy();
-        if (esConnection != null) {
-            esConnection.close();
-        }
-    }
-}

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/etl/ESEtlService.java

@@ -1,202 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es8x.etl;
-
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.sql.DataSource;
-
-import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-
-import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
-import com.alibaba.otter.canal.client.adapter.es8x.support.ES8xTemplate;
-import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection;
-import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
-import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * ES ETL Service
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESEtlService extends AbstractEtlService {
-
-    private ESConnection esConnection;
-    private ESTemplate   esTemplate;
-    private ESSyncConfig config;
-
-    public ESEtlService(ESConnection esConnection, ESSyncConfig config){
-        super("ES", config);
-        this.esConnection = esConnection;
-        this.esTemplate = new ES8xTemplate(esConnection);
-        this.config = config;
-    }
-
-    public EtlResult importData(List<String> params) {
-        ESMapping mapping = config.getEsMapping();
-        logger.info("start etl to import data to index: {}", mapping.getIndex());
-        String sql = mapping.getSql();
-        return importData(sql, params);
-    }
-
-    protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
-                                       AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount,
-                                       List<String> errMsg) {
-        try {
-            ESMapping mapping = (ESMapping) adapterMapping;
-            Util.sqlRS(ds, sql, values, rs -> {
-                int count = 0;
-                try {
-                    ESBulkRequest esBulkRequest = this.esConnection.new ES8xBulkRequest();
-
-                    long batchBegin = System.currentTimeMillis();
-                    while (rs.next()) {
-                        Map<String, Object> esFieldData = new LinkedHashMap<>();
-                        Object idVal = null;
-                        for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
-
-                            String fieldName = fieldItem.getFieldName();
-                            if (mapping.getSkips().contains(fieldName)) {
-                                continue;
-                            }
-
-                            // 如果是主键字段则不插入
-                            if (fieldItem.getFieldName().equals(mapping.getId())) {
-                                idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                            } else {
-                                Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                                esFieldData.put(Util.cleanColumn(fieldName), val);
-                            }
-
-                        }
-
-                        if (!mapping.getRelations().isEmpty()) {
-                            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                                Map<String, Object> relations = new HashMap<>();
-                                relations.put("name", relationMapping.getName());
-                                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                                    FieldItem parentFieldItem = mapping.getSchemaItem()
-                                        .getSelectFields()
-                                        .get(relationMapping.getParent());
-                                    Object parentVal;
-                                    try {
-                                        parentVal = esTemplate.getValFromRS(mapping,
-                                            rs,
-                                            parentFieldItem.getFieldName(),
-                                            parentFieldItem.getFieldName());
-                                    } catch (SQLException e) {
-                                        throw new RuntimeException(e);
-                                    }
-                                    if (parentVal != null) {
-                                        relations.put("parent", parentVal.toString());
-                                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                                    }
-                                }
-                                esFieldData.put(Util.cleanColumn(relationField), relations);
-                            });
-                        }
-
-                        if (idVal != null) {
-                            String parentVal = (String) esFieldData.remove("$parent_routing");
-                            if (mapping.isUpsert()) {
-                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(
-                                    mapping.getIndex(),
-                                    idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-
-                                if (StringUtils.isNotEmpty(parentVal)) {
-                                    esUpdateRequest.setRouting(parentVal);
-                                }
-
-                                esBulkRequest.add(esUpdateRequest);
-                            } else {
-                                ESIndexRequest esIndexRequest = this.esConnection.new ES8xIndexRequest(
-                                    mapping.getIndex(),
-                                    idVal.toString()).setSource(esFieldData);
-                                if (StringUtils.isNotEmpty(parentVal)) {
-                                    esIndexRequest.setRouting(parentVal);
-                                }
-                                esBulkRequest.add(esIndexRequest);
-                            }
-                        } else {
-                            idVal = esFieldData.get(mapping.getPk());
-                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
-                                .setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
-                                .size(10000);
-                            SearchResponse response = esSearchRequest.getResponse();
-                            for (SearchHit hit : response.getHits()) {
-                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(
-                                    mapping.getIndex(),
-                                    hit.getId()).setDoc(esFieldData);
-                                esBulkRequest.add(esUpdateRequest);
-                            }
-                        }
-
-                        if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
-                            && esBulkRequest.numberOfActions() > 0) {
-                            long esBatchBegin = System.currentTimeMillis();
-                            ESBulkResponse rp = esBulkRequest.bulk();
-                            if (rp.hasFailures()) {
-                                rp.processFailBulkResponse("全量数据 etl 异常 ");
-                            }
-
-                            if (logger.isTraceEnabled()) {
-                                logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                                    (System.currentTimeMillis() - batchBegin),
-                                    (System.currentTimeMillis() - esBatchBegin),
-                                    esBulkRequest.numberOfActions(),
-                                    mapping.getIndex());
-                            }
-                            batchBegin = System.currentTimeMillis();
-                            esBulkRequest.resetBulk();
-                        }
-                        count++;
-                        impCount.incrementAndGet();
-                    }
-
-                    if (esBulkRequest.numberOfActions() > 0) {
-                        long esBatchBegin = System.currentTimeMillis();
-                        ESBulkResponse rp = esBulkRequest.bulk();
-                        if (rp.hasFailures()) {
-                            rp.processFailBulkResponse("全量数据 etl 异常 ");
-                        }
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                                (System.currentTimeMillis() - batchBegin),
-                                (System.currentTimeMillis() - esBatchBegin),
-                                esBulkRequest.numberOfActions(),
-                                mapping.getIndex());
-                        }
-                    }
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    errMsg.add(mapping.getIndex() + " etl failed! ==>" + e.getMessage());
-                    throw new RuntimeException(e);
-                }
-                return count;
-            });
-
-            return true;
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-            return false;
-        }
-    }
-}

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ES8xTemplate.java

@@ -1,467 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es8x.support;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import javax.sql.DataSource;
-
-import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.cluster.metadata.MappingMetadata;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
-import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection.ESSearchRequest;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-public class ES8xTemplate implements ESTemplate {
-
-    private static final Logger                               logger         = LoggerFactory
-        .getLogger(ESTemplate.class);
-
-    private static final int                                  MAX_BATCH_SIZE = 1000;
-
-    private ESConnection                                      esConnection;
-
-    private ESBulkRequest                                     esBulkRequest;
-
-    // es 字段类型本地缓存
-    private static ConcurrentMap<String, Map<String, String>> esFieldTypes   = new ConcurrentHashMap<>();
-
-    public ES8xTemplate(ESConnection esConnection){
-        this.esConnection = esConnection;
-        this.esBulkRequest = this.esConnection.new ES8xBulkRequest();
-    }
-
-    public ESBulkRequest getBulk() {
-        return esBulkRequest;
-    }
-
-    public void resetBulkRequestBuilder() {
-        this.esBulkRequest.resetBulk();
-    }
-
-    @Override
-    public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
-            String parentVal = (String) esFieldData.remove("$parent_routing");
-            if (mapping.isUpsert()) {
-                ESUpdateRequest updateRequest = esConnection.new ES8xUpdateRequest(mapping.getIndex(), pkVal.toString())
-                    .setDoc(esFieldData)
-                    .setDocAsUpsert(true);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    updateRequest.setRouting(parentVal);
-                }
-                getBulk().add(updateRequest);
-            } else {
-                ESIndexRequest indexRequest = esConnection.new ES8xIndexRequest(mapping.getIndex(), pkVal.toString())
-                    .setSource(esFieldData);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    indexRequest.setRouting(parentVal);
-                }
-                getBulk().add(indexRequest);
-            }
-            commitBulk();
-        } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
-                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
-                .size(10000);
-            SearchResponse response = esSearchRequest.getResponse();
-
-            for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.getIndex(),
-                    hit.getId()).setDoc(esFieldData);
-                getBulk().add(esUpdateRequest);
-                commitBulk();
-            }
-        }
-    }
-
-    @Override
-    public void update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        Map<String, Object> esFieldDataTmp = new LinkedHashMap<>(esFieldData.size());
-        esFieldData.forEach((k, v) -> esFieldDataTmp.put(Util.cleanColumn(k), v));
-        append4Update(mapping, pkVal, esFieldDataTmp);
-        commitBulk();
-    }
-
-    @Override
-    public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
-        if (paramsTmp.isEmpty()) {
-            return;
-        }
-        ESMapping mapping = config.getEsMapping();
-        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
-        paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
-
-        // 查询sql批量更新
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
-        List<Object> values = new ArrayList<>();
-        paramsTmp.forEach((fieldName, value) -> {
-            sql.append("_v.").append(fieldName).append("=? AND ");
-            values.add(value);
-        });
-        // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
-        int len = sql.length();
-        sql.delete(len - 4, len);
-        Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
-            int count = 0;
-            try {
-                while (rs.next()) {
-                    Object idVal = getIdValFromRS(mapping, rs);
-                    append4Update(mapping, idVal, esFieldData);
-                    commitBulk();
-                    count++;
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return count;
-        });
-        if (logger.isTraceEnabled()) {
-            logger.trace("Update ES by query affected {} records", syncCount);
-        }
-    }
-
-    @Override
-    public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
-            ESDeleteRequest esDeleteRequest = this.esConnection.new ES8xDeleteRequest(mapping.getIndex(),
-                pkVal.toString());
-            getBulk().add(esDeleteRequest);
-            commitBulk();
-        } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
-                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
-                .size(10000);
-            SearchResponse response = esSearchRequest.getResponse();
-            for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.getIndex(),
-                    hit.getId()).setDoc(esFieldData);
-                getBulk().add(esUpdateRequest);
-                commitBulk();
-            }
-        }
-    }
-
-    @Override
-    public void commit() {
-        if (getBulk().numberOfActions() > 0) {
-            ESBulkResponse response = getBulk().bulk();
-            if (response.hasFailures()) {
-                response.processFailBulkResponse("ES sync commit error ");
-            }
-            resetBulkRequestBuilder();
-        }
-    }
-
-    @Override
-    public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName,
-                               String columnName) throws SQLException {
-        fieldName = Util.cleanColumn(fieldName);
-        columnName = Util.cleanColumn(columnName);
-        String esType = getEsType(mapping, fieldName);
-
-        Object value = resultSet.getObject(columnName);
-        if (value instanceof Boolean) {
-            if (!"boolean".equals(esType)) {
-                value = resultSet.getByte(columnName);
-            }
-        }
-
-        // 如果是对象类型
-        if (mapping.getObjFields().containsKey(fieldName)) {
-            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
-        } else {
-            return ESSyncUtil.typeConvert(value, esType);
-        }
-    }
-
-    @Override
-    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,
-                                  Map<String, Object> esFieldData) throws SQLException {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = value;
-            }
-
-            if (!fieldItem.getFieldName().equals(mapping.getId())
-                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
-
-        return resultIdVal;
-    }
-
-    @Override
-    public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = value;
-                break;
-            }
-        }
-        return resultIdVal;
-    }
-
-    @Override
-    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
-                                  Map<String, Object> esFieldData) throws SQLException {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
-            }
-
-            for (ColumnItem columnItem : fieldItem.getColumnItems()) {
-                if (dmlOld.containsKey(columnItem.getColumnName())
-                    && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                    esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
-                        getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
-                    break;
-                }
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
-
-        return resultIdVal;
-    }
-
-    @Override
-    public Object getValFromData(ESMapping mapping, Map<String, Object> dmlData, String fieldName, String columnName) {
-        String esType = getEsType(mapping, fieldName);
-        Object value = dmlData.get(columnName);
-        if (value instanceof Byte) {
-            if ("boolean".equals(esType)) {
-                value = ((Byte) value).intValue() != 0;
-            }
-        }
-
-        // 如果是对象类型
-        if (mapping.getObjFields().containsKey(fieldName)) {
-            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
-        } else {
-            return ESSyncUtil.typeConvert(value, esType);
-        }
-    }
-
-    @Override
-    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
-                                       Map<String, Object> esFieldData) {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
-            Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = value;
-            }
-
-            if (!fieldItem.getFieldName().equals(mapping.getId())
-                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationData(mapping, schemaItem, dmlData, esFieldData);
-        return resultIdVal;
-    }
-
-    @Override
-    public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
-                                       Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
-            if (!columnItem.getOwner().equals(owner)) {
-                continue;
-            }
-            String columnName = columnItem.getColumnName();
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
-            }
-
-            if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
-                    getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationData(mapping, schemaItem, dmlOld, esFieldData);
-        return resultIdVal;
-    }
-
-    /**
-     * 如果大于批量数则提交批次
-     */
-    private void commitBulk() {
-        if (getBulk().numberOfActions() >= MAX_BATCH_SIZE) {
-            commit();
-        }
-    }
-
-    private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.getId() != null) {
-            String parentVal = (String) esFieldData.remove("$parent_routing");
-            if (mapping.isUpsert()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.getIndex(),
-                    pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    esUpdateRequest.setRouting(parentVal);
-                }
-                getBulk().add(esUpdateRequest);
-            } else {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.getIndex(),
-                    pkVal.toString()).setDoc(esFieldData);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    esUpdateRequest.setRouting(parentVal);
-                }
-                getBulk().add(esUpdateRequest);
-            }
-        } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.getIndex())
-                .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal))
-                .size(10000);
-            SearchResponse response = esSearchRequest.getResponse();
-            for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ES8xUpdateRequest(mapping.getIndex(),
-                    hit.getId()).setDoc(esFieldData);
-                getBulk().add(esUpdateRequest);
-            }
-        }
-    }
-
-    /**
-     * 获取es mapping中的属性类型
-     *
-     * @param mapping mapping配置
-     * @param fieldName 属性名
-     * @return 类型
-     */
-    @SuppressWarnings("unchecked")
-    private String getEsType(ESMapping mapping, String fieldName) {
-        String key = mapping.getIndex() + "-" + mapping.getType();
-        Map<String, String> fieldType = esFieldTypes.get(key);
-        if (fieldType != null) {
-            return fieldType.get(fieldName);
-        } else {
-            MappingMetadata mappingMetaData = esConnection.getMapping(mapping.getIndex());
-
-            if (mappingMetaData == null) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.getIndex());
-            }
-
-            fieldType = new LinkedHashMap<>();
-
-            Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
-            Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
-            for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
-                Map<String, Object> value = (Map<String, Object>) entry.getValue();
-                if (value.containsKey("properties")) {
-                    fieldType.put(entry.getKey(), "object");
-                } else {
-                    fieldType.put(entry.getKey(), (String) value.get("type"));
-                }
-            }
-            esFieldTypes.put(key, fieldType);
-
-            return fieldType.get(fieldName);
-        }
-    }
-
-    private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet,
-                                       Map<String, Object> esFieldData) {
-        // 添加父子文档关联信息
-        if (!mapping.getRelations().isEmpty()) {
-            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                Map<String, Object> relations = new HashMap<>();
-                relations.put("name", relationMapping.getName());
-                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                    FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
-                    Object parentVal;
-                    try {
-                        parentVal = getValFromRS(mapping,
-                            resultSet,
-                            parentFieldItem.getFieldName(),
-                            parentFieldItem.getFieldName());
-                    } catch (SQLException e) {
-                        throw new RuntimeException(e);
-                    }
-                    if (parentVal != null) {
-                        relations.put("parent", parentVal.toString());
-                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                    }
-                }
-                esFieldData.put(relationField, relations);
-            });
-        }
-    }
-
-    private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map<String, Object> dmlData,
-                                 Map<String, Object> esFieldData) {
-        // 添加父子文档关联信息
-        if (!mapping.getRelations().isEmpty()) {
-            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                Map<String, Object> relations = new HashMap<>();
-                relations.put("name", relationMapping.getName());
-                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                    FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
-                    String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName();
-                    Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName);
-                    if (parentVal != null) {
-                        relations.put("parent", parentVal.toString());
-                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                    }
-                }
-                esFieldData.put(relationField, relations);
-            });
-        }
-    }
-}

client-adapter/es8x/src/main/java/com/alibaba/otter/canal/client/adapter/es8x/support/ESConnection.java

@@ -1,449 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es8x.support;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.KeyStore;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateFactory;
-import java.util.Arrays;
-import java.util.Map;
-
-import javax.net.ssl.SSLContext;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.ssl.SSLContextBuilder;
-import org.apache.http.ssl.SSLContexts;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateRequestBuilder;
-import org.elasticsearch.client.*;
-import org.elasticsearch.client.indices.GetMappingsRequest;
-import org.elasticsearch.client.indices.GetMappingsResponse;
-import org.elasticsearch.cluster.metadata.MappingMetadata;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
-
-/**
- * ES 连接器, 只支持 Rest 方式
- *
- * @author ymz 2023-03-02
- * @version 1.0.0
- */
-public class ESConnection {
-
-    private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
-
-    private RestHighLevelClient restHighLevelClient;
-
-    public ESConnection(String[] hosts, Map<String, String> properties) throws UnknownHostException{
-        String caPath = properties.get("security.ca.path");
-        if (StringUtils.isNotEmpty(caPath)) {
-            connectEsWithCa(hosts, properties, caPath);
-        } else {
-            connectEsWithoutCa(hosts, properties);
-        }
-    }
-
-    private void connectEsWithCa(String[] hosts, Map<String, String> properties, String caPath) {
-        Path caCertificatePath = Paths.get(caPath);
-        try (InputStream is = Files.newInputStream(caCertificatePath)) {
-            CertificateFactory factory = CertificateFactory.getInstance("X.509");
-            Certificate trustedCa = factory.generateCertificate(is);
-            KeyStore trustStore = KeyStore.getInstance("pkcs12");
-            trustStore.load(null, null);
-            trustStore.setCertificateEntry("ca", trustedCa);
-            SSLContextBuilder sslContextBuilder = SSLContexts.custom().loadTrustMaterial(trustStore, null);
-            final SSLContext sslContext = sslContextBuilder.build();
-
-            HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
-            RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
-            String nameAndPwd = properties.get("security.auth");
-            if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
-                String[] nameAndPwdArr = nameAndPwd.split(":");
-                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-                credentialsProvider.setCredentials(AuthScope.ANY,
-                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-                restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> {
-                    httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
-                    return httpClientBuilder.setSSLContext(sslContext);
-                });
-            }
-            restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build())
-                .setApiCompatibilityMode(true)
-                .build();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private void connectEsWithoutCa(String[] hosts, Map<String, String> properties) {
-        HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
-        RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
-        String nameAndPwd = properties.get("security.auth");
-        if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
-            String[] nameAndPwdArr = nameAndPwd.split(":");
-            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            credentialsProvider.setCredentials(AuthScope.ANY,
-                new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-            restClientBuilder.setHttpClientConfigCallback(
-                httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
-        }
-        restHighLevelClient = new RestHighLevelClientBuilder(restClientBuilder.build()).setApiCompatibilityMode(true)
-            .build();
-    }
-
-    public void close() {
-        try {
-            restHighLevelClient.close();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public MappingMetadata getMapping(String index) {
-        MappingMetadata mappingMetaData = null;
-
-        Map<String, MappingMetadata> mappings;
-        try {
-            GetMappingsRequest request = new GetMappingsRequest();
-            request.indices(index);
-            GetMappingsResponse response = restHighLevelClient.indices().getMapping(request, RequestOptions.DEFAULT);
-
-            mappings = response.mappings();
-        } catch (NullPointerException e) {
-            throw new IllegalArgumentException("Not found the mapping info of index: " + index);
-        } catch (IOException e) {
-            logger.error(e.getMessage(), e);
-            return null;
-        }
-        mappingMetaData = mappings.get(index);
-
-        return mappingMetaData;
-    }
-
-    public class ES8xIndexRequest implements ESBulkRequest.ESIndexRequest {
-
-        private IndexRequestBuilder indexRequestBuilder;
-
-        private IndexRequest        indexRequest;
-
-        public ES8xIndexRequest(String index, String id){
-            indexRequest = new IndexRequest(index);
-            indexRequest.id(id);
-
-        }
-
-        public ES8xIndexRequest setSource(Map<String, ?> source) {
-
-            indexRequest.source(source);
-
-            return this;
-        }
-
-        public ES8xIndexRequest setRouting(String routing) {
-
-            indexRequest.routing(routing);
-
-            return this;
-        }
-
-        public IndexRequestBuilder getIndexRequestBuilder() {
-            return indexRequestBuilder;
-        }
-
-        public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) {
-            this.indexRequestBuilder = indexRequestBuilder;
-        }
-
-        public IndexRequest getIndexRequest() {
-            return indexRequest;
-        }
-
-        public void setIndexRequest(IndexRequest indexRequest) {
-            this.indexRequest = indexRequest;
-        }
-    }
-
-    public class ES8xUpdateRequest implements ESBulkRequest.ESUpdateRequest {
-
-        private UpdateRequestBuilder updateRequestBuilder;
-
-        private UpdateRequest        updateRequest;
-
-        public ES8xUpdateRequest(String index, String id){
-
-            updateRequest = new UpdateRequest(index, id);
-        }
-
-        public ES8xUpdateRequest setDoc(Map source) {
-
-            updateRequest.doc(source);
-
-            return this;
-        }
-
-        public ES8xUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) {
-
-            updateRequest.docAsUpsert(shouldUpsertDoc);
-
-            return this;
-        }
-
-        public ES8xUpdateRequest setRouting(String routing) {
-
-            updateRequest.routing(routing);
-
-            return this;
-        }
-
-        public UpdateRequestBuilder getUpdateRequestBuilder() {
-            return updateRequestBuilder;
-        }
-
-        public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) {
-            this.updateRequestBuilder = updateRequestBuilder;
-        }
-
-        public UpdateRequest getUpdateRequest() {
-            return updateRequest;
-        }
-
-        public void setUpdateRequest(UpdateRequest updateRequest) {
-            this.updateRequest = updateRequest;
-        }
-    }
-
-    public class ES8xDeleteRequest implements ESBulkRequest.ESDeleteRequest {
-
-        private DeleteRequestBuilder deleteRequestBuilder;
-
-        private DeleteRequest        deleteRequest;
-
-        public ES8xDeleteRequest(String index, String id){
-
-            deleteRequest = new DeleteRequest(index, id);
-
-        }
-
-        public DeleteRequestBuilder getDeleteRequestBuilder() {
-            return deleteRequestBuilder;
-        }
-
-        public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) {
-            this.deleteRequestBuilder = deleteRequestBuilder;
-        }
-
-        public DeleteRequest getDeleteRequest() {
-            return deleteRequest;
-        }
-
-        public void setDeleteRequest(DeleteRequest deleteRequest) {
-            this.deleteRequest = deleteRequest;
-        }
-    }
-
-    public class ESSearchRequest {
-
-        private SearchRequestBuilder searchRequestBuilder;
-
-        private SearchRequest        searchRequest;
-
-        private SearchSourceBuilder  sourceBuilder;
-
-        public ESSearchRequest(String index){
-
-            searchRequest = new SearchRequest(index);
-            sourceBuilder = new SearchSourceBuilder();
-
-        }
-
-        public ESSearchRequest setQuery(QueryBuilder queryBuilder) {
-
-            sourceBuilder.query(queryBuilder);
-
-            return this;
-        }
-
-        public ESSearchRequest size(int size) {
-
-            sourceBuilder.size(size);
-
-            return this;
-        }
-
-        public SearchResponse getResponse() {
-
-            searchRequest.source(sourceBuilder);
-            try {
-                return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-
-        }
-
-        public SearchRequestBuilder getSearchRequestBuilder() {
-            return searchRequestBuilder;
-        }
-
-        public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) {
-            this.searchRequestBuilder = searchRequestBuilder;
-        }
-
-        public SearchRequest getSearchRequest() {
-            return searchRequest;
-        }
-
-        public void setSearchRequest(SearchRequest searchRequest) {
-            this.searchRequest = searchRequest;
-        }
-    }
-
-    public class ES8xBulkRequest implements ESBulkRequest {
-
-        private BulkRequestBuilder bulkRequestBuilder;
-
-        private BulkRequest        bulkRequest;
-
-        public ES8xBulkRequest(){
-            bulkRequest = new BulkRequest();
-        }
-
-        public void resetBulk() {
-
-            bulkRequest = new BulkRequest();
-
-        }
-
-        public ES8xBulkRequest add(ESIndexRequest esIndexRequest) {
-            ES8xIndexRequest eir = (ES8xIndexRequest) esIndexRequest;
-
-            bulkRequest.add(eir.indexRequest);
-
-            return this;
-        }
-
-        public ES8xBulkRequest add(ESUpdateRequest esUpdateRequest) {
-            ES8xUpdateRequest eur = (ES8xUpdateRequest) esUpdateRequest;
-
-            bulkRequest.add(eur.updateRequest);
-
-            return this;
-        }
-
-        public ES8xBulkRequest add(ESDeleteRequest esDeleteRequest) {
-            ES8xDeleteRequest edr = (ES8xDeleteRequest) esDeleteRequest;
-
-            bulkRequest.add(edr.deleteRequest);
-
-            return this;
-        }
-
-        public int numberOfActions() {
-            return bulkRequest.numberOfActions();
-        }
-
-        public ESBulkResponse bulk() {
-            try {
-                BulkResponse responses = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
-                return new ES8xBulkResponse(responses);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-
-        }
-
-        public BulkRequestBuilder getBulkRequestBuilder() {
-            return bulkRequestBuilder;
-        }
-
-        public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) {
-            this.bulkRequestBuilder = bulkRequestBuilder;
-        }
-
-        public BulkRequest getBulkRequest() {
-            return bulkRequest;
-        }
-
-        public void setBulkRequest(BulkRequest bulkRequest) {
-            this.bulkRequest = bulkRequest;
-        }
-    }
-
-    public static class ES8xBulkResponse implements ESBulkRequest.ESBulkResponse {
-
-        private BulkResponse bulkResponse;
-
-        public ES8xBulkResponse(BulkResponse bulkResponse){
-            this.bulkResponse = bulkResponse;
-        }
-
-        @Override
-        public boolean hasFailures() {
-            return bulkResponse.hasFailures();
-        }
-
-        @Override
-        public void processFailBulkResponse(String errorMsg) {
-            for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
-                if (!itemResponse.isFailed()) {
-                    continue;
-                }
-
-                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
-                    logger.error(itemResponse.getFailureMessage());
-                } else {
-                    throw new RuntimeException(errorMsg + itemResponse.getFailureMessage());
-                }
-            }
-        }
-    }
-
-    public RestHighLevelClient getRestHighLevelClient() {
-        return restHighLevelClient;
-    }
-
-    public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
-        this.restHighLevelClient = restHighLevelClient;
-    }
-
-    private HttpHost createHttpHost(String uriStr) {
-        URI uri = URI.create(uriStr);
-        if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
-            return HttpHost.create(uri.toString());
-        }
-        try {
-            return HttpHost.create(new URI(uri
-                .getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
-                    .toString());
-        } catch (URISyntaxException ex) {
-            throw new IllegalStateException(ex);
-        }
-    }
-}

client-adapter/es8x/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -1 +0,0 @@
-es8=com.alibaba.otter.canal.client.adapter.es8x.ES8xAdapter

client-adapter/es8x/src/main/resources/es8/biz_order.yml

@@ -1,20 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-esMapping:
-  _index: customer
-  _id: _id
-  relations:
-    customer_order:
-      name: order
-      parent: customer_id
-  sql: "select concat('oid_', t.id) as _id,
-        t.customer_id,
-        t.id as order_id,
-        t.serial_code as order_serial,
-        t.c_time as order_time
-        from biz_order t"
-  skips:
-    - customer_id
-  etlCondition: "where t.c_time>={}"
-  commitBatch: 3000

client-adapter/es8x/src/main/resources/es8/customer.yml

@@ -1,46 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-esMapping:
-  _index: customer
-  _id: id
-  relations:
-    customer_order:
-      name: customer
-  sql: "select t.id, t.name, t.email from customer t"
-  etlCondition: "where t.c_time>={}"
-  commitBatch: 3000
-
-
-#{
-#  "mappings":{
-#    "_doc":{
-#      "properties":{
-#        "id": {
-#          "type": "long"
-#        },
-#        "name": {
-#          "type": "text"
-#        },
-#        "email": {
-#          "type": "text"
-#        },
-#        "order_id": {
-#          "type": "long"
-#        },
-#        "order_serial": {
-#          "type": "text"
-#        },
-#        "order_time": {
-#          "type": "date"
-#        },
-#        "customer_order":{
-#          "type":"join",
-#          "relations":{
-#            "customer":"order"
-#          }
-#        }
-#      }
-#    }
-#  }
-#}

client-adapter/es8x/src/main/resources/es8/mytest_user.yml

@@ -1,15 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-esMapping:
-  _index: mytest_user
-  _id: _id
-  #  upsert: true
-  #  pk: id
-  sql: "select a.id as _id, a.name, a.role_id, b.role_name,
-        a.c_time from user a
-        left join role b on b.id=a.role_id"
-  #  objFields:
-  #    _labels: array:;
-  etlCondition: "where a.c_time>={}"
-  commitBatch: 3000

client-adapter/es8x/src/test/java/com/alibaba/otter/canal/client/adapter/es8x/test/ESConnectionTest.java

@@ -1,44 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es8x.test;
-
-import com.alibaba.otter.canal.client.adapter.es8x.support.ESConnection;
-import org.elasticsearch.cluster.metadata.MappingMetadata;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.springframework.util.Assert;
-
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-
-@Ignore
-public class ESConnectionTest {
-
-    ESConnection esConnection;
-
-    @Before
-    public void init() throws UnknownHostException {
-        String[] hosts = new String[]{"127.0.0.1:9200"};
-        Map<String, String> properties = new HashMap<>();
-        properties.put("cluster.name", "elasticsearch");
-        esConnection = new ESConnection(hosts, properties);
-    }
-
-    @Test
-    public void test01() {
-        MappingMetadata mappingMetaData = esConnection.getMapping("mytest_user");
-
-        Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
-        Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
-        for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
-            Map<String, Object> value = (Map<String, Object>) entry.getValue();
-            if (value.containsKey("properties")) {
-                System.out.println(entry.getKey() + " object");
-            } else {
-                System.out.println(entry.getKey() + " " + value.get("type"));
-                Assert.notNull(entry.getKey(), "null column name");
-                Assert.notNull(value.get("type"), "null column type");
-            }
-        }
-    }
-}

client-adapter/es8x/src/test/java/com/alibaba/otter/canal/client/adapter/es8x/test/TestConstant.java

@@ -1,40 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es8x.test;
-
-import com.alibaba.druid.pool.DruidDataSource;
-
-import java.sql.SQLException;
-
-public class TestConstant {
-
-    public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
-    public final static String jdbcUser = "root";
-    public final static String jdbcPassword = "121212";
-
-    public final static String esHosts = "127.0.0.1:9300";
-    public final static String clusterName = "elasticsearch";
-
-    public final static DruidDataSource dataSource;
-
-    static {
-        dataSource = new DruidDataSource();
-        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
-        dataSource.setUrl(jdbcUrl);
-        dataSource.setUsername(jdbcUser);
-        dataSource.setPassword(jdbcPassword);
-        dataSource.setInitialSize(1);
-        dataSource.setMinIdle(1);
-        dataSource.setMaxActive(1);
-        dataSource.setMaxWait(60000);
-        dataSource.setTimeBetweenEvictionRunsMillis(60000);
-        dataSource.setMinEvictableIdleTimeMillis(300000);
-        dataSource.setPoolPreparedStatements(false);
-        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
-        dataSource.setValidationQuery("select 1");
-        try {
-            dataSource.init();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-    }
-
-}

client-adapter/escore/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfig.java

@@ -6,7 +6,6 @@ import java.util.List;
 import java.util.Map;
 
 import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import org.springframework.beans.factory.annotation.Value;
 
 /**
  * ES 映射配置
@@ -29,13 +28,13 @@ public class ESSyncConfig implements AdapterConfig {
     private String    esVersion = "es6";
 
     public void validate() {
-        if (esMapping.index == null) {
+        if (esMapping._index == null) {
             throw new NullPointerException("esMapping._index");
         }
-        if ("es6".equals(esVersion) && esMapping.type == null) {
+        if ("es6".equals(esVersion) && esMapping._type == null) {
             throw new NullPointerException("esMapping._type");
         }
-        if (esMapping.id == null && esMapping.getPk() == null) {
+        if (esMapping._id == null && esMapping.getPk() == null) {
             throw new NullPointerException("esMapping._id or esMapping.pk");
         }
         if (esMapping.sql == null) {
@@ -97,14 +96,9 @@ public class ESSyncConfig implements AdapterConfig {
 
     public static class ESMapping implements AdapterMapping {
 
-        @Value("${_index}")
+        private String                       _index;
-        private String index;
+        private String                       _type;
-
+        private String                       _id;
-        @Value("${_type}")
-        private String type;
-
-        @Value("${_id}")
-        private String id;
         private boolean                      upsert          = false;
         private String                       pk;
         private Map<String, RelationMapping> relations       = new LinkedHashMap<>();
@@ -120,28 +114,28 @@ public class ESSyncConfig implements AdapterConfig {
 
         private SchemaItem                   schemaItem;                             // sql解析结果模型
 
-        public String getIndex() {
+        public String get_index() {
-            return index;
+            return _index;
         }
 
-        public void setIndex(String index) {
+        public void set_index(String _index) {
-            this.index = index;
+            this._index = _index;
         }
 
-        public String getType() {
+        public String get_type() {
-            return type;
+            return _type;
         }
 
-        public void setType(String type) {
+        public void set_type(String _type) {
-            this.type = type;
+            this._type = _type;
         }
 
-        public String getId() {
+        public String get_id() {
-            return id;
+            return _id;
         }
 
-        public void setId(String id) {
+        public void set_id(String _id) {
-            this.id = id;
+            this._id = _id;
         }
 
         public boolean isUpsert() {

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/ESSyncConfigLoader.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.es.core.config;
 
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -8,6 +7,7 @@ import java.util.Properties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 
 /**
@@ -28,7 +28,7 @@ public class ESSyncConfigLoader {
         String esv = envProperties.getProperty("es.version");
         Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs(esv);
         configContentMap.forEach((fileName, content) -> {
-            ESSyncConfig config = YamlUtils.ymlToObj(null, content, ESSyncConfig.class, null, envProperties);
+            ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties);
             if (config == null) {
                 return;
             }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SchemaItem.java

@@ -136,8 +136,8 @@ public class SchemaItem {
     }
 
     public FieldItem getIdFieldItem(ESSyncConfig.ESMapping mapping) {
-        if (mapping.getId() != null) {
+        if (mapping.get_id() != null) {
-            return getSelectFields().get(mapping.getId());
+            return getSelectFields().get(mapping.get_id());
         } else {
             return getSelectFields().get(mapping.getPk());
         }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java

@@ -1,8 +1,12 @@
 package com.alibaba.otter.canal.client.adapter.es.core.monitor;
 
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
+import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 import java.io.File;
 import java.util.Properties;
-
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
 import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -10,14 +14,6 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
-
-;
-
 public class ESConfigMonitor {
 
     private static final Logger   logger = LoggerFactory.getLogger(ESConfigMonitor.class);
@@ -64,7 +60,7 @@ public class ESConfigMonitor {
             try {
                 // 加载新增的配置文件
                 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                ESSyncConfig config = YamlUtils.ymlToObj(null,
+                ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
                     configContent,
                     ESSyncConfig.class,
                     null,
@@ -97,7 +93,7 @@ public class ESConfigMonitor {
                         onFileDelete(file);
                         return;
                     }
-                    ESSyncConfig config = YamlUtils.ymlToObj(null,
+                    ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
                         configContent,
                         ESSyncConfig.class,
                         null,

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java

@@ -54,13 +54,13 @@ public class ESSyncService {
             for (ESSyncConfig config : esSyncConfigs) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("Prepared to sync index: {}, destination: {}",
-                        config.getEsMapping().getIndex(),
+                        config.getEsMapping().get_index(),
                         dml.getDestination());
                 }
                 this.sync(config, dml);
                 if (logger.isTraceEnabled()) {
                     logger.trace("Sync completed: {}, destination: {}",
-                        config.getEsMapping().getIndex(),
+                        config.getEsMapping().get_index(),
                         dml.getDestination());
                 }
             }
@@ -73,7 +73,7 @@ public class ESSyncService {
             if (logger.isDebugEnabled()) {
                 StringBuilder configIndexes = new StringBuilder();
                 esSyncConfigs
-                    .forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().getIndex()).append(" "));
+                    .forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().get_index()).append(" "));
                 logger.debug("DML: {} \nAffected indexes: {}",
                     JSON.toJSONString(dml, JSONWriter.Feature.WriteNulls),
                     configIndexes.toString());
@@ -105,10 +105,10 @@ public class ESSyncService {
                 logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}",
                     (System.currentTimeMillis() - begin),
                     dml.getDestination(),
-                    config.getEsMapping().getIndex());
+                    config.getEsMapping().get_index());
             }
         } catch (Throwable e) {
-            logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().getIndex(), dml);
+            logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
             throw new RuntimeException(e);
         }
     }
@@ -210,7 +210,7 @@ public class ESSyncService {
                 // ------主表 查询sql来更新------
                 if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
                     ESMapping mapping = config.getEsMapping();
-                    String idFieldName = mapping.getId() == null ? mapping.getPk() : mapping.getId();
+                    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
                     FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName);
 
                     boolean idFieldSimple = true;
@@ -336,7 +336,7 @@ public class ESSyncService {
 
             // ------是主表------
             if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
-                if (mapping.getId() != null) {
+                if (mapping.get_id() != null) {
                     FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
                     // 主键为简单字段
                     if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
@@ -349,7 +349,7 @@ public class ESSyncService {
                             logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}",
                                 config.getDestination(),
                                 dml.getTable(),
-                                mapping.getIndex(),
+                                mapping.get_index(),
                                 idVal);
                         }
                         esTemplate.delete(mapping, idVal, null);
@@ -368,7 +368,7 @@ public class ESSyncService {
                             logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}",
                                 config.getDestination(),
                                 dml.getTable(),
-                                mapping.getIndex(),
+                                mapping.get_index(),
                                 pkVal);
                         }
                         esFieldData.remove(pkFieldItem.getFieldName());
@@ -438,7 +438,7 @@ public class ESSyncService {
             logger.trace("Single table insert to es index, destination:{}, table: {}, index: {}, id: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 idVal);
         }
         esTemplate.insert(mapping, idVal, esFieldData);
@@ -461,7 +461,7 @@ public class ESSyncService {
             logger.trace("Main table insert to es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 sql.replace("\n", " "));
         }
         Util.sqlRS(ds, sql, rs -> {
@@ -475,7 +475,7 @@ public class ESSyncService {
                             "Main table insert to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
                             config.getDestination(),
                             dml.getTable(),
-                            mapping.getIndex(),
+                            mapping.get_index(),
                             idVal);
                     }
                     esTemplate.insert(mapping, idVal, esFieldData);
@@ -497,7 +497,7 @@ public class ESSyncService {
             logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 sql.replace("\n", " "));
         }
         Util.sqlRS(ds, sql, rs -> {
@@ -519,7 +519,7 @@ public class ESSyncService {
                             "Main table delete to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
                             config.getDestination(),
                             dml.getTable(),
-                            mapping.getIndex(),
+                            mapping.get_index(),
                             idVal);
                     }
                     esTemplate.delete(mapping, idVal, esFieldData);
@@ -554,7 +554,7 @@ public class ESSyncService {
 
                     String fieldName = fieldItem.getFieldName();
                     // 判断是否是主键
-                    if (fieldName.equals(mapping.getId())) {
+                    if (fieldName.equals(mapping.get_id())) {
                         fieldName = "_id";
                     }
                     paramsTmp.put(fieldName, value);
@@ -566,7 +566,7 @@ public class ESSyncService {
             logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex());
+                mapping.get_index());
         }
         esTemplate.updateByQuery(config, paramsTmp, esFieldData);
     }
@@ -617,7 +617,7 @@ public class ESSyncService {
             logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 sql.toString().replace("\n", " "));
         }
         Util.sqlRS(ds, sql.toString(), values, rs -> {
@@ -661,7 +661,7 @@ public class ESSyncService {
                                     entry.getKey().getColumn().getColumnName());
                                 String fieldName = fieldItem.getFieldName();
                                 // 判断是否是主键
-                                if (fieldName.equals(mapping.getId())) {
+                                if (fieldName.equals(mapping.get_id())) {
                                     fieldName = "_id";
                                 }
                                 paramsTmp.put(fieldName, value);
@@ -673,7 +673,7 @@ public class ESSyncService {
                         logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}",
                             config.getDestination(),
                             dml.getTable(),
-                            mapping.getIndex());
+                            mapping.get_index());
                     }
                     esTemplate.updateByQuery(config, paramsTmp, esFieldData);
                 }
@@ -721,7 +721,7 @@ public class ESSyncService {
             logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 sql.toString().replace("\n", " "));
         }
         Util.sqlRS(ds, sql.toString(), rs -> {
@@ -775,7 +775,7 @@ public class ESSyncService {
                                 .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
                             String fieldName = fieldItem.getFieldName();
                             // 判断是否是主键
-                            if (fieldName.equals(mapping.getId())) {
+                            if (fieldName.equals(mapping.get_id())) {
                                 fieldName = "_id";
                             }
                             paramsTmp.put(fieldName, value);
@@ -787,7 +787,7 @@ public class ESSyncService {
                             "Join table update es index by query whole sql, destination:{}, table: {}, index: {}",
                             config.getDestination(),
                             dml.getTable(),
-                            mapping.getIndex());
+                            mapping.get_index());
                     }
                     esTemplate.updateByQuery(config, paramsTmp, esFieldData);
                 }
@@ -817,7 +817,7 @@ public class ESSyncService {
             logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 idVal);
         }
         esTemplate.update(mapping, idVal, esFieldData);
@@ -840,7 +840,7 @@ public class ESSyncService {
             logger.trace("Main table update to es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
                 config.getDestination(),
                 dml.getTable(),
-                mapping.getIndex(),
+                mapping.get_index(),
                 sql.replace("\n", " "));
         }
         Util.sqlRS(ds, sql, rs -> {
@@ -854,7 +854,7 @@ public class ESSyncService {
                             "Main table update to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
                             config.getDestination(),
                             dml.getTable(),
-                            mapping.getIndex(),
+                            mapping.get_index(),
                             idVal);
                     }
                     esTemplate.update(mapping, idVal, esFieldData);

client-adapter/hbase/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -21,6 +21,7 @@
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-shaded-client</artifactId>
+            <version>1.1.2</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.hbase.config;
 
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -8,6 +7,7 @@ import java.util.Properties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 
 /**
@@ -32,7 +32,8 @@ public class MappingConfigLoader {
 
         Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("hbase");
         configContentMap.forEach((fileName, content) -> {
-            MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
+            MappingConfig config = YmlConfigBinder
+                .bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
             if (config == null) {
                 return;
             }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.hbase.monitor;
 
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter;
 import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
 import java.io.File;
 import java.util.Properties;
 import org.apache.commons.io.filefilter.FileFilterUtils;
@@ -59,7 +59,8 @@ public class HbaseConfigMonitor {
             try {
                 // 加载新增的配置文件
                 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                MappingConfig config = YamlUtils.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                MappingConfig config = YmlConfigBinder
+                    .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
                 if (config == null) {
                     return;
                 }
@@ -87,7 +88,8 @@ public class HbaseConfigMonitor {
                         onFileDelete(file);
                         return;
                     }
-                    MappingConfig config = YamlUtils.ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                    MappingConfig config = YmlConfigBinder
+                        .bindYmlToObj(null, configContent, MappingConfig.class, null, envProperties);
                     if (config == null) {
                         return;
                     }

client-adapter/kudu/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.8-SNAPSHOT</version>
+        <version>1.1.7-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -21,15 +21,18 @@
         <dependency>
             <groupId>org.apache.kudu</groupId>
             <artifactId>kudu-client</artifactId>
+            <version>1.6.0</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
+            <version>4.12</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
+            <version>5.1.48</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.kudu.config;
 
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -8,6 +7,7 @@ import java.util.Properties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 
 /**
@@ -30,7 +30,11 @@ public class KuduMappingConfigLoader {
 
         Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
         configContentMap.forEach((fileName, content) -> {
-            KuduMappingConfig config = YamlUtils.ymlToObj(null, content, KuduMappingConfig.class, null, envProperties);
+            KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
+                content,
+                KuduMappingConfig.class,
+                null,
+                envProperties);
             if (config == null) {
                 return;
             }

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.client.adapter.kudu.monitor;
 
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
 import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
 import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
 import java.io.File;
 import java.util.Properties;
 import org.apache.commons.io.filefilter.FileFilterUtils;
@@ -36,7 +36,7 @@ public class KuduConfigMonitor {
         File confDir = Util.getConfDirPath(adapterName);
         try {
             FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+                    FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
             FileListener listener = new FileListener();
             observer.addListener(listener);
             fileMonitor = new FileAlterationMonitor(3000, observer);
@@ -69,15 +69,19 @@ public class KuduConfigMonitor {
             try {
                 // 加载新增的配置文件
                 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                KuduMappingConfig config = YamlUtils
+                KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
-                    .ymlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
+                    configContent,
+                    KuduMappingConfig.class,
+                    null,
+                    envProperties);
                 if (config == null) {
                     return;
                 }
                 config.validate();
                 boolean result = kuduAdapter.addConfig(file.getName(), config);
                 if (result) {
-                    logger.info("Add a new kudu mapping config: {} to canal adapter", file.getName());
+                    logger.info("Add a new kudu mapping config: {} to canal adapter",
+                            file.getName());
                 }
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
@@ -91,14 +95,17 @@ public class KuduConfigMonitor {
             try {
                 if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
                     // 加载配置文件
-                    String configContent = MappingConfigsLoader
+                    String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
-                        .loadConfig(adapterName + File.separator + file.getName());
+                                                                           + file.getName());
                     if (configContent == null) {
                         onFileDelete(file);
                         return;
                     }
-                    KuduMappingConfig config = YamlUtils
+                    KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
-                        .ymlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
+                        configContent,
+                        KuduMappingConfig.class,
+                        null,
+                        envProperties);
                     if (config == null) {
                         return;
                     }