[maven-release-plugin] prepare release canal-1.1.8

Signed-off-by: BruceAko <chongzhi@hust.edu.cn>

.travis.yml

@@ -14,7 +14,7 @@ cache:
 install: true
 
 script:
-  - travis_wait 30 ./mvnw clean install -DskipTests=false -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true
+  - travis_wait 30 ./mvnw -T 1C clean install -DskipTests=false -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

README.md

@@ -4,6 +4,8 @@
 ![license](https://img.shields.io/github/license/alibaba/canal.svg)
 [![average time to resolve an issue](http://isitmaintained.com/badge/resolution/alibaba/canal.svg)](http://isitmaintained.com/project/alibaba/canal "average time to resolve an issue")
 [![percentage of issues still open](http://isitmaintained.com/badge/open/alibaba/canal.svg)](http://isitmaintained.com/project/alibaba/canal "percentage of issues still open")
+[![Leaderboard](https://img.shields.io/badge/Canal-%E6%9F%A5%E7%9C%8B%E8%B4%A1%E7%8C%AE%E6%8E%92%E8%A1%8C%E6%A6%9C-orange)](https://opensource.alibaba.com/contribution_leaderboard/details?projectValue=canal)
+
 
 ## 简介
 
@@ -85,15 +87,25 @@ canal 特别设计了 client-server 模式，交互协议使用 protobuf 3.0 , c
 - canal php客户端: [https://github.com/xingwenge/canal-php](https://github.com/xingwenge/canal-php)
 - canal Python客户端：[https://github.com/haozi3156666/canal-python](https://github.com/haozi3156666/canal-python)
 - canal Rust客户端：[https://github.com/laohanlinux/canal-rs](https://github.com/laohanlinux/canal-rs)
+- canal Nodejs客户端：[https://github.com/marmot-z/canal-nodejs](https://github.com/marmot-z/canal-nodejs)
 
 canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递到 MQ 系统中，比如 Kafka/RocketMQ，可以借助于 MQ 的多语言能力 
 
 - 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart)
 
-## 相关开源
+## 基于canal开发的工具
+
+-  canal2sql(基于binlog生成SQL) : [https://github.com/zhuchao941/canal2sql]
+
+## 相关开源&产品
 
 - [canal 消费端开源项目: Otter](http://github.com/alibaba/otter)
 - [阿里巴巴去 Oracle 数据迁移同步工具: yugong](http://github.com/alibaba/yugong)
+- [阿里巴巴离线同步开源项目 DataX](https://github.com/alibaba/datax)
+- [阿里巴巴数据库连接池开源项目 Druid](https://github.com/alibaba/druid)
+- [阿里巴巴实时数据同步工具 DTS](https://www.aliyun.com/product/dts)
 
 ## 问题反馈
 - 报告 issue: [github issues](https://github.com/alibaba/canal/issues)
+
+本项目的Issues会被同步沉淀至[阿里云开发者社区](https://developer.aliyun.com/ask)

admin/admin-ui/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <artifactId>canal.admin</artifactId>
     <groupId>com.alibaba.otter</groupId>
-    <version>1.1.5-SNAPSHOT</version>
+    <version>1.1.8</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

admin/admin-ui/src/views/canalServer/CanalInstanceUpdate.vue

@@ -59,7 +59,7 @@ export default {
       canalInstanceDetail(this.$route.query.id).then(response => {
         const data = response.data
         this.form.id = data.id
-        this.form.name = data.name + '/instance.propertios'
+        this.form.name = data.name + '/instance.properties'
         this.form.content = data.content
         this.form.clusterServerId = data.clusterServerId
       })

admin/admin-ui/src/views/login/index.vue

@@ -66,8 +66,8 @@ export default {
       }
     }
     const validatePassword = (rule, value, callback) => {
-      if (value.length < 6) {
+      if (value.length < 5) {
-        callback(new Error('The password can not be less than 6 digits'))
+        callback(new Error('The password can not be less than 5 digits'))
       } else {
         callback()
       }

admin/admin-web/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.admin</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>canal.admin-web</artifactId>
@@ -30,6 +30,8 @@
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
+            <!-- 支持arm64需要mysql 8.0,升级为8.0的新版驱动 -->
+            <version>8.0.33</version>
         </dependency>
         <dependency>
             <groupId>com.github.ben-manes.caffeine</groupId>

admin/admin-web/src/main/bin/restart.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
-sh stop.sh
+case $(uname) in
+Linux)
+  bin_abs_path=$(readlink -f $(dirname $0))
+  ;;
+*)
+  bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
+  ;;
+esac
 
-sh startup.sh
+sh "$bin_abs_path"/stop.sh
+sh "$bin_abs_path"/startup.sh

admin/admin-web/src/main/bin/startup.sh

@@ -56,14 +56,29 @@ in
   exit;;
 esac
 
+JavaVersion=`$JAVA -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }' | awk  -F '.' '{print $1}'`
 str=`file -L $JAVA | grep 64-bit`
-if [ -n "$str" ]; then
+
-	JAVA_OPTS="-server -Xms2048m -Xmx3072m"
+JAVA_OPTS="$JAVA_OPTS -Xss1m -XX:+AggressiveOpts -XX:-UseBiasedLocking -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$base/logs"
+if [ $JavaVersion -ge 11 ] ; then
+  #JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$base_log/gc.log:time "
+  JAVA_OPTS="$JAVA_OPTS"
 else
-	JAVA_OPTS="-server -Xms1024m -Xmx1024m"
+  #JAVA_OPTS="$JAVA_OPTS -Xloggc:$base/logs/canal/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime"
+  JAVA_OPTS="$JAVA_OPTS -XX:+UseFastAccessorMethods -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
+fi
+
+if [ -n "$str" ]; then
+	if [ $JavaVersion -ge 11 ] ; then
+    # For G1
+    JAVA_OPTS="-server -Xms2g -Xmx3g -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent $JAVA_OPTS"
+  else
+	  JAVA_OPTS="-server -Xms2g -Xmx3g -Xmn1g -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC $JAVA_OPTS"
+	fi
+else
+	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m $JAVA_OPTS"
 fi
 
-JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -XX:+PrintAdaptiveSizePolicy -XX:+PrintTenuringDistribution"
 JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
 CANAL_OPTS="-DappName=canal-admin"
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/common/Threads.java

@@ -12,6 +12,6 @@ public class Threads {
                                                             pool,
                                                             0L,
                                                             TimeUnit.MILLISECONDS,
-                                                            new ArrayBlockingQueue<Runnable>(pool * 20),
+                                                            new ArrayBlockingQueue<>(pool * 20),
                                                             DaemonThreadFactory.daemonThreadFactory);
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/EbeanConfig.java

@@ -1,18 +1,19 @@
 package com.alibaba.otter.canal.admin.config;
 
+import io.ebean.EbeanServer;
+import io.ebean.EbeanServerFactory;
+import io.ebean.config.ServerConfig;
+import io.ebean.config.UnderscoreNamingConvention;
+
 import java.util.ArrayList;
 import java.util.List;
 
 import javax.sql.DataSource;
 
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import io.ebean.EbeanServer;
-import io.ebean.EbeanServerFactory;
-import io.ebean.config.ServerConfig;
-import io.ebean.config.UnderscoreNamingConvention;
-
 /**
  * Ebean 配置
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/WebConfig.java

@@ -5,6 +5,8 @@ import java.io.PrintWriter;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
@@ -26,8 +28,24 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 @Configuration
 public class WebConfig implements WebMvcConfigurer {
 
+    @Value(value = "${canal.adminUser}")
+    private String user;
+
+    @Value(value = "${canal.adminPasswd}")
+    private String passwd;
+
     @Override
     public void addInterceptors(InterceptorRegistry registry) {
+        if (StringUtils.isEmpty(user)) {
+            throw new IllegalArgumentException(
+                "canal.adminUser is empty , pls check https://github.com/alibaba/canal/issues/4941");
+        }
+
+        if (StringUtils.isEmpty(passwd)) {
+            throw new IllegalArgumentException(
+                "canal.adminPasswd is empty , pls check https://github.com/alibaba/canal/issues/4941");
+        }
+
         registry.addInterceptor(new HandlerInterceptor() {
 
             @Override
@@ -84,7 +102,9 @@ public class WebConfig implements WebMvcConfigurer {
             }
         })
             .addPathPatterns("/api/**")
-            .excludePathPatterns("/api/**/config/**")
+            .excludePathPatterns("/api/**/config/server_polling")
+            .excludePathPatterns("/api/**/config/instances_polling")
+            .excludePathPatterns("/api/**/config/instance_polling/**")
             .excludePathPatterns("/api/**/user/login")
             .excludePathPatterns("/api/**/user/logout")
             .excludePathPatterns("/api/**/user/info");

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnector.java

@@ -108,9 +108,7 @@ public class SimpleAdminConnector implements AdminConnector {
             }
 
             connected = true;
-        } catch (IOException e) {
+        } catch (IOException | NoSuchAlgorithmException e) {
-            throw new ServiceException(e);
-        } catch (NoSuchAlgorithmException e) {
             throw new ServiceException(e);
         }
     }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java

@@ -2,11 +2,13 @@ package com.alibaba.otter.canal.admin.connector;
 
 import java.util.function.Function;
 
-import com.alibaba.otter.canal.admin.config.SpringContext;
+import org.assertj.core.util.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.core.env.Environment;
 
+import com.alibaba.otter.canal.admin.config.SpringContext;
+
 public class SimpleAdminConnectors {
 
     private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnectors.class);
@@ -25,7 +27,8 @@ public class SimpleAdminConnectors {
             connector.connect();
             return function.apply(connector);
         } catch (Exception e) {
-            logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user,passwd);
+            logger.error("connect to ip:{},port:{},user:{},password:{}, failed",
+                    ip, port, user, getDesensitizationPassword(passwd));
             logger.error(e.getMessage());
         } finally {
             connector.disconnect();
@@ -33,4 +36,14 @@ public class SimpleAdminConnectors {
 
         return null;
     }
+
+    private static String getDesensitizationPassword(String password) {
+        String defaultPassword = "*****";
+
+        if (Strings.isNullOrEmpty(password) || password.length() < 5) {
+            return defaultPassword;
+        } else {
+            return String.format("%s*****", password.substring(0, 3));
+        }
+    }
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalClusterController.java

@@ -4,10 +4,17 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
 
-import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson2.JSONArray;
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson2.JSONObject;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalCluster;
 import com.alibaba.otter.canal.admin.model.NodeServer;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalConfigController.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.admin.controller;
 
-import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -9,6 +8,7 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalConfig;
 import com.alibaba.otter.canal.admin.service.CanalConfigService;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.admin.controller;
 import java.util.List;
 import java.util.Map;
 
-import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -15,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 import com.alibaba.otter.canal.admin.model.Pager;
@@ -143,8 +143,7 @@ public class CanalInstanceController {
      * @return 是否成功
      */
     @PutMapping(value = "/instance/status/{id}")
-    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option,
+    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option, @PathVariable String env) {
-                                            @PathVariable String env) {
         return BaseModel.getInstance(canalInstanceConfigService.instanceOperation(id, option));
     }
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/PollingConfigController.java

@@ -2,15 +2,11 @@ package com.alibaba.otter.canal.admin.controller;
 
 import java.security.NoSuchAlgorithmException;
 
+import org.apache.commons.lang.RandomStringUtils;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.*;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestHeader;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
 
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalConfig;
@@ -28,7 +24,7 @@ import com.alibaba.otter.canal.protocol.SecurityUtil;
 @RequestMapping("/api/{env}/config")
 public class PollingConfigController {
 
-    private static final byte[] seeds = "canal is best!".getBytes();
+    private static final byte[] seeds = RandomStringUtils.random(16).getBytes();
 
     @Value(value = "${canal.adminUser}")
     String                      user;
@@ -46,14 +42,15 @@ public class PollingConfigController {
     public BaseModel<CanalConfig> canalConfigPoll(@RequestHeader String user, @RequestHeader String passwd,
                                                   @RequestParam String ip, @RequestParam Integer port,
                                                   @RequestParam String md5, @RequestParam boolean register,
-                                                  @RequestParam String cluster, @PathVariable String env) {
+                                                  @RequestParam String cluster, @RequestParam String name,
+                                                  @PathVariable String env) {
         if (!auth(user, passwd)) {
             throw new RuntimeException("auth :" + user + " is failed");
         }
 
         if (StringUtils.isEmpty(md5) && register) {
             // do something
-            pollingConfigService.autoRegister(ip, port, cluster);
+            pollingConfigService.autoRegister(ip, port, cluster, StringUtils.trimToNull(name));
         }
 
         CanalConfig canalConfig = pollingConfigService.getChangedConfig(ip, port, md5);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/UserController.java

@@ -1,20 +1,28 @@
 package com.alibaba.otter.canal.admin.controller;
 
-import com.alibaba.otter.canal.admin.service.UserService;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-
-import com.alibaba.otter.canal.admin.model.BaseModel;
-import com.alibaba.otter.canal.admin.model.User;
-
-import javax.servlet.http.HttpServletRequest;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
+import javax.servlet.http.HttpServletRequest;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.alibaba.otter.canal.admin.model.BaseModel;
+import com.alibaba.otter.canal.admin.model.User;
+import com.alibaba.otter.canal.admin.service.UserService;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+
 /**
  * 用户管理控制层
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/handler/CustomExceptionHandler.java

@@ -1,7 +1,5 @@
 package com.alibaba.otter.canal.admin.handler;
 
-import com.alibaba.otter.canal.admin.common.exception.ServiceException;
-import com.alibaba.otter.canal.admin.model.BaseModel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
@@ -10,6 +8,9 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
+import com.alibaba.otter.canal.admin.common.exception.ServiceException;
+import com.alibaba.otter.canal.admin.model.BaseModel;
+
 /**
  * MVC异常拦截器
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalCluster.java

@@ -3,10 +3,11 @@ package com.alibaba.otter.canal.admin.model;
 import io.ebean.Finder;
 import io.ebean.annotation.WhenModified;
 
+import java.util.Date;
+
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
-import java.util.Date;
 
 /**
  * Canal集群信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalInstanceConfig.java

@@ -1,12 +1,18 @@
 package com.alibaba.otter.canal.admin.model;
 
-import java.util.Date;
-
-import javax.persistence.*;
-
 import io.ebean.Finder;
 import io.ebean.annotation.WhenModified;
 
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.Transient;
+
 /**
  * Canal实例配置信息实体类
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Model.java

@@ -1,13 +1,13 @@
 package com.alibaba.otter.canal.admin.model;
 
+import io.ebean.Ebean;
+import io.ebean.EbeanServer;
+
 import javax.persistence.MappedSuperclass;
 import javax.persistence.OptimisticLockException;
 
 import org.apache.commons.beanutils.PropertyUtils;
 
-import io.ebean.Ebean;
-import io.ebean.EbeanServer;
-
 /**
  * EBean Model扩展类
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/NodeServer.java

@@ -5,7 +5,13 @@ import io.ebean.annotation.WhenModified;
 
 import java.util.Date;
 
-import javax.persistence.*;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.Table;
 
 /**
  * 节点信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Pager.java

@@ -9,7 +9,7 @@ public class Pager<T> implements Serializable {
     private static final long serialVersionUID = -986577815091763517L;
 
     private Long              count            = 0L;
-    private List<T>           items            = new ArrayList<T>();
+    private List<T>           items            = new ArrayList<>();
     private Integer           page             = 1;
     private Integer           size             = 20;
     private Long              offset           = 0L;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/User.java

@@ -3,11 +3,12 @@ package com.alibaba.otter.canal.admin.model;
 import io.ebean.Finder;
 import io.ebean.annotation.WhenCreated;
 
+import java.util.Date;
+
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
 import javax.persistence.Transient;
-import java.util.Date;
 
 /**
  * 用户信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/NodeServerService.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.admin.service;
 
+import java.util.List;
+
 import com.alibaba.otter.canal.admin.model.NodeServer;
 import com.alibaba.otter.canal.admin.model.Pager;
 
-import java.util.List;
-
 public interface NodeServerService {
 
     void save(NodeServer nodeServer);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/PollingConfigService.java

@@ -5,7 +5,7 @@ import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 
 public interface PollingConfigService {
 
-    public boolean autoRegister(String ip, Integer adminPort, String cluster);
+    public boolean autoRegister(String ip, Integer adminPort, String cluster, String name);
 
     CanalConfig getChangedConfig(String ip, Integer port, String md5);
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/PollingConfigServiceImpl.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.admin.service.impl;
 
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang.StringUtils;
@@ -28,11 +29,11 @@ public class PollingConfigServiceImpl implements PollingConfigService {
     @Autowired
     CanalClusterService canalClusterService;
 
-    public boolean autoRegister(String ip, Integer adminPort, String cluster) {
+    public boolean autoRegister(String ip, Integer adminPort, String cluster, String name) {
         NodeServer server = NodeServer.find.query().where().eq("ip", ip).eq("adminPort", adminPort).findOne();
         if (server == null) {
             server = new NodeServer();
-            server.setName(ip);
+            server.setName(Optional.ofNullable(name).orElse(ip));
             server.setIp(ip);
             server.setAdminPort(adminPort);
             server.setTcpPort(adminPort + 1);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/UserServiceImpl.java

@@ -21,6 +21,8 @@ public class UserServiceImpl implements UserService {
 
     private static byte[] seeds = "canal is best!".getBytes();
 
+    private static final Integer PASSWORD_LENGTH = 5;
+
     public User find4Login(String username, String password) {
         if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) {
             return null;
@@ -43,6 +45,9 @@ public class UserServiceImpl implements UserService {
     }
 
     public void update(User user) {
+        if (user.getPassword().length() < PASSWORD_LENGTH) {
+            throw new ServiceException("The new password is too short,must more than 6 digits");
+        }
         User userTmp = User.find.query().where().eq("username", user.getUsername()).findOne();
         if (userTmp == null) {
             throw new ServiceException();

admin/admin-web/src/main/resources/application.yml

@@ -18,4 +18,4 @@ spring.datasource:
 
 canal:
   adminUser: admin
-  adminPasswd: admin
+  adminPasswd:

admin/admin-web/src/main/resources/canal-template.properties

@@ -8,20 +8,24 @@ canal.register.ip =
 canal.port = 11111
 canal.metrics.pull.port = 11112
 # canal instance user/passwd
-canal.user = canal
+# canal.user = canal
-canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
+# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
 
 # canal admin config
-canal.admin.manager = 127.0.0.1:8089
+#canal.admin.manager = 127.0.0.1:8089
 canal.admin.port = 11110
 canal.admin.user = admin
 canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+# admin auto register
+#canal.admin.register.auto = true
+#canal.admin.register.cluster =
+#canal.admin.register.name =
 
 canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, RocketMQ
+# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -60,6 +64,9 @@ canal.instance.filter.query.ddl = false
 canal.instance.filter.table.error = false
 canal.instance.filter.rows = false
 canal.instance.filter.transaction.entry = false
+canal.instance.filter.dml.insert = false
+canal.instance.filter.dml.update = false
+canal.instance.filter.dml.delete = false
 
 # binlog format/image check
 canal.instance.binlog.format = ROW,STATEMENT,MIXED 
@@ -86,10 +93,6 @@ canal.instance.tsdb.snapshot.interval = 24
 # purge snapshot expire , default 360 hour(15 days)
 canal.instance.tsdb.snapshot.expire = 360
 
-# aliyun ak/sk , support rds/mq
-canal.aliyun.accessKey =
-canal.aliyun.secretKey =
-
 #################################################
 ######### 		destinations		#############
 #################################################
@@ -99,6 +102,9 @@ canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
 canal.auto.scan = true
 canal.auto.scan.interval = 5
+# set this value to 'true' means that when binlog pos not found, skip to latest.
+# WARN: pls keep 'false' in production env, or if you know what you want.
+canal.auto.reset.latest.pos.mode = false
 
 canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
 #canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
@@ -111,29 +117,71 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml
 
 ##################################################
-######### 		     MQ 		     #############
+######### 	      MQ Properties      #############
 ##################################################
-canal.mq.servers = 127.0.0.1:6667
+# aliyun ak/sk , support rds/mq
-canal.mq.retries = 0
+canal.aliyun.accessKey =
-canal.mq.batchSize = 16384
+canal.aliyun.secretKey =
-canal.mq.maxRequestSize = 1048576
+canal.aliyun.uid=
-canal.mq.lingerMs = 100
+
-canal.mq.bufferMemory = 33554432
+canal.mq.flatMessage = true
 canal.mq.canalBatchSize = 50
 canal.mq.canalGetTimeout = 100
-canal.mq.flatMessage = true
-canal.mq.compressionType = none
-canal.mq.acks = all
-#canal.mq.properties. =
-canal.mq.producerGroup = test
 # Set this value to "cloud", if you want open message trace feature in aliyun.
 canal.mq.accessChannel = local
-# aliyun mq namespace
+
-#canal.mq.namespace =
+canal.mq.database.hash = true
+canal.mq.send.thread.size = 30
+canal.mq.build.thread.size = 8
 
 ##################################################
-#########     Kafka Kerberos Info    #############
+######### 		     Kafka 		     #############
 ##################################################
-canal.mq.kafka.kerberos.enable = false
+kafka.bootstrap.servers = 127.0.0.1:6667
-canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
+kafka.acks = all
-canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
+kafka.compression.type = none
+kafka.batch.size = 16384
+kafka.linger.ms = 1
+kafka.max.request.size = 1048576
+kafka.buffer.memory = 33554432
+kafka.max.in.flight.requests.per.connection = 1
+kafka.retries = 0
+
+kafka.kerberos.enable = false
+kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
+kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf
+
+# sasl demo
+# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
+# kafka.sasl.mechanism = SCRAM-SHA-512
+# kafka.security.protocol = SASL_PLAINTEXT
+
+##################################################
+######### 		    RocketMQ	     #############
+##################################################
+rocketmq.producer.group = test
+rocketmq.enable.message.trace = false
+rocketmq.customized.trace.topic =
+rocketmq.namespace =
+rocketmq.namesrv.addr = 127.0.0.1:9876
+rocketmq.retry.times.when.send.failed = 0
+rocketmq.vip.channel.enabled = false
+rocketmq.tag =
+
+##################################################
+######### 		    RabbitMQ	     #############
+##################################################
+rabbitmq.host =
+rabbitmq.virtual.host =
+rabbitmq.exchange =
+rabbitmq.username =
+rabbitmq.password =
+rabbitmq.deliveryMode =
+
+
+##################################################
+######### 		      Pulsar         #############
+##################################################
+pulsarmq.serverUrl =
+pulsarmq.roleToken =
+pulsarmq.topicTenantPrefix =

admin/admin-web/src/main/resources/canal_manager.sql

@@ -11,8 +11,8 @@ SET FOREIGN_KEY_CHECKS = 0;
 DROP TABLE IF EXISTS `canal_adapter_config`;
 CREATE TABLE `canal_adapter_config` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `category` varchar(45) NOT NULL,
+  `category` varchar(255) NOT NULL,
-  `name` varchar(45) NOT NULL,
+  `name` varchar(255) NOT NULL,
   `status` varchar(45) DEFAULT NULL,
   `content` text NOT NULL,
   `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
@@ -25,7 +25,7 @@ CREATE TABLE `canal_adapter_config` (
 DROP TABLE IF EXISTS `canal_cluster`;
 CREATE TABLE `canal_cluster` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `name` varchar(63) NOT NULL,
+  `name` varchar(255) NOT NULL,
   `zk_hosts` varchar(255) NOT NULL,
   `modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`)
@@ -39,7 +39,7 @@ CREATE TABLE `canal_config` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
   `cluster_id` bigint(20) DEFAULT NULL,
   `server_id` bigint(20) DEFAULT NULL,
-  `name` varchar(45) NOT NULL,
+  `name` varchar(255) NOT NULL,
   `status` varchar(45) DEFAULT NULL,
   `content` text NOT NULL,
   `content_md5` varchar(128) NOT NULL,
@@ -56,7 +56,7 @@ CREATE TABLE `canal_instance_config` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
   `cluster_id` bigint(20) DEFAULT NULL,
   `server_id` bigint(20) DEFAULT NULL,
-  `name` varchar(45) NOT NULL,
+  `name` varchar(255) NOT NULL,
   `status` varchar(45) DEFAULT NULL,
   `content` text NOT NULL,
   `content_md5` varchar(128) DEFAULT NULL,
@@ -72,8 +72,8 @@ DROP TABLE IF EXISTS `canal_node_server`;
 CREATE TABLE `canal_node_server` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
   `cluster_id` bigint(20) DEFAULT NULL,
-  `name` varchar(63) NOT NULL,
+  `name` varchar(255) NOT NULL,
-  `ip` varchar(63) NOT NULL,
+  `ip` varchar(255) NOT NULL,
   `admin_port` int(11) DEFAULT NULL,
   `tcp_port` int(11) DEFAULT NULL,
   `metric_port` int(11) DEFAULT NULL,
@@ -88,10 +88,10 @@ CREATE TABLE `canal_node_server` (
 DROP TABLE IF EXISTS `canal_user`;
 CREATE TABLE `canal_user` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `username` varchar(31) NOT NULL,
+  `username` varchar(255) NOT NULL,
-  `password` varchar(128) NOT NULL,
+  `password` varchar(255) NOT NULL,
-  `name` varchar(31) NOT NULL,
+  `name` varchar(255) NOT NULL,
-  `roles` varchar(31) NOT NULL,
+  `roles` varchar(255) NOT NULL,
   `introduction` varchar(255) DEFAULT NULL,
   `avatar` varchar(255) DEFAULT NULL,
   `creation_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,

admin/pom.xml

@@ -1,9 +1,15 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>com.alibaba.otter</groupId>
+        <artifactId>canal</artifactId>
+        <version>1.1.8</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.admin</artifactId>
-    <version>1.1.5-SNAPSHOT</version>
+    <version>1.1.8</version>
     <packaging>pom</packaging>
     <name>canal admin module for otter ${project.version}</name>
     <modules>
@@ -25,71 +31,29 @@
             <dependency>
                 <groupId>org.springframework.cloud</groupId>
                 <artifactId>spring-cloud-context</artifactId>
-                <version>2.0.0.RELEASE</version>
+                <version>3.0.6</version>
             </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-core</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-aop</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-context</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-jdbc</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-orm</artifactId>
-                <version>5.0.5.RELEASE</version>
-            </dependency>
-            <dependency>
-                <groupId>org.springframework</groupId>
-                <artifactId>spring-test</artifactId>
-                <version>5.0.5.RELEASE</version>
-                <scope>test</scope>
-            </dependency>
-
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>
-                <version>2.0.1.RELEASE</version>
+                <version>2.5.15</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
-            <dependency>
-                <groupId>mysql</groupId>
-                <artifactId>mysql-connector-java</artifactId>
-                <version>8.0.17</version>
-            </dependency>
             <dependency>
                 <groupId>com.github.ben-manes.caffeine</groupId>
                 <artifactId>caffeine</artifactId>
                 <version>2.6.2</version>
             </dependency>
-            <dependency>
-                <groupId>commons-beanutils</groupId>
-                <artifactId>commons-beanutils</artifactId>
-                <version>1.8.2</version>
-            </dependency>
-            <dependency>
-                <groupId>commons-lang</groupId>
-                <artifactId>commons-lang</artifactId>
-                <version>2.6</version>
-            </dependency>
             <dependency>
                 <groupId>io.ebean</groupId>
                 <artifactId>ebean</artifactId>
-                <version>11.41.1</version>
+                <version>11.45.1</version>
+            </dependency>
+            <dependency>
+                <groupId>org.yaml</groupId>
+                <artifactId>snakeyaml</artifactId>
+                <version>2.0</version>
             </dependency>
         </dependencies>
     </dependencyManagement>

charts/README.md

@@ -0,0 +1,160 @@
+# helm charts 使用说明
+
+包含`canal-admin`和`canal-server`两个chart，基本配置保存在k8s的`secret`里，`canal.properties`和`instance.properties`保存在mysql数据库中，使用zookeeper注册中心实现高可用。
+
+- [helm charts 使用说明](#helm-charts-使用说明)
+  - [创建zk集群](#创建zk集群)
+  - [初始化数据库](#初始化数据库)
+  - [部署canal-admin chart](#部署canal-admin-chart)
+    - [配置values.yaml](#配置valuesyaml)
+    - [配置集群信息](#配置集群信息)
+  - [部署canal-server chart](#部署canal-server-chart)
+    - [配置values.yaml](#配置valuesyaml-1)
+  - [instance 实例创建](#instance-实例创建)
+
+## 创建zk集群
+
+```sh
+helm install canal-zookeeper oci://registry-1.docker.io/bitnamicharts/zookeeper
+```
+
+如需挂载磁盘参考 [bitnami/zookeeper](`https://github.com/bitnami/charts/tree/main/bitnami/zookeeper`)
+
+
+## 初始化数据库
+
+[/admin/admin-web/src/main/resources/canal_manager.sql](/admin/admin-web/src/main/resources/canal_manager.sql)
+
+[/deployer/src/main/resources/spring/tsdb/sql/create_table.sql](/deployer/src/main/resources/spring/tsdb/sql/create_table.sql)
+
+## 部署canal-admin chart
+
+### 配置values.yaml
+
+```yaml
+# 主要配置
+admin:
+  config: |
+    server:
+      port: 8089
+    spring:
+      jackson:
+        date-format: yyyy-MM-dd HH:mm:ss
+        time-zone: GMT+8
+    spring.datasource:
+      address: your_db_host:3306
+      database: canal_manager
+      username: ****
+      password: ****
+      driver-class-name: com.mysql.jdbc.Driver
+      url: jdbc:mysql://${spring.datasource.address}/${spring.datasource.database}?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
+      hikari:
+        maximum-pool-size: 30
+        minimum-idle: 1
+    canal:
+      adminUser: admin
+      adminPasswd: admin
+    # debug: true
+```
+
+全量参数见：[canal-admin/values.yaml](./canal-admin/values.yaml)，测试用例见：[tests/admin-values.yaml](./tests/admin-values.yaml)
+
+```sh
+helm install canal-admin -f ./admin-values.yaml ./canal-admin
+```
+
+### 配置集群信息
+
+![cluster](./images/cluster.png)
+
+等待pod启动后，使用默认账户`admin`密码`123456`登录
+
+创建集群`default`并配置`zk地址`为`canal-zookeeper.default.svc.cluster.local:2181`
+
+配置完成后，点击`配置`-`主配置`，修改以下配置项，不是覆盖整个文件
+
+```yaml
+canal.user = admin
+canal.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+
+canal.zkServers = canal-zookeeper.default.svc.cluster.local:2181
+# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ 看实际需求，此处改为了rabbitMQ
+canal.serverMode = rabbitMQ
+
+#canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
+canal.instance.tsdb.url = jdbc:mysql://you_db_host:3306/canal_tsdb?useUnicode=true&characterEncoding=UTF-8&useSSL=false
+canal.instance.tsdb.dbUsername = ****
+canal.instance.tsdb.dbPassword = ****
+
+#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
+canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
+
+#canal.instance.global.spring.xml = classpath:spring/file-instance.xml
+canal.instance.global.spring.xml = classpath:spring/default-instance.xml
+
+# 如果用的是阿里云polardb需要配置
+canal.aliyun.accessKey = ****
+canal.aliyun.secretKey = ****
+canal.aliyun.uid = 主账号id
+
+# 我们是推送到rabbitmq所以需要配置
+rabbitmq.host = ****:5672
+rabbitmq.virtual.host = ****
+rabbitmq.exchange = ****
+# canal.aliyun.accessKey
+rabbitmq.username = ****
+# canal.aliyun.secretKey
+rabbitmq.password = ****
+rabbitmq.deliveryMode = direct
+```
+
+## 部署canal-server chart
+
+### 配置values.yaml
+
+```yaml
+# 主要配置
+server:
+  config: |
+    canal.port = 11111
+    canal.metrics.pull.port = 11112
+
+    # register ip
+    canal.register.ip =
+
+    # canal admin config
+    canal.admin.manager = canal-admin.default:8089
+    canal.admin.port = 11110
+    canal.admin.user = admin
+    canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+    # admin auto register
+    canal.admin.register.auto = true
+    canal.admin.register.cluster = default
+```
+
+```sh
+helm install canal-server -f ./server-values.yaml ./canal-server
+```
+
+全量参数见：[canal-server/values.yaml](./canal-server/values.yaml)，测试用例见：[tests/server-values.yaml](./tests/server-values.yaml)
+
+![server](./images/server.png)
+
+至此，canal-server已自动创建，并自动注册。
+
+## instance 实例创建
+
+主要配置如下
+
+```yaml
+# 配置你要监听的数据库
+canal.instance.master.address=127.0.0.1:3306
+canal.instance.dbUsername=****
+canal.instance.dbPassword=****
+
+# 过滤监听哪些表
+canal.instance.filter.regex=.*\\..*
+
+# 如果是推送到rabbitmq，需要配置 Routing Key
+canal.mq.topic=你的Routing Key
+```

charts/canal-admin/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/canal-admin/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: canal-admin
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 1.1.8-alpha-3
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

charts/canal-admin/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "canal-admin.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "canal-admin.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "canal-admin.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "canal-admin.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

charts/canal-admin/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "canal-admin.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "canal-admin.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "canal-admin.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "canal-admin.labels" -}}
+helm.sh/chart: {{ include "canal-admin.chart" . }}
+{{ include "canal-admin.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "canal-admin.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "canal-admin.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "canal-admin.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "canal-admin.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

charts/canal-admin/templates/deployment.yaml

@@ -0,0 +1,75 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "canal-admin.fullname" . }}
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "canal-admin.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "canal-admin.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "canal-admin.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            {{- toYaml .Values.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml .Values.readinessProbe | nindent 12 }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - mountPath: /home/admin/canal-admin/conf/application.yml
+              name: secret
+              subPath: application.yml
+            {{- with .Values.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+      volumes:
+        - name: secret
+          secret:
+            defaultMode: 420
+            secretName: {{ include "canal-admin.fullname" . }}
+        {{- with .Values.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/canal-admin/templates/hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "canal-admin.fullname" . }}
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "canal-admin.fullname" . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/canal-admin/templates/ingress.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "canal-admin.fullname" . }}
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.ingress.className }}
+  ingressClassName: {{ . }}
+  {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- with .pathType }}
+            pathType: {{ . }}
+            {{- end }}
+            backend:
+              service:
+                name: {{ include "canal-admin.fullname" $ }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- end }}
+    {{- end }}
+{{- end }}

charts/canal-admin/templates/secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "canal-admin.fullname" . }}
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+type: Opaque
+data:
+  application.yml: {{tpl .Values.admin.config $ | b64enc | quote }}

charts/canal-admin/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "canal-admin.fullname" . }}
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "canal-admin.selectorLabels" . | nindent 4 }}

charts/canal-admin/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "canal-admin.serviceAccountName" . }}
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

charts/canal-admin/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "canal-admin.fullname" . }}-test-connection"
+  labels:
+    {{- include "canal-admin.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "canal-admin.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

charts/canal-admin/values.yaml

@@ -0,0 +1,146 @@
+# Default values for canal-admin.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: canal/canal-admin
+  # This sets the pull policy for images.
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+# This is for the secretes for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: ""
+fullnameOverride: ""
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: false
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: false
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  port: 8089
+
+# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+livenessProbe:
+  httpGet:
+    path: /
+    port: http
+readinessProbe:
+  httpGet:
+    path: /
+    port: http
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+admin:
+  config: |
+    server:
+      port: 8089
+    spring:
+      jackson:
+        date-format: yyyy-MM-dd HH:mm:ss
+        time-zone: GMT+8
+    spring.datasource:
+      address: you_db_host:3306
+      database: canal_manager
+      username: ****
+      password: ****
+      driver-class-name: com.mysql.jdbc.Driver
+      url: jdbc:mysql://${spring.datasource.address}/${spring.datasource.database}?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
+      hikari:
+        maximum-pool-size: 30
+        minimum-idle: 1
+    canal:
+      adminUser: admin
+      adminPasswd: admin
+    # debug: true

charts/canal-server/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/canal-server/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: canal-server
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 1.1.8-alpha-3
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

charts/canal-server/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "canal-server.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "canal-server.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "canal-server.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "canal-server.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

charts/canal-server/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "canal-server.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "canal-server.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "canal-server.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "canal-server.labels" -}}
+helm.sh/chart: {{ include "canal-server.chart" . }}
+{{ include "canal-server.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "canal-server.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "canal-server.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "canal-server.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "canal-server.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

charts/canal-server/templates/hpa.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "canal-server.fullname" . }}
+  labels:
+    {{- include "canal-server.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "canal-server.fullname" . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/canal-server/templates/secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "canal-server.fullname" . }}
+  labels:
+    {{- include "canal-server.labels" . | nindent 4 }}
+type: Opaque
+data:
+  canal.properties: {{tpl .Values.server.config $ | b64enc | quote }}

charts/canal-server/templates/service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-headless" (include "canal-server.fullname" .) | trunc 63 | trimSuffix "-" }}
+  labels:
+    {{- include "canal-server.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.ports.admin }}
+      targetPort: admin
+      protocol: TCP
+      name: tcp-admin
+    - port: {{ .Values.service.ports.tcp }}
+      targetPort: tcp
+      protocol: TCP
+      name: tcp-tcp
+    - port: {{ .Values.service.ports.metric }}
+      targetPort: metric
+      protocol: TCP
+      name: tcp-metric
+  selector:
+    {{- include "canal-server.selectorLabels" . | nindent 4 }}

charts/canal-server/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "canal-server.serviceAccountName" . }}
+  labels:
+    {{- include "canal-server.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

charts/canal-server/templates/statefulsets.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "canal-server.fullname" . }}
+  labels:
+    {{- include "canal-server.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ printf "%s-headless" (include "canal-server.fullname" .) | trunc 63 | trimSuffix "-" }}
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "canal-server.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "canal-server.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "canal-server.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: admin
+              containerPort: {{ .Values.service.ports.admin }}
+              protocol: TCP
+            - name: tcp
+              containerPort: {{ .Values.service.ports.tcp }}
+              protocol: TCP
+            - name: metric
+              containerPort: {{ .Values.service.ports.metric }}
+              protocol: TCP
+          livenessProbe:
+            {{- toYaml .Values.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml .Values.readinessProbe | nindent 12 }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - mountPath: /home/admin/canal-server/conf/canal.properties
+              name: secret
+              subPath: canal.properties
+            {{- with .Values.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+      volumes:
+        - name: secret
+          secret:
+            defaultMode: 420
+            secretName: {{ include "canal-server.fullname" . }}
+        {{- with .Values.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/canal-server/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "canal-server.fullname" . }}-test-connection"
+  labels:
+    {{- include "canal-server.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "canal-server.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

charts/canal-server/values.yaml

@@ -0,0 +1,144 @@
+# Default values for canal-server.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: canal/canal-server
+  # This sets the pull policy for images.
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "1.1.8-alpha-3"
+
+# This is for the secretes for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# This is to override the chart name.
+nameOverride: ""
+fullnameOverride: ""
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: false
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: false
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
+service:
+  # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+  type: ClusterIP
+  # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+  # port: 80
+  ports:
+    admin: 11110
+    tcp: 11111
+    metric: 11112
+
+# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+livenessProbe:
+  httpGet:
+    path: /
+    port: metric
+readinessProbe:
+  httpGet:
+    path: /
+    port: metric
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+server:
+  config: |
+    canal.port = 11111
+    canal.metrics.pull.port = 11112
+
+    # register ip
+    canal.register.ip =
+
+    # canal admin config
+    canal.admin.manager = canal-admin.default:8089
+    canal.admin.port = 11110
+    canal.admin.user = admin
+    canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+    # admin auto register
+    canal.admin.register.auto = true
+    canal.admin.register.cluster = default

charts/tests/admin-values.yaml

@@ -0,0 +1,34 @@
+ingress:
+  enabled: true
+  hosts:
+    - host: admin.canal.com
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+
+admin:
+  config: |
+    server:
+      port: 8089
+    spring:
+      jackson:
+        date-format: yyyy-MM-dd HH:mm:ss
+        time-zone: GMT+8
+    spring.datasource:
+      address: you_db_host:3306
+      database: canal_manager
+      username: ****
+      password: ****
+      driver-class-name: com.mysql.jdbc.Driver
+      url: jdbc:mysql://${spring.datasource.address}/${spring.datasource.database}?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true
+      hikari:
+        maximum-pool-size: 30
+        minimum-idle: 1
+    canal:
+      adminUser: admin
+      adminPasswd: admin
+    debug: true
+    
+image:
+  repository: registry.us-east-1.aliyuncs.com/sll0/canal-admin
+  tag: "latest"

charts/tests/server-values.yaml

@@ -0,0 +1,20 @@
+server:
+  config: |
+    canal.port = 11111
+    canal.metrics.pull.port = 11112
+
+    # register ip
+    canal.register.ip =
+
+    # canal admin config
+    canal.admin.manager = canal-admin.default:8089
+    canal.admin.port = 11110
+    canal.admin.user = admin
+    canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
+    # admin auto register
+    canal.admin.register.auto = true
+    canal.admin.register.cluster = default
+    
+image:
+  repository: registry.us-east-1.aliyuncs.com/sll0/canal-server
+  tag: "v1.1.8-alpha-3"

charts/tests/test.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+helm install canal-admin -f ./admin-values.yaml ../canal-admin --dry-run
+
+helm install canal-server -f ./server-values.yaml ../canal-server --dry-run

client-adapter/clickhouse/pom.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.8</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.clickhouse</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter clickhouse module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>ru.yandex.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../launcher/target/classes/clickhouse" overwrite="true">
+                                    <fileset dir="${project.basedir}/target/classes/clickhouse" erroronmissingdir="true">
+                                        <include name="*.yml" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseAdapter.java

@@ -0,0 +1,340 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.filter.stat.StatFilter;
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.util.JdbcUtils;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.ConfigLoader;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MirrorDbConfig;
+import com.alibaba.otter.canal.client.adapter.clickhouse.monitor.ClickHouseConfigMonitor;
+import com.alibaba.otter.canal.client.adapter.clickhouse.service.ClickHouseBatchSyncService;
+import com.alibaba.otter.canal.client.adapter.clickhouse.service.ClickHouseEtlService;
+import com.alibaba.otter.canal.client.adapter.clickhouse.service.ClickHouseMirrorDbBatchSyncService;
+import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.*;
+
+/**
+ * ClickHouse Adapter implementation
+ *
+ * @author: Xander
+ * @date: Created in 2023/11/10 1:13
+ * @email: zhrunxin33@gmail.com
+ */
+@SPI
+public class ClickHouseAdapter implements OuterAdapter {
+
+    private static final Logger                         logger = LoggerFactory.getLogger(ClickHouseAdapter.class);
+
+    // Store the mapping of filename and configuration, load yml files below
+    // resource path
+    private Map<String, MappingConfig>              clickHouseMapping   = new ConcurrentHashMap<>();
+
+    // Schema -> Table -> MappingConfig
+    private Map<String, Map<String, MappingConfig>> mappingConfigCache  = new ConcurrentHashMap<>();
+
+    // Mirror DB Configuration, don't need to load column mapping
+    private Map<String, MirrorDbConfig>             mirrorDbConfigCache = new ConcurrentHashMap<>();
+
+    private DruidDataSource                             dataSource;
+
+    private ClickHouseBatchSyncService clickHouseBatchSyncService;
+
+    private ClickHouseMirrorDbBatchSyncService clickHouseMirrorDbBatchSyncService;
+
+    private Properties                                  envProperties;
+
+    // Launch configuration
+    private OuterAdapterConfig                      configuration;
+
+    private ClickHouseConfigMonitor clickHouseConfigMonitor;
+
+    public Map<String, MappingConfig> getClickHouseMapping() {
+        return clickHouseMapping;
+    }
+
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        this.envProperties = envProperties;
+        this.configuration = configuration;
+        // Load DB type from adapter.launch/bootstrap.yml
+        Map<String, String> properties = configuration.getProperties();
+        String dbType = JdbcUtils.getDbType(properties.get("jdbc.url"), null);
+        // 当.yml文件编码格式存在问题，此处clickhouse yml文件构建 可能会抛出异常
+        Map<String, MappingConfig> clickHouseMappingTmp = ConfigLoader.load(envProperties);
+        // 过滤不匹配的key的配置
+        clickHouseMappingTmp.forEach((key, config) -> {
+            addConfig(key, config);
+        });
+
+        if (clickHouseMapping.isEmpty()) {
+            throw new RuntimeException("No clickhouse adapter found for config key: " + configuration.getKey());
+        }
+
+        // 初始化连接池
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName(properties.get("jdbc.driverClassName"));
+        dataSource.setUrl(properties.get("jdbc.url"));
+        dataSource.setUsername(properties.get("jdbc.username"));
+        dataSource.setPassword(properties.get("jdbc.password"));
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(30);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setUseUnfairLock(true);
+        dataSource.setDefaultAutoCommit(false); // disable auto commit or disable Transactional
+        dataSource.setDbType(dbType);
+
+        if ("true".equals(properties.getOrDefault("druid.stat.enable", "true"))) {
+            StatFilter statFilter = new StatFilter();
+            statFilter.setSlowSqlMillis(Long.parseLong(properties.getOrDefault("druid.stat.slowSqlMillis", "1000")));
+            statFilter.setMergeSql(true);
+            statFilter.setLogSlowSql(true);
+            dataSource.setProxyFilters(Collections.singletonList(statFilter));
+        }
+
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            logger.error("ERROR ## failed to initial datasource: " + properties.get("jdbc.url"), e);
+        }
+
+        String threads = properties.get("threads");
+        String scheduleTime = properties.get("scheduleTime");
+        String batchSize = properties.get("batchSize");
+
+        boolean skipDupException = BooleanUtils.toBoolean(configuration.getProperties()
+                .getOrDefault("skipDupException", "true"));
+        clickHouseBatchSyncService = new ClickHouseBatchSyncService(dataSource,
+                threads != null ? Integer.valueOf(threads) : null,
+                batchSize != null ? Integer.valueOf(batchSize) : null,
+                scheduleTime != null ? Long.valueOf(scheduleTime) : null,
+                skipDupException);
+
+        clickHouseMirrorDbBatchSyncService = new ClickHouseMirrorDbBatchSyncService(mirrorDbConfigCache,
+                dataSource,
+                threads != null ? Integer.valueOf(threads) : null,
+                batchSize != null ? Integer.valueOf(batchSize) : null,
+                scheduleTime != null ? Long.valueOf(scheduleTime) : null,
+                clickHouseBatchSyncService.getColumnsTypeCache(),
+                skipDupException);
+
+        clickHouseConfigMonitor = new ClickHouseConfigMonitor();
+        clickHouseConfigMonitor.init(configuration.getKey(), this, envProperties);
+    }
+
+    /**
+     * Sync main entrance
+     *
+     * @param dmls 数据包
+     */
+    @Override
+    public void sync(List<Dml> dmls) {
+        if (dmls == null || dmls.isEmpty()) {
+            return;
+        }
+        try {
+            // If mappingConfigCache(column mapping) is empty, that must be mirroring synchronize
+            if (!mappingConfigCache.isEmpty()) {
+                clickHouseBatchSyncService.sync(mappingConfigCache, dmls, envProperties);
+            }
+            clickHouseMirrorDbBatchSyncService.sync(dmls);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Destroy
+     */
+    @Override
+    public void destroy() {
+        if (clickHouseConfigMonitor != null) {
+            clickHouseConfigMonitor.destroy();
+        }
+
+        if (clickHouseBatchSyncService != null) {
+            clickHouseBatchSyncService.close();
+        }
+
+        if (dataSource != null) {
+            dataSource.close();
+        }
+    }
+
+
+    /**
+     * ETL方法
+     *
+     * @param task 任务名, 对应配置名
+     * @param params etl筛选条件
+     * @return ETL结果
+     */
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        EtlResult etlResult = new EtlResult();
+        MappingConfig config = clickHouseMapping.get(task);
+        ClickHouseEtlService clickhouseEtlService = new ClickHouseEtlService(dataSource, config);
+        if (config != null) {
+            return clickhouseEtlService.importData(params);
+        } else {
+            StringBuilder resultMsg = new StringBuilder();
+            boolean resSucc = true;
+            for (MappingConfig configTmp : clickHouseMapping.values()) {
+                // 取所有的destination为task的配置
+                if (configTmp.getDestination().equals(task)) {
+                    EtlResult etlRes = clickhouseEtlService.importData(params);
+                    if (!etlRes.getSucceeded()) {
+                        resSucc = false;
+                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
+                    } else {
+                        resultMsg.append(etlRes.getResultMessage()).append("\n");
+                    }
+                }
+            }
+            if (resultMsg.length() > 0) {
+                etlResult.setSucceeded(resSucc);
+                if (resSucc) {
+                    etlResult.setResultMessage(resultMsg.toString());
+                } else {
+                    etlResult.setErrorMessage(resultMsg.toString());
+                }
+                return etlResult;
+            }
+        }
+        etlResult.setSucceeded(false);
+        etlResult.setErrorMessage("Task not found");
+        return etlResult;
+    }
+
+    /**
+     * 获取总数方法
+     *
+     * @param task 任务名, 对应配置名
+     * @return 总数
+     */
+    @Override
+    public Map<String, Object> count(String task) {
+        MappingConfig config = clickHouseMapping.get(task);
+        MappingConfig.DbMapping dbMapping = config.getDbMapping();
+        String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType());
+        Connection conn = null;
+        Map<String, Object> res = new LinkedHashMap<>();
+        try {
+            conn = dataSource.getConnection();
+            Util.sqlRS(conn, sql, rs -> {
+                try {
+                    if (rs.next()) {
+                        Long rowCount = rs.getLong("cnt");
+                        res.put("count", rowCount);
+                    }
+                } catch (SQLException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            });
+        } catch (SQLException e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+        res.put("targetTable", SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()));
+
+        return res;
+    }
+
+    /**
+     * 获取对应canal instance name 或 mq topic
+     *
+     * @param task 任务名, 对应配置名
+     * @return destination
+     */
+    @Override
+    public String getDestination(String task) {
+        MappingConfig config = clickHouseMapping.get(task);
+        if (config != null) {
+            return config.getDestination();
+        }
+        return null;
+    }
+
+    private void addSyncConfigToCache(String configName, MappingConfig mappingConfig) {
+        if (!mappingConfig.getDbMapping().getMirrorDb()) {
+            String key;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
+                        + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
+                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
+            } else {
+                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
+                        + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
+            }
+            Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key,
+                    k1 -> new ConcurrentHashMap<>());
+            configMap.put(configName, mappingConfig);
+        } else {
+            // mirrorDB
+            String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
+                    + mappingConfig.getDbMapping().getDatabase();
+            mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
+        }
+    }
+
+    public boolean addConfig(String fileName, MappingConfig config) {
+        if (match(config)) {
+            clickHouseMapping.put(fileName, config);
+            addSyncConfigToCache(fileName, config);
+            FileName2KeyMapping.register(getClass().getAnnotation(SPI.class).value(), fileName,
+                    configuration.getKey());
+            return true;
+        }
+        return false;
+    }
+
+    public void updateConfig(String fileName, MappingConfig config) {
+        if (config.getOuterAdapterKey() != null && !config.getOuterAdapterKey()
+                .equals(configuration.getKey())) {
+            // 理论上不允许改这个 因为本身就是通过这个关联起Adapter和Config的
+            throw new RuntimeException("not allow to change outAdapterKey");
+        }
+        clickHouseMapping.put(fileName, config);
+        addSyncConfigToCache(fileName, config);
+    }
+
+    public void deleteConfig(String fileName) {
+        clickHouseMapping.remove(fileName);
+        for (Map<String, MappingConfig> configMap : mappingConfigCache.values()) {
+            if (configMap != null) {
+                configMap.remove(fileName);
+            }
+        }
+        FileName2KeyMapping.unregister(getClass().getAnnotation(SPI.class).value(), fileName);
+    }
+
+    private boolean match(MappingConfig config) {
+        boolean sameMatch = config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                .equalsIgnoreCase(configuration.getKey());
+        boolean prefixMatch = config.getOuterAdapterKey() == null && configuration.getKey()
+                .startsWith(StringUtils
+                        .join(new String[]{Util.AUTO_GENERATED_PREFIX, config.getDestination(),
+                                config.getGroupId()}, '-'));
+        return sameMatch || prefixMatch;
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/config/ConfigLoader.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
+
+/**
+ * CLICKHOUSE表映射配置加载器
+ *
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+public class ConfigLoader {
+
+    private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
+
+    /**
+     * 加载CLICKHOUSE表映射配置
+     *
+     * @return 配置名/配置文件名--对象
+     */
+    public static Map<String, MappingConfig> load(Properties envProperties) {
+        logger.info("## Start loading clickhouse mapping config ... ");
+
+        Map<String, MappingConfig> result = new LinkedHashMap<>();
+
+        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("clickhouse");
+        configContentMap.forEach((fileName, content) -> {
+            MappingConfig config = YamlUtils.ymlToObj(null, content, MappingConfig.class, null, envProperties);
+            if (config == null) {
+                return;
+            }
+            try {
+                config.validate();
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
+            }
+            result.put(fileName, config);
+        });
+
+        logger.info("## ClickHouse mapping config loaded");
+        return result;
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/config/MappingConfig.java

@@ -0,0 +1,227 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
+
+/**
+ * CLICKHOUSE表映射配置
+ *
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+public class MappingConfig implements AdapterConfig {
+
+    private String    dataSourceKey;      // 数据源key
+
+    private String    destination;        // canal实例或MQ的topic
+
+    private String    groupId;            // groupId
+
+    private String    outerAdapterKey;    // 对应适配器的key
+
+    private boolean   concurrent = false; // 是否并行同步
+
+    private DbMapping dbMapping;          // db映射配置
+
+    public String getDataSourceKey() {
+        return dataSourceKey;
+    }
+
+    public void setDataSourceKey(String dataSourceKey) {
+        this.dataSourceKey = dataSourceKey;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getOuterAdapterKey() {
+        return outerAdapterKey;
+    }
+
+    public void setOuterAdapterKey(String outerAdapterKey) {
+        this.outerAdapterKey = outerAdapterKey;
+    }
+
+    public boolean getConcurrent() {
+        return concurrent;
+    }
+
+    public void setConcurrent(boolean concurrent) {
+        this.concurrent = concurrent;
+    }
+
+    public DbMapping getDbMapping() {
+        return dbMapping;
+    }
+
+    public void setDbMapping(DbMapping dbMapping) {
+        this.dbMapping = dbMapping;
+    }
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public AdapterMapping getMapping() {
+        return dbMapping;
+    }
+
+    public void validate() {
+        if (dbMapping.database == null || dbMapping.database.isEmpty()) {
+            throw new NullPointerException("dbMapping.database");
+        }
+        if (!dbMapping.getMirrorDb() && (dbMapping.table == null || dbMapping.table.isEmpty())) {
+            throw new NullPointerException("dbMapping.table");
+        }
+        if (!dbMapping.getMirrorDb() && (dbMapping.targetTable == null || dbMapping.targetTable.isEmpty())) {
+            throw new NullPointerException("dbMapping.targetTable");
+        }
+    }
+
+    public static class DbMapping implements AdapterMapping {
+
+        private boolean             mirrorDb        = false;                 // 是否镜像库
+        private String              database;                                // 数据库名或schema名
+        private String              table;                                   // 表名
+        private Map<String, String> targetPk        = new LinkedHashMap<>(); // 目标表主键字段
+        private boolean             mapAll          = false;                 // 映射所有字段
+        private String              targetDb;                                // 目标库名
+        private String              targetTable;                             // 目标表名
+        private Map<String, String> targetColumns;                           // 目标表字段映射
+
+        private boolean             caseInsensitive = false;                 // 目标表不区分大小写，默认是否
+
+        private String              etlCondition;                            // etl条件sql
+
+        private int                 readBatch       = 5000;
+        private int                 commitBatch     = 5000;                  // etl等批量提交大小
+
+        private Map<String, String> allMapColumns;
+
+        public boolean getMirrorDb() {
+            return mirrorDb;
+        }
+
+        public void setMirrorDb(boolean mirrorDb) {
+            this.mirrorDb = mirrorDb;
+        }
+
+        public String getDatabase() {
+            return database;
+        }
+
+        public void setDatabase(String database) {
+            this.database = database;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
+        public void setTable(String table) {
+            this.table = table;
+        }
+
+        public Map<String, String> getTargetPk() {
+            return targetPk;
+        }
+
+        public void setTargetPk(Map<String, String> targetPk) {
+            this.targetPk = targetPk;
+        }
+
+        public Boolean getMapAll() {
+            return mapAll;
+        }
+
+        public void setMapAll(Boolean mapAll) {
+            this.mapAll = mapAll;
+        }
+
+        public String getTargetDb() {
+            return targetDb;
+        }
+
+        public void setTargetDb(String targetDb) {
+            this.targetDb = targetDb;
+        }
+
+        public String getTargetTable() {
+            return targetTable;
+        }
+
+        public void setTargetTable(String targetTable) {
+            this.targetTable = targetTable;
+        }
+
+        public Map<String, String> getTargetColumns() {
+            if (targetColumns != null) {
+                targetColumns.forEach((key, value) -> {
+                    if (StringUtils.isEmpty(value)) {
+                        targetColumns.put(key, key);
+                    }
+                });
+            }
+            return targetColumns;
+        }
+
+        public void setTargetColumns(Map<String, String> targetColumns) {
+            this.targetColumns = targetColumns;
+        }
+
+        public boolean isCaseInsensitive() {
+            return caseInsensitive;
+        }
+
+        public void setCaseInsensitive(boolean caseInsensitive) {
+            this.caseInsensitive = caseInsensitive;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public int getReadBatch() {
+            return readBatch;
+        }
+
+        public void setReadBatch(int readBatch) {
+            this.readBatch = readBatch;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public Map<String, String> getAllMapColumns() {
+            return allMapColumns;
+        }
+
+        public void setAllMapColumns(Map<String, String> allMapColumns) {
+            this.allMapColumns = allMapColumns;
+        }
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/config/MirrorDbConfig.java

@@ -0,0 +1,51 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.config;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+
+public class MirrorDbConfig {
+
+    private String                     fileName;
+    private MappingConfig              mappingConfig;
+    private Map<String, MappingConfig> tableConfig = new ConcurrentHashMap<>();
+
+    public static MirrorDbConfig create(String fileName, MappingConfig mappingConfig) {
+        return new MirrorDbConfig(fileName, mappingConfig);
+    }
+
+    public MirrorDbConfig(String fileName, MappingConfig mappingConfig){
+        this.fileName = fileName;
+        this.mappingConfig = mappingConfig;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+
+    public MappingConfig getMappingConfig() {
+        return mappingConfig;
+    }
+
+    public void setMappingConfig(MappingConfig mappingConfig) {
+        this.mappingConfig = mappingConfig;
+    }
+
+    public Map<String, MappingConfig> getTableConfig() {
+        return tableConfig;
+    }
+
+    public void setTableConfig(Map<String, MappingConfig> tableConfig) {
+        this.tableConfig = tableConfig;
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/monitor/ClickHouseConfigMonitor.java

@@ -0,0 +1,131 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.monitor;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.client.adapter.support.YamlUtils;
+
+/**
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+
+public class ClickHouseConfigMonitor {
+
+    private static final Logger   logger      = LoggerFactory.getLogger(ClickHouseConfigMonitor.class);
+
+    private static final String   adapterName = "clickhouse";
+
+    private String                key;
+
+    private ClickHouseAdapter     clickHouseAdapter;
+
+    private Properties            envProperties;
+
+    private FileAlterationMonitor fileMonitor;
+
+    public void init(String key, ClickHouseAdapter clickHouseAdapter, Properties envProperties) {
+        this.key = key;
+        this.clickHouseAdapter = clickHouseAdapter;
+        this.envProperties = envProperties;
+        File confDir = Util.getConfDirPath(adapterName);
+        try {
+            FileAlterationObserver observer = new FileAlterationObserver(confDir,
+                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
+            FileListener listener = new FileListener();
+            observer.addListener(listener);
+            fileMonitor = new FileAlterationMonitor(3000, observer);
+            fileMonitor.start();
+
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    public void destroy() {
+        try {
+            fileMonitor.stop();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private class FileListener extends FileAlterationListenerAdaptor {
+
+        @Override
+        public void onFileCreate(File file) {
+            super.onFileCreate(file);
+            try {
+                // 加载新增的配置文件
+                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
+                MappingConfig config = YamlUtils
+                    .ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                if (config == null) {
+                    return;
+                }
+                config.validate();
+                boolean result = clickHouseAdapter.addConfig(file.getName(), config);
+                if (result) {
+                    logger.info("Add a new clickhouse mapping config: {} to canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileChange(File file) {
+            super.onFileChange(file);
+
+            try {
+                if (clickHouseAdapter.getClickHouseMapping().containsKey(file.getName())) {
+                    // 加载配置文件
+                    String configContent = MappingConfigsLoader
+                        .loadConfig(adapterName + File.separator + file.getName());
+                    if (configContent == null) {
+                        onFileDelete(file);
+                        return;
+                    }
+                    MappingConfig config = YamlUtils
+                        .ymlToObj(null, configContent, MappingConfig.class, null, envProperties);
+                    if (config == null) {
+                        return;
+                    }
+                    config.validate();
+                    clickHouseAdapter.updateConfig(file.getName(), config);
+                    logger.info("Change a clickhouse mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        @Override
+        public void onFileDelete(File file) {
+            super.onFileDelete(file);
+
+            try {
+                if (clickHouseAdapter.getClickHouseMapping().containsKey(file.getName())) {
+                    clickHouseAdapter.deleteConfig(file.getName());
+
+                    logger.info("Delete a clickhouse mapping config: {} of canal adapter", file.getName());
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseBatchSyncService.java

@@ -0,0 +1,555 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.service;
+
+import java.sql.Connection;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter.Feature;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig.DbMapping;
+import com.alibaba.otter.canal.client.adapter.clickhouse.support.BatchExecutor;
+import com.alibaba.otter.canal.client.adapter.clickhouse.support.SingleDml;
+import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+
+/**
+ * ClickHouse batch synchronize
+ *
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+public class ClickHouseBatchSyncService {
+
+    private static final Logger                         logger  = LoggerFactory.getLogger(ClickHouseBatchSyncService.class);
+
+    private DruidDataSource                             dataSource;
+
+    private Map<String, Map<String, Integer>>           columnsTypeCache;     // Cache of instance.schema.table -> <columnName, jdbcType>
+
+    private Map<MappingConfig, List<SingleDml>>[]       bufferPools;          // Data buffer pool store sync data, List<Dml> dispersed as arrays according to hash value
+
+    private BatchExecutor[]                             batchExecutors;       // Batch Executor
+
+    private BatchExecutor                               alterExecutors;       // Alter Single Executor(update/delete/truncate)
+
+    private ExecutorService[]                           executorThreads;      // Be initialized once
+
+    private ScheduledExecutorService[]                  scheduledExecutors;
+
+    private int                                         threads = 3;          // Default parallel thread count
+    private int                                         batchSize = 1000;
+    private long                                        scheduleTime = 10;
+    private boolean                                     skipDupException;
+
+    public Map<String, Map<String, Integer>> getColumnsTypeCache() {
+        return columnsTypeCache;
+    }
+
+    public ClickHouseBatchSyncService(DruidDataSource dataSource, Integer threads, Integer batchSize, Long scheduleTime, boolean skipDupException){
+        this(dataSource, threads, batchSize, scheduleTime, new ConcurrentHashMap<>(), skipDupException);
+    }
+
+    @SuppressWarnings("unchecked")
+    public ClickHouseBatchSyncService(DruidDataSource dataSource, Integer threads, Integer batchSize, Long scheduleTime, Map<String, Map<String, Integer>> columnsTypeCache,
+                                      boolean skipDupException){
+        this.dataSource = dataSource;
+        this.columnsTypeCache = columnsTypeCache;
+        this.skipDupException = skipDupException;
+        try {
+            if (threads != null) {
+                this.threads = threads;
+            }
+            if (batchSize != null) {
+                this.batchSize = batchSize;
+            }
+            if (scheduleTime != null) {
+                this.scheduleTime = scheduleTime;
+            }
+            this.alterExecutors = new BatchExecutor(dataSource);
+            this.bufferPools = new ConcurrentHashMap[this.threads];
+            this.batchExecutors = new BatchExecutor[this.threads];
+            this.executorThreads = new ExecutorService[this.threads];
+            this.scheduledExecutors = new ScheduledExecutorService[this.threads];
+            for (int i = 0; i < this.threads; i++) {
+                bufferPools[i] = new ConcurrentHashMap<>();
+                batchExecutors[i] = new BatchExecutor(dataSource);
+                executorThreads[i] = Executors.newSingleThreadExecutor();
+                scheduledExecutors[i] = Executors.newSingleThreadScheduledExecutor();
+            }
+            scheduleBatchSync();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Timing-driven event
+     * start schedule batch sync threadPool
+     */
+    private void scheduleBatchSync() {
+        for (int i = 0; i < scheduledExecutors.length; i++) {
+            int index = i;
+            scheduledExecutors[i].scheduleAtFixedRate(()->{
+                List<Future<Boolean>> futures = new ArrayList<>();
+                for (MappingConfig mappingConfig : bufferPools[index].keySet()) {
+                    List<SingleDml> dmls = bufferPools[index].get(mappingConfig);
+                    if (dmls == null || dmls.isEmpty()) {
+                        return;
+                    }
+                    List<SingleDml> tempDmls;
+                    synchronized (dmls) {
+                        tempDmls = new ArrayList<>(dmls);
+                        dmls.clear();
+                    }
+                    futures.add(executorThreads[index].submit(()->{
+                        try {
+                            insert(batchExecutors[index], mappingConfig, tempDmls);
+                            batchExecutors[index].commit();
+                            return true;
+                        } catch (Exception e){
+                            batchExecutors[index].rollback();
+                            throw new RuntimeException(e);
+                        }
+                    }));
+                }
+            }, 0, scheduleTime, TimeUnit.SECONDS);
+        }
+        logger.info("Schedule batch executors has started successfully!");
+    }
+
+    /**
+     * 批量同步回调
+     *
+     * @param dmls 批量 DML
+     * @param function 回调方法
+     */
+    public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
+        boolean toExecute = false;
+        for (Dml dml : dmls) {
+            if (!toExecute) {
+                toExecute = function.apply(dml);
+            } else {
+                function.apply(dml);
+            }
+        }
+    }
+
+    /**
+     * Distribute dmls into different partition
+     *
+     * @param mappingConfig {@link com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter#mappingConfigCache }
+     * @param dmls received DML
+     */
+    public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls, Properties envProperties) {
+        sync(dmls, dml -> {
+            if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
+                // DDL(Cache need to update when DDL was executed)
+            columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
+            return false;
+        } else {
+            // DML
+            String destination = StringUtils.trimToEmpty(dml.getDestination());
+            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
+            String database = dml.getDatabase();
+            String table = dml.getTable();
+            Map<String, MappingConfig> configMap;
+            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
+            } else {
+                configMap = mappingConfig.get(destination + "_" + database + "-" + table);
+            }
+
+            if (configMap == null) {
+                return false;
+            }
+
+            if (configMap.values().isEmpty()) {
+                return false;
+            }
+
+            for (MappingConfig config : configMap.values()) {
+                distributeDml(config, dml);
+            }
+            return true;
+        }
+    }   );
+    }
+
+    /**
+     * Dml distributor
+     */
+    private void distributeDml(MappingConfig config, Dml dml) {
+        if (config != null) {
+            try {
+                String type = dml.getType();
+                if (type == null) return;
+
+                if (type.equalsIgnoreCase("INSERT")) {
+                    appendDmlBufferPartition(config, dml);
+                } else {
+                    boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
+                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
+
+                    if (type.equalsIgnoreCase("UPDATE")) {
+                        for (SingleDml singleDml : singleDmls) {
+                            update(alterExecutors, config, singleDml);
+                        }
+                    } else if (type.equalsIgnoreCase("DELETE")) {
+                        for (SingleDml singleDml : singleDmls) {
+                            delete(alterExecutors, config, singleDml);
+                        }
+                    } else if (type.equalsIgnoreCase("TRUNCATE")) {
+                        truncate(alterExecutors, config);
+                    }
+                }
+                if (logger.isDebugEnabled()) {
+                    logger.debug("DML: {}", JSON.toJSONString(dml, Feature.WriteNulls));
+                }
+            } catch (SQLException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    public void appendDmlBufferPartition(MappingConfig config, Dml dml) {
+        boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
+        List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
+
+        singleDmls.forEach(singleDml -> {
+            int hash = mappingHash(config.getDbMapping());
+            if (!config.getConcurrent()) {
+                hash = 0;
+            }
+            List<SingleDml> dmls = bufferPools[hash].computeIfAbsent(config, k -> new ArrayList<>());
+            synchronized (dmls) {
+                dmls.add(singleDml);
+                logger.info("Append one data into pool, id {}", singleDml.getData().get("id"));
+            }
+            // Check the size of the List, achieve when it reaches the maximum value
+            if (dmls.size() >= batchSize) syncToClickHouse(config, hash);
+        });
+    }
+
+    /**
+     * sync when size of list{@link #bufferPools } reaches the maximum value
+     *
+     * @param config key
+     * @param index parallel thread index
+     */
+    private void syncToClickHouse(MappingConfig config, int index) {
+        List<SingleDml> dmls = bufferPools[index].get(config);
+        logger.info("schema:{} table:{} reaches the maximum value, ready to synchronize, size {}", config.getDbMapping().getDatabase(), config.getDbMapping().getTable(), dmls.size());
+        if (dmls ==null || dmls.isEmpty()) {
+            return;
+        }
+        List<SingleDml> tempDmls;
+        synchronized (dmls) {
+            tempDmls = new ArrayList<>(dmls);
+            dmls.clear();
+        }
+        executorThreads[index].submit(() -> {
+            try {
+                insert(batchExecutors[index], config, tempDmls);
+                batchExecutors[index].commit();
+                return true;
+            } catch (Exception e) {
+                batchExecutors[index].rollback();
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    /**
+     * Insert
+     *
+     * @param batchExecutor batch translational executor
+     * @param config corresponding configuration object
+     * @param dmls DMLs
+     */
+    private void insert(BatchExecutor batchExecutor, MappingConfig config, List<SingleDml> dmls) throws SQLException {
+        if (dmls == null || dmls.isEmpty()) {
+            return;
+        }
+        List<SingleDml> clearDmls = dmls.stream().filter(e -> e.getData() != null && !e.getData().isEmpty()).collect(Collectors.toList());
+        if (clearDmls == null || clearDmls.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+        String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
+        Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, clearDmls.get(0).getData());
+
+        StringBuilder insertSql = new StringBuilder();
+        insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())).append(" (");
+
+        columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick)
+            .append(targetColumnName)
+            .append(backtick)
+            .append(","));
+        int len = insertSql.length();
+        insertSql.delete(len - 1, len).append(") VALUES (");
+        int mapLen = columnsMap.size();
+        for (int i = 0; i < mapLen; i++) {
+            insertSql.append("?,");
+        }
+        len = insertSql.length();
+        insertSql.delete(len - 1, len).append(")");
+
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        List<List<Map<String, ?>>> values = new ArrayList<>();
+        boolean flag = false;
+        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+
+            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+            }
+            for (int i = 0; i < clearDmls.size(); i++) {
+                Map<String, Object> dmlData = clearDmls.get(i).getData();
+                List<Map<String, ?>> item;
+                if (flag == false) {
+                    item = new ArrayList<>();
+                    values.add(item);
+                } else {
+                    item = values.get(i);
+                }
+                Object value = dmlData.get(srcColumnName);
+                BatchExecutor.setValue(item, type, value);
+            }
+            flag = true;
+        }
+
+        try {
+            batchExecutor.batchExecute(insertSql.toString(), values);
+        } catch (SQLException e) {
+            if (skipDupException
+                && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001:"))) {
+                // ignore
+                // TODO 增加更多关系数据库的主键冲突的错误码
+            } else {
+                throw e;
+            }
+        }
+        if (logger.isTraceEnabled()) {
+            logger.trace("Insert into target table, sql: {}", insertSql);
+        }
+
+    }
+
+    /**
+     * 更新操作
+     *
+     * @param config 配置项
+     * @param dml DML数据
+     */
+    private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
+        Map<String, Object> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        Map<String, Object> old = dml.getOld();
+        if (old == null || old.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+        String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
+        Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
+
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        StringBuilder updateSql = new StringBuilder();
+        updateSql.append("ALTER TABLE ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())).append(" UPDATE ");
+        List<Map<String, ?>> values = new ArrayList<>();
+        boolean hasMatched = false;
+        for (String srcColumnName : old.keySet()) {
+            List<String> targetColumnNames = new ArrayList<>();
+            columnsMap.forEach((targetColumn, srcColumn) -> {
+                if (srcColumnName.equalsIgnoreCase(srcColumn)) {
+                    targetColumnNames.add(targetColumn);
+                }
+            });
+            if (!targetColumnNames.isEmpty()) {
+                hasMatched = true;
+                for (String targetColumnName : targetColumnNames) {
+                    updateSql.append(backtick).append(targetColumnName).append(backtick).append("=?, ");
+                    Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+                    if (type == null) {
+                        throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+                    }
+                    BatchExecutor.setValue(values, type, data.get(srcColumnName));
+                }
+            }
+        }
+        if (!hasMatched) {
+            logger.warn("Did not matched any columns to update ");
+            return;
+        }
+        int len = updateSql.length();
+        updateSql.delete(len - 2, len).append(" WHERE ");
+
+        // 拼接主键
+        appendCondition(dbMapping, updateSql, ctype, values, data, old);
+        batchExecutor.execute(updateSql.toString(), values);
+        if (logger.isTraceEnabled()) {
+            logger.trace("Alter update target table, sql: {}", updateSql);
+        }
+    }
+
+    /**
+     * 删除操作
+     *
+     * @param config
+     * @param dml
+     */
+    private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
+        Map<String, Object> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        DbMapping dbMapping = config.getDbMapping();
+        Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
+
+        StringBuilder sql = new StringBuilder();
+        sql.append("ALTER TABLE ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())).append(" DELETE WHERE ");
+
+        List<Map<String, ?>> values = new ArrayList<>();
+        // 拼接主键
+        appendCondition(dbMapping, sql, ctype, values, data);
+        batchExecutor.execute(sql.toString(), values);
+        if (logger.isTraceEnabled()) {
+            logger.trace("Alter delete from target table, sql: {}", sql);
+        }
+    }
+
+    /**
+     * truncate操作
+     *
+     * @param config
+     */
+    private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException {
+        DbMapping dbMapping = config.getDbMapping();
+        StringBuilder sql = new StringBuilder();
+        sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()));
+        batchExecutor.execute(sql.toString(), new ArrayList<>());
+        if (logger.isTraceEnabled()) {
+            logger.trace("Truncate target table, sql: {}", sql);
+        }
+    }
+
+    /**
+     * 获取目标字段类型
+     *
+     * @param conn sql connection
+     * @param config 映射配置
+     * @return 字段sqlType
+     */
+    private Map<String, Integer> getTargetColumnType(Connection conn, MappingConfig config) {
+        DbMapping dbMapping = config.getDbMapping();
+        String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable();
+        Map<String, Integer> columnType = columnsTypeCache.get(cacheKey);
+        if (columnType == null) {
+            synchronized (ClickHouseBatchSyncService.class) {
+                columnType = columnsTypeCache.get(cacheKey);
+                if (columnType == null) {
+                    columnType = new LinkedHashMap<>();
+                    final Map<String, Integer> columnTypeTmp = columnType;
+                    String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " WHERE 1=2";
+                    Util.sqlRS(conn, sql, rs -> {
+                        try {
+                            ResultSetMetaData rsd = rs.getMetaData();
+                            int columnCount = rsd.getColumnCount();
+                            for (int i = 1; i <= columnCount; i++) {
+                                int colType = rsd.getColumnType(i);
+                                // 修复year类型作为date处理时的data truncated问题
+                                if ("YEAR".equals(rsd.getColumnTypeName(i))) {
+                                    colType = Types.VARCHAR;
+                                }
+                                columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), colType);
+                            }
+                            columnsTypeCache.put(cacheKey, columnTypeTmp);
+                        } catch (SQLException e) {
+                            logger.error(e.getMessage(), e);
+                        }
+                    });
+                }
+            }
+        }
+        return columnType;
+    }
+
+    /**
+     * 拼接主键 where条件
+     */
+    private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map<String, Integer> ctype,
+                                 List<Map<String, ?>> values, Map<String, Object> d) {
+        appendCondition(dbMapping, sql, ctype, values, d, null);
+    }
+
+    private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map<String, Integer> ctype,
+                                 List<Map<String, ?>> values, Map<String, Object> d, Map<String, Object> o) {
+        String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
+
+        // 拼接主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = Util.cleanColumn(targetColumnName);
+            }
+            sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND ");
+            Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
+            if (type == null) {
+                throw new RuntimeException("Target column: " + targetColumnName + " not matched");
+            }
+            // 如果有修改主键的情况
+            if (o != null && o.containsKey(srcColumnName)) {
+                BatchExecutor.setValue(values, type, o.get(srcColumnName));
+            } else {
+                BatchExecutor.setValue(values, type, d.get(srcColumnName));
+            }
+        }
+        int len = sql.length();
+        sql.delete(len - 4, len);
+    }
+
+    /**
+     * make sure the same table in one index
+     *
+     * @param dbMapping
+     * @return
+     */
+    private int mappingHash(MappingConfig.DbMapping dbMapping) {
+        int hash = dbMapping.getDatabase().toLowerCase().hashCode() + dbMapping.getTable().toLowerCase().hashCode();
+        hash = Math.abs(hash) % threads;
+        return Math.abs(hash);
+    }
+
+    public void close() {
+        for (int i = 0; i < threads; i++) {
+            executorThreads[i].shutdown();
+        }
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseEtlService.java

@@ -0,0 +1,197 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.service;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.sql.DataSource;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig.DbMapping;
+import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.*;
+
+/**
+ * ClickHouse ETL 操作业务类
+ *
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+public class ClickHouseEtlService extends AbstractEtlService {
+
+    private DataSource    targetDS;
+    private MappingConfig config;
+
+    public ClickHouseEtlService(DataSource targetDS, MappingConfig config){
+        super("CLICKHOUSE", config);
+        this.targetDS = targetDS;
+        this.config = config;
+    }
+
+    /**
+     * 导入数据
+     */
+    public EtlResult importData(List<String> params) {
+        DbMapping dbMapping = config.getDbMapping();
+        DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
+        String sql = "SELECT * FROM " + SyncUtil.getSourceDbTableName(dbMapping, dataSource.getDbType());
+        return importData(sql, params);
+    }
+
+    /**
+     * 执行导入
+     */
+    protected boolean executeSqlImport(DataSource srcDS, String sql, List<Object> values,
+                                       AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
+        try {
+            DbMapping dbMapping = (DbMapping) mapping;
+            Map<String, String> columnsMap = new LinkedHashMap<>();
+            Map<String, Integer> columnType = new LinkedHashMap<>();
+            DruidDataSource dataSource = (DruidDataSource) srcDS;
+            String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
+
+            Util.sqlRS(targetDS,
+                "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()) + " LIMIT 1 ",
+                rs -> {
+                    try {
+
+                        ResultSetMetaData rsd = rs.getMetaData();
+                        int columnCount = rsd.getColumnCount();
+                        List<String> columns = new ArrayList<>();
+                        for (int i = 1; i <= columnCount; i++) {
+                            columnType.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
+                            columns.add(rsd.getColumnName(i));
+                        }
+
+                        columnsMap.putAll(SyncUtil.getColumnsMap(dbMapping, columns));
+                        return true;
+                    } catch (Exception e) {
+                        logger.error(e.getMessage(), e);
+                        return false;
+                    }
+                });
+
+            Util.sqlRS(srcDS, sql, values, rs -> {
+                int idx = 1;
+
+                try {
+                    boolean completed = false;
+
+                    StringBuilder insertSql = new StringBuilder();
+                    insertSql.append("INSERT INTO ")
+                        .append(SyncUtil.getDbTableName(dbMapping, dataSource.getDbType()))
+                        .append(" (");
+                    columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(backtick)
+                        .append(targetColumnName)
+                        .append(backtick)
+                        .append(","));
+
+                    int len = insertSql.length();
+                    insertSql.delete(len - 1, len).append(") VALUES (");
+                    int mapLen = columnsMap.size();
+                    for (int i = 0; i < mapLen; i++) {
+                        insertSql.append("?,");
+                    }
+                    len = insertSql.length();
+                    insertSql.delete(len - 1, len).append(")");
+                    logger.info("executeSqlImport sql:{}", insertSql.toString());
+                    try (Connection connTarget = targetDS.getConnection();
+                            PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) {
+                        connTarget.setAutoCommit(false);
+
+                        while (rs.next()) {
+                            completed = false;
+
+                            pstmt.clearParameters();
+
+                            // 删除数据
+                            Map<String, Object> pkVal = new LinkedHashMap<>();
+                            StringBuilder deleteSql = new StringBuilder(
+                                "ALTER TABLE " + SyncUtil.getDbTableName(dbMapping, dataSource.getDbType())
+                                                                        + " DELETE WHERE ");
+                            appendCondition(dbMapping, deleteSql, pkVal, rs, backtick);
+                            try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) {
+                                int k = 1;
+                                for (Object val : pkVal.values()) {
+                                    pstmt2.setObject(k++, val);
+                                }
+                                pstmt2.execute();
+                            }
+
+                            int i = 1;
+                            for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
+                                String targetColumnName = entry.getKey();
+                                String srcColumnName = entry.getValue();
+                                if (srcColumnName == null) {
+                                    srcColumnName = targetColumnName;
+                                }
+
+                                Integer type = columnType.get(targetColumnName.toLowerCase());
+                                Object value = rs.getObject(srcColumnName);
+                                if (value != null) {
+                                    SyncUtil.setPStmt(type, pstmt, value, i);
+                                } else {
+                                    pstmt.setNull(i, type);
+                                }
+
+                                i++;
+                            }
+
+                            pstmt.execute();
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("Insert into target table, sql: {}", insertSql);
+                            }
+
+                            if (idx % dbMapping.getCommitBatch() == 0) {
+                                connTarget.commit();
+                                completed = true;
+                            }
+                            idx++;
+                            impCount.incrementAndGet();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("successful import count:" + impCount.get());
+                            }
+                        }
+                        if (!completed) {
+                            connTarget.commit();
+                        }
+                    }
+
+                } catch (Exception e) {
+                    logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e);
+                    errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage());
+                }
+                return idx;
+            });
+            return true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            return false;
+        }
+    }
+
+    /**
+     * 拼接目标表主键where条件
+     */
+    private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map<String, Object> values,
+                                        ResultSet rs, String backtick) throws SQLException {
+        // 拼接主键
+        for (Map.Entry<String, String> entry : dbMapping.getTargetPk().entrySet()) {
+            String targetColumnName = entry.getKey();
+            String srcColumnName = entry.getValue();
+            if (srcColumnName == null) {
+                srcColumnName = targetColumnName;
+            }
+            sql.append(backtick).append(targetColumnName).append(backtick).append("=? AND ");
+            values.put(targetColumnName, rs.getObject(srcColumnName));
+        }
+        int len = sql.length();
+        sql.delete(len - 4, len);
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/service/ClickHouseMirrorDbBatchSyncService.java

@@ -0,0 +1,180 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.service;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter.Feature;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MirrorDbConfig;
+import com.alibaba.otter.canal.client.adapter.clickhouse.support.SyncUtil;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+/**
+ * CLICKHOUSE镜像库同步操作业务
+ *
+ * @author: Xander
+ * @date: Created in 2023/11/10 22:23
+ * @email: zhrunxin33@gmail.com
+ * @version 1.1.8
+ */
+public class ClickHouseMirrorDbBatchSyncService {
+
+    private static final Logger         logger = LoggerFactory.getLogger(ClickHouseMirrorDbBatchSyncService.class);
+
+    // 镜像库配置
+    private Map<String, MirrorDbConfig> mirrorDbConfigCache;
+    private DruidDataSource             dataSource;
+    // clickhouseSyncService代理
+    private ClickHouseBatchSyncService  clickHouseBatchSyncService;
+
+    public ClickHouseMirrorDbBatchSyncService(Map<String, MirrorDbConfig> mirrorDbConfigCache,
+                                              DruidDataSource dataSource, Integer threads, Integer batchSize,
+                                              Long scheduleTime, Map<String, Map<String, Integer>> columnsTypeCache,
+                                              boolean skipDupException){
+        this.mirrorDbConfigCache = mirrorDbConfigCache;
+        this.dataSource = dataSource;
+        this.clickHouseBatchSyncService = new ClickHouseBatchSyncService(dataSource,
+            threads,
+            batchSize,
+            scheduleTime,
+            columnsTypeCache,
+            skipDupException);
+    }
+
+    /**
+     * 批量同步方法
+     *
+     * @param dmls 批量 DML，包含DDL
+     */
+    public void sync(List<Dml> dmls) {
+        List<Dml> dmlList = new ArrayList<>();
+        for (Dml dml : dmls) {
+            String destination = StringUtils.trimToEmpty(dml.getDestination());
+            String database = dml.getDatabase();
+            MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(destination + "." + database);
+            if (mirrorDbConfig == null) {
+                continue;
+            }
+            if (mirrorDbConfig.getMappingConfig() == null) {
+                continue;
+            }
+            if (dml.getGroupId() != null && StringUtils.isNotEmpty(mirrorDbConfig.getMappingConfig().getGroupId())) {
+                if (!mirrorDbConfig.getMappingConfig().getGroupId().equals(dml.getGroupId())) {
+                    continue; // 如果groupId不匹配则过滤
+                }
+            }
+
+            if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
+                // 确保执行DDL前DML已执行完
+                syncDml(dmlList);
+                dmlList.clear();
+
+                // DDL
+                if (logger.isDebugEnabled()) {
+                    logger.debug("DDL: {}", JSON.toJSONString(dml, Feature.WriteNulls));
+                }
+                executeDdl(mirrorDbConfig, dml);
+                clickHouseBatchSyncService.getColumnsTypeCache()
+                    .remove(destination + "." + database + "." + dml.getTable());
+                mirrorDbConfig.getTableConfig().remove(dml.getTable()); // 删除对应库表配置
+            } else {
+                // DML
+                initMappingConfig(dml.getTable(), mirrorDbConfig.getMappingConfig(), mirrorDbConfig, dml);
+                dmlList.add(dml);
+            }
+        }
+        syncDml(dmlList);
+    }
+
+    /**
+     * 批量同步Dml
+     *
+     * @param dmlList Dml列表，不包含DDL
+     */
+    private void syncDml(List<Dml> dmlList) {
+        if (dmlList == null || dmlList.isEmpty()) {
+            return;
+        }
+        clickHouseBatchSyncService.sync(dmlList, dml -> {
+            MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase());
+            if (mirrorDbConfig == null) {
+                return false;
+            }
+            String table = dml.getTable();
+            MappingConfig config = mirrorDbConfig.getTableConfig().get(table);
+
+            if (config == null) {
+                return false;
+            }
+            clickHouseBatchSyncService.appendDmlBufferPartition(config, dml);
+            return true;
+        });
+    }
+
+    /**
+     * 初始化表配置
+     *
+     * @param key 配置key: destination.database.table
+     * @param baseConfigMap db sync config
+     * @param dml DML
+     */
+    private void initMappingConfig(String key, MappingConfig baseConfigMap, MirrorDbConfig mirrorDbConfig, Dml dml) {
+        MappingConfig mappingConfig = mirrorDbConfig.getTableConfig().get(key);
+        if (mappingConfig == null) {
+            // 构造表配置
+            mappingConfig = new MappingConfig();
+            mappingConfig.setDataSourceKey(baseConfigMap.getDataSourceKey());
+            mappingConfig.setDestination(baseConfigMap.getDestination());
+            mappingConfig.setGroupId(baseConfigMap.getGroupId());
+            mappingConfig.setOuterAdapterKey(baseConfigMap.getOuterAdapterKey());
+            mappingConfig.setConcurrent(baseConfigMap.getConcurrent());
+            MappingConfig.DbMapping dbMapping = new MappingConfig.DbMapping();
+            mappingConfig.setDbMapping(dbMapping);
+            dbMapping.setDatabase(dml.getDatabase());
+            dbMapping.setTable(dml.getTable());
+            dbMapping.setTargetDb(dml.getDatabase());
+            dbMapping.setTargetTable(dml.getTable());
+            dbMapping.setMapAll(true);
+            List<String> pkNames = dml.getPkNames();
+            Map<String, String> pkMapping = new LinkedHashMap<>();
+            pkNames.forEach(pkName -> pkMapping.put(pkName, pkName));
+            dbMapping.setTargetPk(pkMapping);
+
+            mirrorDbConfig.getTableConfig().put(key, mappingConfig);
+        }
+    }
+
+    /**
+     * DDL 操作
+     *
+     * @param ddl DDL
+     */
+    private void executeDdl(MirrorDbConfig mirrorDbConfig, Dml ddl) {
+        try (Connection conn = dataSource.getConnection(); Statement statement = conn.createStatement()) {
+            // 替换反引号
+            String sql = ddl.getSql();
+            String backtick = SyncUtil.getBacktickByDbType(dataSource.getDbType());
+            if (!"`".equals(backtick)) {
+                sql = sql.replaceAll("`", backtick);
+            }
+            statement.execute(sql);
+            // 移除对应配置
+            mirrorDbConfig.getTableConfig().remove(ddl.getTable());
+            if (logger.isTraceEnabled()) {
+                logger.trace("Execute DDL sql: {} for database: {}", ddl.getSql(), ddl.getDatabase());
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/support/BatchExecutor.java

@@ -0,0 +1,113 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.support;
+
+import java.io.Closeable;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.sql.DataSource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * sql批量执行器
+ *
+ * @author rewerma 2018-11-7 下午06:45:49
+ * @version 1.0.0
+ */
+public class BatchExecutor implements Closeable {
+
+    private static final Logger logger = LoggerFactory.getLogger(BatchExecutor.class);
+
+    private DataSource          dataSource;
+    private Connection          conn;
+    private AtomicInteger       idx    = new AtomicInteger(0);
+
+    public BatchExecutor(DataSource dataSource){
+        this.dataSource = dataSource;
+    }
+
+    public Connection getConn() {
+        if (conn == null) {
+            try {
+                conn = dataSource.getConnection();
+                this.conn.setAutoCommit(false);
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+        return conn;
+    }
+
+    public static void setValue(List<Map<String, ?>> values, int type, Object value) {
+        Map<String, Object> valueItem = new HashMap<>();
+        valueItem.put("type", type);
+        valueItem.put("value", value);
+        values.add(valueItem);
+    }
+
+    public void execute(String sql, List<Map<String, ?>> values) throws SQLException {
+        PreparedStatement pstmt = getConn().prepareStatement(sql);
+        int len = values.size();
+        for (int i = 0; i < len; i++) {
+            int type = (Integer) values.get(i).get("type");
+            Object value = values.get(i).get("value");
+            SyncUtil.setPStmt(type, pstmt, value, i + 1);
+        }
+
+        pstmt.execute();
+        idx.incrementAndGet();
+        pstmt.close();
+    }
+
+    public void batchExecute(String sql, List<List<Map<String, ?>>> batchValues) throws SQLException {
+        PreparedStatement pstmt = getConn().prepareStatement(sql);
+        for (int i = 0; i < batchValues.size(); i++) {
+            List<Map<String, ?>> values = batchValues.get(i);
+            for (int j = 0; j < values.size(); j++) {
+                int type = (Integer) values.get(j).get("type");
+                Object value = values.get(j).get("value");
+                SyncUtil.setPStmt(type, pstmt, value, j + 1);
+            }
+            pstmt.addBatch();
+        }
+
+        pstmt.executeBatch();
+        idx.addAndGet(batchValues.size());
+        pstmt.close();
+    }
+
+    public void commit() throws SQLException {
+        getConn().commit();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor commit " + idx.get() + " rows");
+        }
+        idx.set(0);
+    }
+
+    public void rollback() throws SQLException {
+        getConn().rollback();
+        if (logger.isTraceEnabled()) {
+            logger.trace("Batch executor rollback " + idx.get() + " rows");
+        }
+        idx.set(0);
+    }
+
+    @Override
+    public void close() {
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException e) {
+                logger.error(e.getMessage(), e);
+            } finally {
+                conn = null;
+            }
+        }
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/support/SingleDml.java

@@ -0,0 +1,112 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.support;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.util.LinkedCaseInsensitiveMap;
+
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+public class SingleDml {
+
+    private String              destination;
+    private String              database;
+    private String              table;
+    private String              type;
+    private Map<String, Object> data;
+    private Map<String, Object> old;
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Map<String, Object> getData() {
+        return data;
+    }
+
+    public void setData(Map<String, Object> data) {
+        this.data = data;
+    }
+
+    public Map<String, Object> getOld() {
+        return old;
+    }
+
+    public void setOld(Map<String, Object> old) {
+        this.old = old;
+    }
+
+    public static List<SingleDml> dml2SingleDmls(Dml dml, boolean caseInsensitive) {
+        List<SingleDml> singleDmls = new ArrayList<>();
+        if (dml.getData() != null) {
+            int size = dml.getData().size();
+            for (int i = 0; i < size; i++) {
+                SingleDml singleDml = new SingleDml();
+                singleDml.setDestination(dml.getDestination());
+                singleDml.setDatabase(dml.getDatabase());
+                singleDml.setTable(dml.getTable());
+                singleDml.setType(dml.getType());
+                Map<String, Object> data = dml.getData().get(i);
+                if (caseInsensitive) {
+                    data = toCaseInsensitiveMap(data);
+                }
+                singleDml.setData(data);
+                if (dml.getOld() != null) {
+                    Map<String, Object> oldData = dml.getOld().get(i);
+                    if (caseInsensitive) {
+                        oldData = toCaseInsensitiveMap(oldData);
+                    }
+                    singleDml.setOld(oldData);
+                }
+                singleDmls.add(singleDml);
+            }
+        } else if ("TRUNCATE".equalsIgnoreCase(dml.getType())) {
+            SingleDml singleDml = new SingleDml();
+            singleDml.setDestination(dml.getDestination());
+            singleDml.setDatabase(dml.getDatabase());
+            singleDml.setTable(dml.getTable());
+            singleDml.setType(dml.getType());
+            singleDmls.add(singleDml);
+        }
+        return singleDmls;
+    }
+
+    public static List<SingleDml> dml2SingleDmls(Dml dml) {
+        return dml2SingleDmls(dml, false);
+    }
+
+    private static <V> LinkedCaseInsensitiveMap<V> toCaseInsensitiveMap(Map<String, V> data) {
+        LinkedCaseInsensitiveMap map = new LinkedCaseInsensitiveMap();
+        map.putAll(data);
+        return map;
+    }
+}

client-adapter/clickhouse/src/main/java/com/alibaba/otter/canal/client/adapter/clickhouse/support/SyncUtil.java

@@ -0,0 +1,304 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.support;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.sql.*;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.DbType;
+import com.alibaba.otter.canal.client.adapter.clickhouse.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+
+public class SyncUtil {
+
+    private static final Logger logger = LoggerFactory.getLogger(SyncUtil.class);
+
+    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Map<String, Object> data) {
+        return getColumnsMap(dbMapping, data.keySet());
+    }
+
+    public static Map<String, String> getColumnsMap(MappingConfig.DbMapping dbMapping, Collection<String> columns) {
+        Map<String, String> columnsMap;
+        if (dbMapping.getMapAll()) {
+            if (dbMapping.getAllMapColumns() != null) {
+                return dbMapping.getAllMapColumns();
+            }
+            columnsMap = new LinkedHashMap<>();
+            for (String srcColumn : columns) {
+                boolean flag = true;
+                if (dbMapping.getTargetColumns() != null) {
+                    for (Map.Entry<String, String> entry : dbMapping.getTargetColumns().entrySet()) {
+                        if (srcColumn.equals(entry.getValue())) {
+                            columnsMap.put(entry.getKey(), srcColumn);
+                            flag = false;
+                            break;
+                        }
+                    }
+                }
+                if (flag) {
+                    columnsMap.put(srcColumn, srcColumn);
+                }
+            }
+            dbMapping.setAllMapColumns(columnsMap);
+        } else {
+            columnsMap = dbMapping.getTargetColumns();
+        }
+        return columnsMap;
+    }
+
+    /**
+     * 设置 preparedStatement
+     *
+     * @param type sqlType
+     * @param pstmt 需要设置的preparedStatement
+     * @param value 值
+     * @param i 索引号
+     */
+    public static void setPStmt(int type, PreparedStatement pstmt, Object value, int i) throws SQLException {
+        switch (type) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                if (value instanceof Boolean) {
+                    pstmt.setBoolean(i, (Boolean) value);
+                } else if (value instanceof String) {
+                    boolean v = !value.equals("0");
+                    pstmt.setBoolean(i, v);
+                } else if (value instanceof Number) {
+                    boolean v = ((Number) value).intValue() != 0;
+                    pstmt.setBoolean(i, v);
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.CHAR:
+            case Types.NCHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+                if (value instanceof String) {
+                    pstmt.setString(i, (String) value);
+                } else if (value == null) {
+                    pstmt.setNull(i, type);
+                } else {
+                    pstmt.setString(i, value.toString());
+                }
+                break;
+            case Types.TINYINT:
+                // 向上提升一级，处理unsigned情况
+                if (value instanceof Number) {
+                    pstmt.setShort(i, ((Number) value).shortValue());
+                } else if (value instanceof String) {
+                    pstmt.setShort(i, Short.parseShort((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.SMALLINT:
+                if (value instanceof Number) {
+                    pstmt.setInt(i, ((Number) value).intValue());
+                } else if (value instanceof String) {
+                    pstmt.setInt(i, Integer.parseInt((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.INTEGER:
+                if (value instanceof Number) {
+                    pstmt.setLong(i, ((Number) value).longValue());
+                } else if (value instanceof String) {
+                    pstmt.setLong(i, Long.parseLong((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.BIGINT:
+                if (value instanceof Number) {
+                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
+                } else if (value instanceof String) {
+                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+                if (value instanceof BigDecimal) {
+                    pstmt.setBigDecimal(i, (BigDecimal) value);
+                } else if (value instanceof Byte) {
+                    pstmt.setInt(i, ((Byte) value).intValue());
+                } else if (value instanceof Short) {
+                    pstmt.setInt(i, ((Short) value).intValue());
+                } else if (value instanceof Integer) {
+                    pstmt.setInt(i, (Integer) value);
+                } else if (value instanceof Long) {
+                    pstmt.setLong(i, (Long) value);
+                } else if (value instanceof Float) {
+                    pstmt.setBigDecimal(i, new BigDecimal((float) value));
+                } else if (value instanceof Double) {
+                    pstmt.setBigDecimal(i, new BigDecimal((double) value));
+                } else if (value != null) {
+                    pstmt.setBigDecimal(i, new BigDecimal(value.toString()));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.REAL:
+                if (value instanceof Number) {
+                    pstmt.setFloat(i, ((Number) value).floatValue());
+                } else if (value instanceof String) {
+                    pstmt.setFloat(i, Float.parseFloat((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                if (value instanceof Number) {
+                    pstmt.setDouble(i, ((Number) value).doubleValue());
+                } else if (value instanceof String) {
+                    pstmt.setDouble(i, Double.parseDouble((String) value));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                if (value instanceof Blob) {
+                    pstmt.setBlob(i, (Blob) value);
+                } else if (value instanceof byte[]) {
+                    pstmt.setBytes(i, (byte[]) value);
+                } else if (value instanceof String) {
+                    pstmt.setBytes(i, ((String) value).getBytes(StandardCharsets.ISO_8859_1));
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.CLOB:
+                if (value instanceof Clob) {
+                    pstmt.setClob(i, (Clob) value);
+                } else if (value instanceof byte[]) {
+                    pstmt.setBytes(i, (byte[]) value);
+                } else if (value instanceof String) {
+                    Reader clobReader = new StringReader((String) value);
+                    pstmt.setCharacterStream(i, clobReader);
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.DATE:
+                if (value instanceof Date) {
+                    pstmt.setDate(i, (Date) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setDate(i, new Date(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    if (!v.startsWith("0000-00-00")) {
+                        java.util.Date date = Util.parseDate(v);
+                        if (date != null) {
+                            pstmt.setDate(i, new Date(date.getTime()));
+                        } else {
+                            pstmt.setNull(i, type);
+                        }
+                    } else {
+                        pstmt.setObject(i, value);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.TIME:
+                if (value instanceof Time) {
+                    pstmt.setTime(i, (Time) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setTime(i, new Time(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    java.util.Date date = Util.parseDate(v);
+                    if (date != null) {
+                        pstmt.setTime(i, new Time(date.getTime()));
+                    } else {
+                        pstmt.setNull(i, type);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            case Types.TIMESTAMP:
+                if (value instanceof Timestamp) {
+                    pstmt.setTimestamp(i, (Timestamp) value);
+                } else if (value instanceof java.util.Date) {
+                    pstmt.setTimestamp(i, new Timestamp(((java.util.Date) value).getTime()));
+                } else if (value instanceof String) {
+                    String v = (String) value;
+                    if (!v.startsWith("0000-00-00")) {
+                        java.util.Date date = Util.parseDate(v);
+                        if (date != null) {
+                            pstmt.setTimestamp(i, new Timestamp(date.getTime()));
+                        } else {
+                            pstmt.setNull(i, type);
+                        }
+                    } else {
+                        pstmt.setObject(i, value);
+                    }
+                } else {
+                    pstmt.setNull(i, type);
+                }
+                break;
+            default:
+                pstmt.setObject(i, value, type);
+        }
+    }
+
+    public static String getDbTableName(MappingConfig.DbMapping dbMapping, String dbType) {
+        String result = "";
+        String backtick = getBacktickByDbType(dbType);
+        if (StringUtils.isNotEmpty(dbMapping.getTargetDb())) {
+            result += (backtick + dbMapping.getTargetDb() + backtick + ".");
+        }
+        result += (backtick + dbMapping.getTargetTable() + backtick);
+        return result;
+    }
+
+    public static String getSourceDbTableName(MappingConfig.DbMapping dbMapping, String dbType) {
+        String result = "";
+        String backtick = getBacktickByDbType(dbType);
+        if (StringUtils.isNotEmpty(dbMapping.getDatabase())) {
+            result += (backtick + dbMapping.getDatabase() + backtick + ".");
+        }
+
+        result += (backtick + dbMapping.getTable() + backtick);
+        return result;
+    }
+
+    /**
+     * 根据DbType返回反引号或空字符串
+     *
+     * @param dbTypeName DbType名称
+     * @return 反引号或空字符串
+     */
+    public static String getBacktickByDbType(String dbTypeName) {
+        DbType dbType = DbType.of(dbTypeName);
+        if (dbType == null) {
+            dbType = DbType.other;
+        }
+
+        // 只有当dbType为MySQL/MariaDB或OceanBase时返回反引号
+        switch (dbType) {
+            case mysql:
+            case mariadb:
+            case oceanbase:
+                return "`";
+            default:
+                return "";
+        }
+    }
+}

client-adapter/clickhouse/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -0,0 +1 @@
+clickhouse=com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter

client-adapter/clickhouse/src/main/resources/clickhouse/mytest_user.yml

@@ -0,0 +1,31 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+outerAdapterKey: clickhouse1
+concurrent: true
+dbMapping:
+  database: mytest
+  table: user
+  targetTable: mytest.user
+  targetPk:
+    id: id
+#  mapAll: true
+  targetColumns:
+    id:
+    name:
+    role_id:
+    c_time:
+    test1:
+  etlCondition: "where c_time>={}"
+  commitBatch: 3000 # 批量提交的大小
+
+
+## Mirror schema synchronize config
+#dataSourceKey: defaultDS
+#destination: example
+#groupId: g1
+#outerAdapterKey: mysql1
+#concurrent: true
+#dbMapping:
+#  mirrorDb: true
+#  database: mytest

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseBatchSyncServiceTest.java

@@ -0,0 +1,109 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse;
+
+import java.util.*;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.clickhouse.sync.Common;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+@Ignore
+public class ClickHouseBatchSyncServiceTest {
+
+    private ClickHouseAdapter clickHouseAdapter;
+
+    @Before
+    public void init() {
+        clickHouseAdapter = Common.init();
+    }
+
+    @Test
+    public void insert() throws InterruptedException {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("INSERT");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric");
+        data.put("role_id", 1L);
+        data.put("c_time", new Date());
+        data.put("test1", "sdfasdfawe中国asfwef");
+        dml.setData(dataList);
+
+        clickHouseAdapter.sync(Collections.singletonList(dml));
+        Thread.sleep(10000L);
+    }
+
+    @Test
+    public void update() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("UPDATE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric2");
+        dml.setData(dataList);
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("name", "Eric");
+        dml.setOld(oldList);
+
+        clickHouseAdapter.sync(Collections.singletonList(dml));
+    }
+
+    @Test
+    public void delete() {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("DELETE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric2");
+        dml.setData(dataList);
+
+        clickHouseAdapter.sync(Collections.singletonList(dml));
+    }
+
+    @Test
+    public void truncate() throws InterruptedException {
+        Dml dml = new Dml();
+        dml.setDestination("example");
+        dml.setTs(new Date().getTime());
+        dml.setType("TRUNCATE");
+        dml.setDatabase("mytest");
+        dml.setTable("user");
+        List<Map<String, Object>> dataList = new ArrayList<>();
+        Map<String, Object> data = new LinkedHashMap<>();
+        dataList.add(data);
+        data.put("id", 1L);
+        data.put("name", "Eric2");
+        dml.setData(dataList);
+        List<Map<String, Object>> oldList = new ArrayList<>();
+        Map<String, Object> old = new LinkedHashMap<>();
+        oldList.add(old);
+        old.put("name", "Eric");
+        dml.setOld(oldList);
+
+        clickHouseAdapter.sync(Collections.singletonList(dml));
+        Thread.sleep(1000L);
+
+    }
+}

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseBatchSyncThreadSafeTest.java

@@ -0,0 +1,81 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse;
+
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.adapter.clickhouse.sync.Common;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+import ch.qos.logback.classic.Level;
+
+/**
+ * @author: Xander
+ * @date: Created in 2023/11/13 22:27
+ * @email: zhrunxin33@gmail.com
+ * @description: Testing thread safe
+ */
+
+@Ignore
+public class ClickHouseBatchSyncThreadSafeTest {
+
+    private ClickHouseAdapter clickHouseAdapter;
+
+    private ExecutorService   executorService;
+
+    private String[]          operations = new String[] { "INSERT", "UPDATE" };
+
+    private String[]          tables     = new String[] { "user", "customer" };
+
+    @Before
+    public void init() {
+        clickHouseAdapter = Common.init();
+        Common.setLogLevel(Level.INFO);
+        executorService = Executors.newFixedThreadPool(5);
+    }
+
+    @Test
+    public void test01() throws InterruptedException, ExecutionException {
+        ArrayList<Future> list = new ArrayList();
+        AtomicInteger count = new AtomicInteger();
+        for (int i = 0; i < 10; i++) {
+            list.add(executorService.submit(() -> {
+                for (int j = 0; j < 300; j++) {
+                    Random random = new Random();
+                    int cou = count.incrementAndGet();
+                    // test insert
+                    String dmlType = operations[random.nextInt(1)];
+                    Dml dml = new Dml();
+                    dml.setDestination("example");
+                    dml.setTs(new Date().getTime());
+                    dml.setType(dmlType);
+                    dml.setDatabase("mytest");
+                    dml.setTable(tables[(int) Math.round(Math.random())]);
+                    List<Map<String, Object>> dataList = new ArrayList<>();
+                    Map<String, Object> data = new LinkedHashMap<>();
+                    dataList.add(data);
+                    data.put("id", cou);
+                    data.put("name", "Eric" + cou);
+                    data.put("role_id", cou);
+                    data.put("c_time", new Date());
+                    data.put("test1", "sdfasdfawe中国asfwef");
+                    dml.setData(dataList);
+                    clickHouseAdapter.sync(Collections.singletonList(dml));
+                }
+            }));
+
+        }
+        for (Future future : list) {
+            future.get();
+        }
+        Thread.sleep(10000L); // waiting multiple threads execute successfully.
+    }
+
+}

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/TestConstant.java

@@ -0,0 +1,37 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse;
+
+import java.sql.SQLException;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+public class TestConstant {
+
+    public final static String          jdbcUrl      = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
+    public final static String          jdbcUser     = "root";
+    public final static String          jdbcPassword = "121212";
+
+    public final static DruidDataSource dataSource;
+
+    static {
+        dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(jdbcUrl);
+        dataSource.setUsername(jdbcUser);
+        dataSource.setPassword(jdbcPassword);
+        dataSource.setInitialSize(1);
+        dataSource.setMinIdle(1);
+        dataSource.setMaxActive(1);
+        dataSource.setMaxWait(60000);
+        dataSource.setTimeBetweenEvictionRunsMillis(60000);
+        dataSource.setMinEvictableIdleTimeMillis(300000);
+        dataSource.setPoolPreparedStatements(false);
+        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
+        dataSource.setValidationQuery("select 1");
+        try {
+            dataSource.init();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+}

client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/sync/Common.java

@@ -0,0 +1,51 @@
+package com.alibaba.otter.canal.client.adapter.clickhouse.sync;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter;
+import com.alibaba.otter.canal.client.adapter.clickhouse.TestConstant;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+
+/**
+ * @author: Xander
+ * @date: Created in 2023/11/13 0:16
+ * @email: zhrunxin33@gmail.com @description：
+ */
+public class Common {
+
+    public static ClickHouseAdapter init() {
+        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
+
+        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
+        outerAdapterConfig.setName("clickhouse");
+        outerAdapterConfig.setKey("clickhouse1");
+        Map<String, String> properties = new HashMap<>();
+        properties.put("jdbc.driveClassName", "ru.yandex.clickhouse.ClickHouseDriver");
+        properties.put("jdbc.url", "jdbc:clickhouse://127.0.0.1:8123/default");
+        properties.put("jdbc.username", "default");
+        properties.put("jdbc.password", "123456");
+        outerAdapterConfig.setProperties(properties);
+
+        ClickHouseAdapter adapter = new ClickHouseAdapter();
+        adapter.init(outerAdapterConfig, null);
+        return adapter;
+    }
+
+    /**
+     * set global log level
+     *
+     * @param logLevel
+     */
+    public static void setLogLevel(Level logLevel) {
+        ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory
+            .getLogger(Logger.ROOT_LOGGER_NAME);
+        logger.setLevel(logLevel);
+    }
+}

client-adapter/clickhouse/src/test/resources/clickhouse/mytest_customer.yml

@@ -0,0 +1,31 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+outerAdapterKey: clickhouse1
+concurrent: true
+dbMapping:
+  database: mytest
+  table: customer
+  targetTable: mytest.customer
+  targetPk:
+    id: id
+#  mapAll: true
+  targetColumns:
+    id:
+    name:
+    role_id:
+    c_time:
+    test1:
+  etlCondition: "where c_time>={}"
+  commitBatch: 3000 # 批量提交的大小
+
+
+## Mirror schema synchronize config
+#dataSourceKey: defaultDS
+#destination: example
+#groupId: g1
+#outerAdapterKey: mysql1
+#concurrent: true
+#dbMapping:
+#  mirrorDb: true
+#  database: mytest

client-adapter/clickhouse/src/test/resources/clickhouse/mytest_user.yml

@@ -0,0 +1,31 @@
+dataSourceKey: defaultDS
+destination: example
+groupId: g1
+outerAdapterKey: clickhouse1
+concurrent: true
+dbMapping:
+  database: mytest
+  table: user
+  targetTable: mytest.user
+  targetPk:
+    id: id
+#  mapAll: true
+  targetColumns:
+    id:
+    name:
+    role_id:
+    c_time:
+    test1:
+  etlCondition: "where c_time>={}"
+  commitBatch: 3000 # 批量提交的大小
+
+
+## Mirror schema synchronize config
+#dataSourceKey: defaultDS
+#destination: example
+#groupId: g1
+#outerAdapterKey: mysql1
+#concurrent: true
+#dbMapping:
+#  mirrorDb: true
+#  database: mytest

client-adapter/common/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
+        <version>1.1.8</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -18,29 +18,13 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.springframework</groupId>
+            <groupId>com.alibaba.otter</groupId>
-            <artifactId>spring-core</artifactId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.springframework</groupId>
+            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-aop</artifactId>
+            <artifactId>spring-boot</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-context</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-jdbc</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-orm</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-test</artifactId>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>joda-time</groupId>
@@ -50,10 +34,6 @@
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.powermock</groupId>
             <artifactId>powermock-api-mockito</artifactId>
@@ -74,11 +54,15 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>com.diffblue</groupId>
             <artifactId>deeptestutils</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>
-
 </project>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/ProxyOuterAdapter.java

@@ -0,0 +1,87 @@
+package com.alibaba.otter.canal.client.adapter;
+
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.EtlResult;
+import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class ProxyOuterAdapter implements OuterAdapter {
+
+    private OuterAdapter outerAdapter;
+
+    public ProxyOuterAdapter(OuterAdapter outerAdapter) {
+        this.outerAdapter = outerAdapter;
+    }
+
+    private ClassLoader changeCL() {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(outerAdapter.getClass().getClassLoader());
+        return cl;
+    }
+
+    private void revertCL(ClassLoader cl) {
+        Thread.currentThread().setContextClassLoader(cl);
+    }
+
+    @Override
+    public void init(OuterAdapterConfig configuration, Properties envProperties) {
+        ClassLoader cl = changeCL();
+        try {
+            outerAdapter.init(configuration, envProperties);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void sync(List<Dml> dmls) {
+        ClassLoader cl = changeCL();
+        try {
+            outerAdapter.sync(dmls);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public void destroy() {
+        ClassLoader cl = changeCL();
+        try {
+            outerAdapter.destroy();
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public EtlResult etl(String task, List<String> params) {
+        ClassLoader cl = changeCL();
+        try {
+            return OuterAdapter.super.etl(task, params);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public Map<String, Object> count(String task) {
+        ClassLoader cl = changeCL();
+        try {
+            return OuterAdapter.super.count(task);
+        } finally {
+            revertCL(cl);
+        }
+    }
+
+    @Override
+    public String getDestination(String task) {
+        ClassLoader cl = changeCL();
+        try {
+            return OuterAdapter.super.getDestination(task);
+        } finally {
+            revertCL(cl);
+        }
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java

@@ -1,155 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.springframework.util.PropertyPlaceholderHelper;
-import org.springframework.util.StringUtils;
-
-import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory;
-import com.alibaba.otter.canal.client.adapter.config.common.*;
-
-/**
- * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范
- *
- * @author reweerma 2019-2-1 上午9:14:02
- * @version 1.0.0
- */
-public class YmlConfigBinder {
-
-    /**
-     * 将当前内容绑定到指定对象
-     *
-     * @param content yml内容
-     * @param clazz 指定对象类型
-     * @return 对象
-     */
-    public static <T> T bindYmlToObj(String content, Class<T> clazz) {
-        return bindYmlToObj(null, content, clazz, null);
-    }
-
-    /**
-     * 将当前内容绑定到指定对象并指定内容编码格式
-     *
-     * @param content yml内容
-     * @param clazz 指定对象类型
-     * @param charset yml内容编码格式
-     * @return 对象
-     */
-    public static <T> T bindYmlToObj(String content, Class<T> clazz, String charset) {
-        return bindYmlToObj(null, content, clazz, charset);
-    }
-
-    /**
-     * 将当前内容指定前缀部分绑定到指定对象
-     *
-     * @param prefix 指定前缀
-     * @param content yml内容
-     * @param clazz 指定对象类型
-     * @return 对象
-     */
-    public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz) {
-        return bindYmlToObj(prefix, content, clazz, null);
-    }
-
-    /**
-     * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式
-     *
-     * @param prefix 指定前缀
-     * @param content yml内容
-     * @param clazz 指定对象类型
-     * @param charset yml内容编码格式
-     * @return 对象
-     */
-    public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset) {
-        return bindYmlToObj(prefix, content, clazz, charset, null);
-    }
-
-    /**
-     * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers}
-     * 在envProperties中有属性 zookeeper.servers:
-     * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为
-     * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中
-     * zkServers 属性是 List<String> 对象, 则会自动映射成List
-     *
-     * @param prefix 指定前缀
-     * @param content yml内容
-     * @param clazz 指定对象类型
-     * @param charset yml内容编码格式
-     * @return 对象
-     */
-    public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset,
-                                     Properties baseProperties) {
-        try {
-            byte[] contentBytes;
-            if (charset == null) {
-                contentBytes = content.getBytes("UTF-8");
-            } else {
-                contentBytes = content.getBytes(charset);
-            }
-            YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
-            Resource configResource = new ByteArrayResource(contentBytes);
-            PropertySource<?> propertySource = propertySourceLoader.load("manualBindConfig", configResource, null);
-
-            if (propertySource == null) {
-                return null;
-            }
-
-            Properties properties = new Properties();
-            Map<String, Object> propertiesRes = new LinkedHashMap<>();
-            if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) {
-                prefix = prefix + ".";
-            }
-
-            properties.putAll((Map<?, ?>) propertySource.getSource());
-
-            if (baseProperties != null) {
-                baseProperties.putAll(properties);
-                properties = baseProperties;
-            }
-
-            for (Map.Entry<?, ?> entry : ((Map<?, ?>) propertySource.getSource()).entrySet()) {
-                String key = (String) entry.getKey();
-                Object value = entry.getValue();
-
-                if (prefix != null) {
-                    if (key != null && key.startsWith(prefix)) {
-                        key = key.substring(prefix.length());
-                    } else {
-                        continue;
-                    }
-                }
-
-                if (value instanceof String && ((String) value).contains("${") && ((String) value).contains("}")) {
-                    PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}");
-                    value = propertyPlaceholderHelper.replacePlaceholders((String) value, properties);
-                }
-
-                propertiesRes.put(key, value);
-            }
-
-            if (propertiesRes.isEmpty()) {
-                return null;
-            }
-
-            propertySource = new MapPropertySource(propertySource.getName(), propertiesRes);
-
-            T target = clazz.newInstance();
-
-            MutablePropertySources propertySources = new MutablePropertySources();
-            propertySources.addFirst(propertySource);
-
-            PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target);
-            factory.setPropertySources(propertySources);
-            factory.setIgnoreInvalidFields(true);
-            factory.setIgnoreUnknownFields(true);
-
-            factory.bindPropertiesToTarget();
-
-            return target;
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java

@@ -1,97 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * {@link PropertyNamePatternsMatcher} that matches when a property name exactly
- * matches one of the given names, or starts with one of the given names
- * followed by a delimiter. This implementation is optimized for frequent calls.
- *
- * @author Phillip Webb
- * @since 1.2.0
- */
-class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher {
-
-    private final char[]   delimiters;
-
-    private final boolean  ignoreCase;
-
-    private final String[] names;
-
-    protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){
-        this(delimiters, false, names);
-    }
-
-    protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
-        this(delimiters, ignoreCase, new HashSet<String>(Arrays.asList(names)));
-    }
-
-    DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){
-        this.delimiters = delimiters;
-        this.ignoreCase = ignoreCase;
-        this.names = names.toArray(new String[names.size()]);
-    }
-
-    @Override
-    public boolean matches(String propertyName) {
-        char[] propertyNameChars = propertyName.toCharArray();
-        boolean[] match = new boolean[this.names.length];
-        boolean noneMatched = true;
-        for (int i = 0; i < this.names.length; i++) {
-            if (this.names[i].length() <= propertyNameChars.length) {
-                match[i] = true;
-                noneMatched = false;
-            }
-        }
-        if (noneMatched) {
-            return false;
-        }
-        for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) {
-            for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) {
-                if (match[nameIndex]) {
-                    match[nameIndex] = false;
-                    if (charIndex < this.names[nameIndex].length()) {
-                        if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) {
-                            match[nameIndex] = true;
-                            noneMatched = false;
-                        }
-                    } else {
-                        char charAfter = propertyNameChars[this.names[nameIndex].length()];
-                        if (isDelimiter(charAfter)) {
-                            match[nameIndex] = true;
-                            noneMatched = false;
-                        }
-                    }
-                }
-            }
-            if (noneMatched) {
-                return false;
-            }
-        }
-        for (int i = 0; i < match.length; i++) {
-            if (match[i]) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private boolean isCharMatch(char c1, char c2) {
-        if (this.ignoreCase) {
-            return Character.toLowerCase(c1) == Character.toLowerCase(c2);
-        }
-        return c1 == c2;
-    }
-
-    private boolean isDelimiter(char c) {
-        for (char delimiter : this.delimiters) {
-            if (c == delimiter) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java

@@ -1,31 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.beans.PropertyEditorSupport;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-/**
- * {@link PropertyNamePatternsMatcher} that matches when a property name exactly
- * matches one of the given names, or starts with one of the given names
- * followed by a delimiter. This implementation is optimized for frequent calls.
- *
- * @author Phillip Webb
- * @since 1.2.0
- */
-class InetAddressEditor extends PropertyEditorSupport {
-
-    @Override
-    public String getAsText() {
-        return ((InetAddress) getValue()).getHostAddress();
-    }
-
-    @Override
-    public void setAsText(String text) throws IllegalArgumentException {
-        try {
-            setValue(InetAddress.getByName(text));
-        } catch (UnknownHostException ex) {
-            throw new IllegalArgumentException("Cannot locate host", ex);
-        }
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java

@@ -1,52 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import org.springframework.beans.PropertyValue;
-
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
-
-/**
- * A {@link PropertyValue} that can provide information about its origin.
- *
- * @author Andy Wilkinson
- */
-class OriginCapablePropertyValue extends PropertyValue {
-
-    private static final String  ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin";
-
-    private final PropertyOrigin origin;
-
-    OriginCapablePropertyValue(PropertyValue propertyValue){
-        this(propertyValue.getName(),
-            propertyValue.getValue(),
-            (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN));
-    }
-
-    OriginCapablePropertyValue(String name, Object value, String originName, PropertySource<?> originSource){
-        this(name, value, new PropertyOrigin(originSource, originName));
-    }
-
-    OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){
-        super(name, value);
-        this.origin = origin;
-        setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin);
-    }
-
-    public PropertyOrigin getOrigin() {
-        return this.origin;
-    }
-
-    @Override
-    public String toString() {
-        String name = (this.origin != null ? this.origin.getName() : this.getName());
-        String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown");
-        return "'" + name + "' from '" + source + "'";
-    }
-
-    public static PropertyOrigin getOrigin(PropertyValue propertyValue) {
-        if (propertyValue instanceof OriginCapablePropertyValue) {
-            return ((OriginCapablePropertyValue) propertyValue).getOrigin();
-        }
-        return new OriginCapablePropertyValue(propertyValue).getOrigin();
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java

@@ -1,27 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.util.Collection;
-
-import org.springframework.util.PatternMatchUtils;
-
-/**
- * {@link PropertyNamePatternsMatcher} that delegates to
- * {@link PatternMatchUtils#simpleMatch(String[], String)}.
- *
- * @author Phillip Webb
- * @since 1.2.0
- */
-class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher {
-
-    private final String[] patterns;
-
-    PatternPropertyNamePatternsMatcher(Collection<String> patterns){
-        this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {});
-    }
-
-    @Override
-    public boolean matches(String propertyName) {
-        return PatternMatchUtils.simpleMatch(this.patterns, propertyName);
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java

@@ -1,356 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.beans.PropertyDescriptor;
-import java.util.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.PropertyValues;
-import org.springframework.beans.factory.FactoryBean;
-import org.springframework.beans.factory.InitializingBean;
-import org.springframework.beans.support.ResourceEditorRegistrar;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-import org.springframework.context.MessageSource;
-import org.springframework.context.MessageSourceAware;
-import org.springframework.core.convert.ConversionService;
-import org.springframework.util.Assert;
-import org.springframework.util.StringUtils;
-import org.springframework.validation.*;
-
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
-
-/**
- * Validate some {@link Properties} (or optionally
- * {@link org.springframework.core.env.PropertySources}) by binding them to an
- * object of a specified type and then optionally running a {@link Validator}
- * over it.
- *
- * @param <T> the target type
- * @author Dave Syer
- */
-public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, ApplicationContextAware, MessageSourceAware, InitializingBean {
-
-    private static final char[] EXACT_DELIMITERS       = { '_', '.', '[' };
-
-    private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' };
-
-    private static final Log    logger                 = LogFactory.getLog(PropertiesConfigurationFactory.class);
-
-    private boolean             ignoreUnknownFields    = true;
-
-    private boolean             ignoreInvalidFields;
-
-    private boolean             exceptionIfInvalid     = true;
-
-    private PropertySources     propertySources;
-
-    private final T             target;
-
-    private Validator           validator;
-
-    private ApplicationContext  applicationContext;
-
-    private MessageSource       messageSource;
-
-    private boolean             hasBeenBound           = false;
-
-    private boolean             ignoreNestedProperties = false;
-
-    private String              targetName;
-
-    private ConversionService   conversionService;
-
-    private boolean             resolvePlaceholders    = true;
-
-    /**
-     * Create a new {@link PropertiesConfigurationFactory} instance.
-     *
-     * @param target the target object to bind too
-     * @see #PropertiesConfigurationFactory(Class)
-     */
-    public PropertiesConfigurationFactory(T target){
-        Assert.notNull(target, "target must not be null");
-        this.target = target;
-    }
-
-    /**
-     * Create a new {@link PropertiesConfigurationFactory} instance.
-     *
-     * @param type the target type
-     * @see #PropertiesConfigurationFactory(Class)
-     */
-    @SuppressWarnings("unchecked")
-    public PropertiesConfigurationFactory(Class<?> type){
-        Assert.notNull(type, "type must not be null");
-        this.target = (T) BeanUtils.instantiate(type);
-    }
-
-    /**
-     * Flag to disable binding of nested properties (i.e. those with period
-     * separators in their paths). Can be useful to disable this if the name prefix
-     * is empty and you don't want to ignore unknown fields.
-     *
-     * @param ignoreNestedProperties the flag to set (default false)
-     */
-    public void setIgnoreNestedProperties(boolean ignoreNestedProperties) {
-        this.ignoreNestedProperties = ignoreNestedProperties;
-    }
-
-    /**
-     * Set whether to ignore unknown fields, that is, whether to ignore bind
-     * parameters that do not have corresponding fields in the target object.
-     * <p>
-     * Default is "true". Turn this off to enforce that all bind parameters must
-     * have a matching field in the target object.
-     *
-     * @param ignoreUnknownFields if unknown fields should be ignored
-     */
-    public void setIgnoreUnknownFields(boolean ignoreUnknownFields) {
-        this.ignoreUnknownFields = ignoreUnknownFields;
-    }
-
-    /**
-     * Set whether to ignore invalid fields, that is, whether to ignore bind
-     * parameters that have corresponding fields in the target object which are not
-     * accessible (for example because of null values in the nested path).
-     * <p>
-     * Default is "false". Turn this on to ignore bind parameters for nested objects
-     * in non-existing parts of the target object graph.
-     *
-     * @param ignoreInvalidFields if invalid fields should be ignored
-     */
-    public void setIgnoreInvalidFields(boolean ignoreInvalidFields) {
-        this.ignoreInvalidFields = ignoreInvalidFields;
-    }
-
-    /**
-     * Set the target name.
-     *
-     * @param targetName the target name
-     */
-    public void setTargetName(String targetName) {
-        this.targetName = targetName;
-    }
-
-    @Override
-    public void setApplicationContext(ApplicationContext applicationContext) {
-        this.applicationContext = applicationContext;
-    }
-
-    /**
-     * Set the message source.
-     *
-     * @param messageSource the message source
-     */
-    @Override
-    public void setMessageSource(MessageSource messageSource) {
-        this.messageSource = messageSource;
-    }
-
-    /**
-     * Set the property sources.
-     *
-     * @param propertySources the property sources
-     */
-    public void setPropertySources(PropertySources propertySources) {
-        this.propertySources = propertySources;
-    }
-
-    /**
-     * Set the conversion service.
-     *
-     * @param conversionService the conversion service
-     */
-    public void setConversionService(ConversionService conversionService) {
-        this.conversionService = conversionService;
-    }
-
-    /**
-     * Set the validator.
-     *
-     * @param validator the validator
-     */
-    public void setValidator(Validator validator) {
-        this.validator = validator;
-    }
-
-    /**
-     * Set a flag to indicate that an exception should be raised if a Validator is
-     * available and validation fails.
-     *
-     * @param exceptionIfInvalid the flag to set
-     * @deprecated as of 1.5, do not specify a {@link Validator} if validation
-     * should not occur
-     */
-    @Deprecated
-    public void setExceptionIfInvalid(boolean exceptionIfInvalid) {
-        this.exceptionIfInvalid = exceptionIfInvalid;
-    }
-
-    /**
-     * Flag to indicate that placeholders should be replaced during binding. Default
-     * is true.
-     *
-     * @param resolvePlaceholders flag value
-     */
-    public void setResolvePlaceholders(boolean resolvePlaceholders) {
-        this.resolvePlaceholders = resolvePlaceholders;
-    }
-
-    @Override
-    public void afterPropertiesSet() throws Exception {
-        bindPropertiesToTarget();
-    }
-
-    @Override
-    public Class<?> getObjectType() {
-        if (this.target == null) {
-            return Object.class;
-        }
-        return this.target.getClass();
-    }
-
-    @Override
-    public boolean isSingleton() {
-        return true;
-    }
-
-    @Override
-    public T getObject() throws Exception {
-        if (!this.hasBeenBound) {
-            bindPropertiesToTarget();
-        }
-        return this.target;
-    }
-
-    public void bindPropertiesToTarget() throws BindException {
-        Assert.state(this.propertySources != null, "PropertySources should not be null");
-        try {
-            if (logger.isTraceEnabled()) {
-                logger.trace("Property Sources: " + this.propertySources);
-
-            }
-            this.hasBeenBound = true;
-            doBindPropertiesToTarget();
-        } catch (BindException ex) {
-            if (this.exceptionIfInvalid) {
-                throw ex;
-            }
-            logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex);
-        }
-    }
-
-    private void doBindPropertiesToTarget() throws BindException {
-        RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target,
-            this.targetName) : new RelaxedDataBinder(this.target));
-        if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) {
-            dataBinder.setValidator(this.validator);
-        }
-        if (this.conversionService != null) {
-            dataBinder.setConversionService(this.conversionService);
-        }
-        dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE);
-        dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties);
-        dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields);
-        dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields);
-        customizeBinder(dataBinder);
-        if (this.applicationContext != null) {
-            ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext,
-                this.applicationContext.getEnvironment());
-            resourceEditorRegistrar.registerCustomEditors(dataBinder);
-        }
-        Iterable<String> relaxedTargetNames = getRelaxedTargetNames();
-        Set<String> names = getNames(relaxedTargetNames);
-        PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames);
-        dataBinder.bind(propertyValues);
-        if (this.validator != null) {
-            dataBinder.validate();
-        }
-        checkForBindingErrors(dataBinder);
-    }
-
-    private Iterable<String> getRelaxedTargetNames() {
-        return (this.target != null
-                && StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null);
-    }
-
-    private Set<String> getNames(Iterable<String> prefixes) {
-        Set<String> names = new LinkedHashSet<String>();
-        if (this.target != null) {
-            PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
-            for (PropertyDescriptor descriptor : descriptors) {
-                String name = descriptor.getName();
-                if (!name.equals("class")) {
-                    RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name);
-                    if (prefixes == null) {
-                        for (String relaxedName : relaxedNames) {
-                            names.add(relaxedName);
-                        }
-                    } else {
-                        for (String prefix : prefixes) {
-                            for (String relaxedName : relaxedNames) {
-                                names.add(prefix + "." + relaxedName);
-                                names.add(prefix + "_" + relaxedName);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        return names;
-    }
-
-    private PropertyValues getPropertySourcesPropertyValues(Set<String> names, Iterable<String> relaxedTargetNames) {
-        PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames);
-        return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders);
-    }
-
-    private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set<String> names,
-                                                                       Iterable<String> relaxedTargetNames) {
-        if (this.ignoreUnknownFields && !isMapTarget()) {
-            // Since unknown fields are ignored we can filter them out early to save
-            // unnecessary calls to the PropertySource.
-            return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names);
-        }
-        if (relaxedTargetNames != null) {
-            // We can filter properties to those starting with the target name, but
-            // we can't do a complete filter since we need to trigger the
-            // unknown fields check
-            Set<String> relaxedNames = new HashSet<String>();
-            for (String relaxedTargetName : relaxedTargetNames) {
-                relaxedNames.add(relaxedTargetName);
-            }
-            return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames);
-        }
-        // Not ideal, we basically can't filter anything
-        return PropertyNamePatternsMatcher.ALL;
-    }
-
-    private boolean isMapTarget() {
-        return this.target != null && Map.class.isAssignableFrom(this.target.getClass());
-    }
-
-    private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException {
-        BindingResult errors = dataBinder.getBindingResult();
-        if (errors.hasErrors()) {
-            logger.error("Properties configuration failed validation");
-            for (ObjectError error : errors.getAllErrors()) {
-                logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault())
-                                                          + " (" + error + ")" : error);
-            }
-            if (this.exceptionIfInvalid) {
-                throw new BindException(errors);
-            }
-        }
-    }
-
-    /**
-     * Customize the data binder.
-     *
-     * @param dataBinder the data binder that will be used to bind and validate
-     */
-    protected void customizeBinder(DataBinder dataBinder) {
-    }
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java

@@ -1,38 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-/**
- * Strategy interface used to check if a property name matches specific
- * criteria.
- *
- * @author Phillip Webb
- * @since 1.2.0
- */
-interface PropertyNamePatternsMatcher {
-
-    PropertyNamePatternsMatcher ALL  = new PropertyNamePatternsMatcher() {
-
-                                         @Override
-                                         public boolean matches(String propertyName) {
-                                             return true;
-                                         }
-
-                                     };
-
-    PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() {
-
-                                         @Override
-                                         public boolean matches(String propertyName) {
-                                             return false;
-                                         }
-
-                                     };
-
-    /**
-     * Return {@code true} of the property name matches.
-     *
-     * @param propertyName the property name
-     * @return {@code true} if the property name matches
-     */
-    boolean matches(String propertyName);
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java

@@ -1,30 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
-
-/**
- * The origin of a property, specifically its source and its name before any
- * prefix was removed.
- *
- * @author Andy Wilkinson
- * @since 1.3.0
- */
-public class PropertyOrigin {
-
-    private final PropertySource<?> source;
-
-    private final String            name;
-
-    PropertyOrigin(PropertySource<?> source, String name){
-        this.name = name;
-        this.source = source;
-    }
-
-    public PropertySource<?> getSource() {
-        return this.source;
-    }
-
-    public String getName() {
-        return this.name;
-    }
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java

@@ -1,164 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import org.springframework.core.convert.ConversionException;
-import org.springframework.core.env.AbstractEnvironment;
-import org.springframework.core.env.AbstractPropertyResolver;
-import org.springframework.core.env.PropertyResolver;
-import org.springframework.util.ClassUtils;
-
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
-
-/**
- * {@link PropertyResolver} implementation that resolves property values against
- * an underlying set of {@link PropertySources}.
- *
- * @author Chris Beams
- * @author Juergen Hoeller
- * @see PropertySource
- * @see PropertySources
- * @see AbstractEnvironment
- * @since 3.1
- */
-public class PropertySourcesPropertyResolver extends AbstractPropertyResolver {
-
-    private final PropertySources propertySources;
-
-    /**
-     * Create a new resolver against the given property sources.
-     *
-     * @param propertySources the set of {@link PropertySource} objects to use
-     */
-    public PropertySourcesPropertyResolver(PropertySources propertySources){
-        this.propertySources = propertySources;
-    }
-
-    @Override
-    public boolean containsProperty(String key) {
-        if (this.propertySources != null) {
-            for (PropertySource<?> propertySource : this.propertySources) {
-                if (propertySource.containsProperty(key)) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public String getProperty(String key) {
-        return getProperty(key, String.class, true);
-    }
-
-    @Override
-    public <T> T getProperty(String key, Class<T> targetValueType) {
-        return getProperty(key, targetValueType, true);
-    }
-
-    @Override
-    protected String getPropertyAsRawString(String key) {
-        return getProperty(key, String.class, false);
-    }
-
-    protected <T> T getProperty(String key, Class<T> targetValueType, boolean resolveNestedPlaceholders) {
-        if (this.propertySources != null) {
-            for (PropertySource<?> propertySource : this.propertySources) {
-                if (logger.isTraceEnabled()) {
-                    logger
-                        .trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'");
-                }
-                Object value = propertySource.getProperty(key);
-                if (value != null) {
-                    if (resolveNestedPlaceholders && value instanceof String) {
-                        value = resolveNestedPlaceholders((String) value);
-                    }
-                    logKeyFound(key, propertySource, value);
-                    return convertValueIfNecessary(value, targetValueType);
-                }
-            }
-        }
-        if (logger.isDebugEnabled()) {
-            logger.debug("Could not find key '" + key + "' in any property source");
-        }
-        return null;
-    }
-
-    @Deprecated
-    public <T> Class<T> getPropertyAsClass(String key, Class<T> targetValueType) {
-        if (this.propertySources != null) {
-            for (PropertySource<?> propertySource : this.propertySources) {
-                if (logger.isTraceEnabled()) {
-                    logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName()));
-                }
-                Object value = propertySource.getProperty(key);
-                if (value != null) {
-                    logKeyFound(key, propertySource, value);
-                    Class<?> clazz;
-                    if (value instanceof String) {
-                        try {
-                            clazz = ClassUtils.forName((String) value, null);
-                        } catch (Exception ex) {
-                            throw new PropertySourcesPropertyResolver.ClassConversionException((String) value,
-                                targetValueType,
-                                ex);
-                        }
-                    } else if (value instanceof Class) {
-                        clazz = (Class<?>) value;
-                    } else {
-                        clazz = value.getClass();
-                    }
-                    if (!targetValueType.isAssignableFrom(clazz)) {
-                        throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType);
-                    }
-                    @SuppressWarnings("unchecked")
-                    Class<T> targetClass = (Class<T>) clazz;
-                    return targetClass;
-                }
-            }
-        }
-        if (logger.isDebugEnabled()) {
-            logger.debug(String.format("Could not find key '%s' in any property source", key));
-        }
-        return null;
-    }
-
-    /**
-     * Log the given key as found in the given {@link PropertySource}, resulting in
-     * the given value.
-     * <p>
-     * The default implementation writes a debug log message with key and source. As
-     * of 4.3.3, this does not log the value anymore in order to avoid accidental
-     * logging of sensitive settings. Subclasses may override this method to change
-     * the log level and/or log message, including the property's value if desired.
-     *
-     * @param key the key found
-     * @param propertySource the {@code PropertySource} that the key has been found
-     *     in
-     * @param value the corresponding value
-     * @since 4.3.1
-     */
-    protected void logKeyFound(String key, PropertySource<?> propertySource, Object value) {
-        if (logger.isDebugEnabled()) {
-            logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName()
-                         + "' with value of type " + value.getClass().getSimpleName());
-        }
-    }
-
-    @SuppressWarnings("serial")
-    @Deprecated
-    private static class ClassConversionException extends ConversionException {
-
-        public ClassConversionException(Class<?> actual, Class<?> expected){
-            super(String
-                .format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName()));
-        }
-
-        public ClassConversionException(String actual, Class<?> expected, Exception ex){
-            super(
-                String
-                    .format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()),
-                ex);
-        }
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java

@@ -1,233 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Pattern;
-
-import org.springframework.beans.MutablePropertyValues;
-import org.springframework.beans.PropertyValue;
-import org.springframework.beans.PropertyValues;
-import org.springframework.util.Assert;
-import org.springframework.validation.DataBinder;
-
-import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource;
-import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource;
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
-import com.alibaba.otter.canal.client.adapter.config.common.PropertySources;
-
-/**
- * A {@link PropertyValues} implementation backed by a {@link PropertySources},
- * bridging the two abstractions and allowing (for instance) a regular
- * {@link DataBinder} to be used with the latter.
- *
- * @author Dave Syer
- * @author Phillip Webb
- */
-public class PropertySourcesPropertyValues implements PropertyValues {
-
-    private static final Pattern                               COLLECTION_PROPERTY = Pattern
-        .compile("\\[(\\d+)\\](\\.\\S+)?");
-
-    private final PropertySources                              propertySources;
-
-    private final Collection<String>                           nonEnumerableFallbackNames;
-
-    private final PropertyNamePatternsMatcher                  includes;
-
-    private final Map<String, PropertyValue>                   propertyValues      = new LinkedHashMap<String, PropertyValue>();
-
-    private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners    = new ConcurrentHashMap<String, PropertySource<?>>();
-
-    private final boolean                                      resolvePlaceholders;
-
-    /**
-     * Create a new PropertyValues from the given PropertySources.
-     *
-     * @param propertySources a PropertySources instance
-     */
-    public PropertySourcesPropertyValues(PropertySources propertySources){
-        this(propertySources, true);
-    }
-
-    /**
-     * Create a new PropertyValues from the given PropertySources that will
-     * optionally resolve placeholders.
-     *
-     * @param propertySources a PropertySources instance
-     * @param resolvePlaceholders {@code true} if placeholders should be resolved.
-     * @since 1.5.2
-     */
-    public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){
-        this(propertySources, (Collection<String>) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders);
-    }
-
-    /**
-     * Create a new PropertyValues from the given PropertySources.
-     *
-     * @param propertySources a PropertySources instance
-     * @param includePatterns property name patterns to include from system
-     *     properties and environment variables
-     * @param nonEnumerableFallbackNames the property names to try in lieu of an
-     *     {@link EnumerablePropertySource}.
-     */
-    public PropertySourcesPropertyValues(PropertySources propertySources, Collection<String> includePatterns,
-                                         Collection<String> nonEnumerableFallbackNames){
-        this(propertySources,
-            nonEnumerableFallbackNames,
-            new PatternPropertyNamePatternsMatcher(includePatterns),
-            true);
-    }
-
-    /**
-     * Create a new PropertyValues from the given PropertySources.
-     *
-     * @param propertySources a PropertySources instance
-     * @param nonEnumerableFallbackNames the property names to try in lieu of an
-     *     {@link EnumerablePropertySource}.
-     * @param includes the property name patterns to include
-     * @param resolvePlaceholders flag to indicate the placeholders should be
-     *     resolved
-     */
-    PropertySourcesPropertyValues(PropertySources propertySources, Collection<String> nonEnumerableFallbackNames,
-                                  PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){
-        Assert.notNull(propertySources, "PropertySources must not be null");
-        Assert.notNull(includes, "Includes must not be null");
-        this.propertySources = propertySources;
-        this.nonEnumerableFallbackNames = nonEnumerableFallbackNames;
-        this.includes = includes;
-        this.resolvePlaceholders = resolvePlaceholders;
-        PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources);
-        for (PropertySource<?> source : propertySources) {
-            processPropertySource(source, resolver);
-        }
-    }
-
-    private void processPropertySource(PropertySource<?> source, PropertySourcesPropertyResolver resolver) {
-        if (source instanceof CompositePropertySource) {
-            processCompositePropertySource((CompositePropertySource) source, resolver);
-        } else if (source instanceof EnumerablePropertySource) {
-            processEnumerablePropertySource((EnumerablePropertySource<?>) source, resolver, this.includes);
-        } else {
-            processNonEnumerablePropertySource(source, resolver);
-        }
-    }
-
-    private void processCompositePropertySource(CompositePropertySource source,
-                                                PropertySourcesPropertyResolver resolver) {
-        for (PropertySource<?> nested : source.getPropertySources()) {
-            processPropertySource(nested, resolver);
-        }
-    }
-
-    private void processEnumerablePropertySource(EnumerablePropertySource<?> source,
-                                                 PropertySourcesPropertyResolver resolver,
-                                                 PropertyNamePatternsMatcher includes) {
-        if (source.getPropertyNames().length > 0) {
-            for (String propertyName : source.getPropertyNames()) {
-                if (includes.matches(propertyName)) {
-                    Object value = getEnumerableProperty(source, resolver, propertyName);
-                    putIfAbsent(propertyName, value, source);
-                }
-            }
-        }
-    }
-
-    private Object getEnumerableProperty(EnumerablePropertySource<?> source, PropertySourcesPropertyResolver resolver,
-                                         String propertyName) {
-        try {
-            if (this.resolvePlaceholders) {
-                return resolver.getProperty(propertyName, Object.class);
-            }
-        } catch (RuntimeException ex) {
-            // Probably could not resolve placeholders, ignore it here
-        }
-        return source.getProperty(propertyName);
-    }
-
-    private void processNonEnumerablePropertySource(PropertySource<?> source,
-                                                    PropertySourcesPropertyResolver resolver) {
-        // We can only do exact matches for non-enumerable property names, but
-        // that's better than nothing...
-        if (this.nonEnumerableFallbackNames == null) {
-            return;
-        }
-        for (String propertyName : this.nonEnumerableFallbackNames) {
-            if (!source.containsProperty(propertyName)) {
-                continue;
-            }
-            Object value = null;
-            try {
-                value = resolver.getProperty(propertyName, Object.class);
-            } catch (RuntimeException ex) {
-                // Probably could not convert to Object, weird, but ignorable
-            }
-            if (value == null) {
-                value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH));
-            }
-            putIfAbsent(propertyName, value, source);
-        }
-    }
-
-    @Override
-    public PropertyValue[] getPropertyValues() {
-        Collection<PropertyValue> values = this.propertyValues.values();
-        return values.toArray(new PropertyValue[values.size()]);
-    }
-
-    @Override
-    public PropertyValue getPropertyValue(String propertyName) {
-        PropertyValue propertyValue = this.propertyValues.get(propertyName);
-        if (propertyValue != null) {
-            return propertyValue;
-        }
-        for (PropertySource<?> source : this.propertySources) {
-            Object value = source.getProperty(propertyName);
-            propertyValue = putIfAbsent(propertyName, value, source);
-            if (propertyValue != null) {
-                return propertyValue;
-            }
-        }
-        return null;
-    }
-
-    private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource<?> source) {
-        if (value != null && !this.propertyValues.containsKey(propertyName)) {
-            PropertySource<?> collectionOwner = this.collectionOwners
-                .putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source);
-            if (collectionOwner == null || collectionOwner == source) {
-                PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source);
-                this.propertyValues.put(propertyName, propertyValue);
-                return propertyValue;
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public PropertyValues changesSince(PropertyValues old) {
-        MutablePropertyValues changes = new MutablePropertyValues();
-        // for each property value in the new set
-        for (PropertyValue newValue : getPropertyValues()) {
-            // if there wasn't an old one, add it
-            PropertyValue oldValue = old.getPropertyValue(newValue.getName());
-            if (oldValue == null || !oldValue.equals(newValue)) {
-                changes.addPropertyValue(newValue);
-            }
-        }
-        return changes;
-    }
-
-    @Override
-    public boolean contains(String propertyName) {
-        return getPropertyValue(propertyName) != null;
-    }
-
-    @Override
-    public boolean isEmpty() {
-        return this.propertyValues.isEmpty();
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java

@@ -1,127 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.util.EnumSet;
-import java.util.Locale;
-import java.util.Set;
-
-import org.springframework.core.convert.ConversionFailedException;
-import org.springframework.core.convert.ConversionService;
-import org.springframework.core.convert.TypeDescriptor;
-import org.springframework.core.convert.converter.Converter;
-import org.springframework.core.convert.converter.ConverterFactory;
-import org.springframework.core.convert.support.DefaultConversionService;
-import org.springframework.core.convert.support.GenericConversionService;
-import org.springframework.util.Assert;
-
-/**
- * Internal {@link ConversionService} used by {@link RelaxedDataBinder} to
- * support additional relaxed conversion.
- *
- * @author Phillip Webb
- * @author Stephane Nicoll
- * @since 1.1.0
- */
-class RelaxedConversionService implements ConversionService {
-
-    private final ConversionService        conversionService;
-
-    private final GenericConversionService additionalConverters;
-
-    /**
-     * Create a new {@link RelaxedConversionService} instance.
-     *
-     * @param conversionService and option root conversion service
-     */
-    RelaxedConversionService(ConversionService conversionService){
-        this.conversionService = conversionService;
-        this.additionalConverters = new GenericConversionService();
-        DefaultConversionService.addCollectionConverters(this.additionalConverters);
-        this.additionalConverters
-            .addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory());
-        this.additionalConverters.addConverter(new StringToCharArrayConverter());
-    }
-
-    @Override
-    public boolean canConvert(Class<?> sourceType, Class<?> targetType) {
-        return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType))
-               || this.additionalConverters.canConvert(sourceType, targetType);
-    }
-
-    @Override
-    public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) {
-        return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType))
-               || this.additionalConverters.canConvert(sourceType, targetType);
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public <T> T convert(Object source, Class<T> targetType) {
-        Assert.notNull(targetType, "The targetType to convert to cannot be null");
-        return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType));
-    }
-
-    @Override
-    public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) {
-        if (this.conversionService != null) {
-            try {
-                return this.conversionService.convert(source, sourceType, targetType);
-            } catch (ConversionFailedException ex) {
-                // Ignore and try the additional converters
-            }
-        }
-        return this.additionalConverters.convert(source, sourceType, targetType);
-    }
-
-    /**
-     * Clone of Spring's package private StringToEnumConverterFactory, but ignoring
-     * the case of the source.
-     */
-    @SuppressWarnings({ "unchecked", "rawtypes" })
-    private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory<String, Enum> {
-
-        @Override
-        public <T extends Enum> Converter<String, T> getConverter(Class<T> targetType) {
-            Class<?> enumType = targetType;
-            while (enumType != null && !enumType.isEnum()) {
-                enumType = enumType.getSuperclass();
-            }
-            Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum");
-            return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType);
-        }
-
-        private static class StringToEnum<T extends Enum> implements Converter<String, T> {
-
-            private final Class<T> enumType;
-
-            StringToEnum(Class<T> enumType){
-                this.enumType = enumType;
-            }
-
-            @Override
-            public T convert(String source) {
-                if (source.isEmpty()) {
-                    // It's an empty enum identifier: reset the enum value to null.
-                    return null;
-                }
-                source = source.trim();
-                for (T candidate : (Set<T>) EnumSet.allOf(this.enumType)) {
-                    RelaxedNames names = new RelaxedNames(
-                        candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH));
-                    for (String name : names) {
-                        if (name.equals(source)) {
-                            return candidate;
-                        }
-                    }
-                    if (candidate.name().equalsIgnoreCase(source)) {
-                        return candidate;
-                    }
-                }
-                throw new IllegalArgumentException(
-                    "No enum constant " + this.enumType.getCanonicalName() + "." + source);
-            }
-
-        }
-
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java

@@ -1,729 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.beans.PropertyEditor;
-import java.net.InetAddress;
-import java.util.*;
-
-import org.springframework.beans.*;
-import org.springframework.beans.propertyeditors.FileEditor;
-import org.springframework.core.convert.ConversionService;
-import org.springframework.core.convert.TypeDescriptor;
-import org.springframework.core.env.StandardEnvironment;
-import org.springframework.util.LinkedMultiValueMap;
-import org.springframework.util.MultiValueMap;
-import org.springframework.util.StringUtils;
-import org.springframework.validation.AbstractPropertyBindingResult;
-import org.springframework.validation.BeanPropertyBindingResult;
-import org.springframework.validation.DataBinder;
-
-/**
- * Binder implementation that allows caller to bind to maps and also allows
- * property names to match a bit loosely (if underscores or dashes are removed
- * and replaced with camel case for example).
- *
- * @author Dave Syer
- * @author Phillip Webb
- * @author Stephane Nicoll
- * @author Andy Wilkinson
- * @see RelaxedNames
- */
-public class RelaxedDataBinder extends DataBinder {
-
-    private static final Set<Class<?>> EXCLUDED_EDITORS;
-
-    static {
-        Set<Class<?>> excluded = new HashSet<Class<?>>();
-        excluded.add(FileEditor.class);
-        EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
-    }
-
-    private static final Object           BLANK       = new Object();
-
-    private String                        namePrefix;
-
-    private boolean                       ignoreNestedProperties;
-
-    private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<String, String>();
-
-    /**
-     * Create a new {@link RelaxedDataBinder} instance.
-     *
-     * @param target the target into which properties are bound
-     */
-    public RelaxedDataBinder(Object target){
-        super(wrapTarget(target));
-    }
-
-    /**
-     * Create a new {@link RelaxedDataBinder} instance.
-     *
-     * @param target the target into which properties are bound
-     * @param namePrefix An optional prefix to be used when reading properties
-     */
-    public RelaxedDataBinder(Object target, String namePrefix){
-        super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME));
-        this.namePrefix = cleanNamePrefix(namePrefix);
-    }
-
-    private String cleanNamePrefix(String namePrefix) {
-        if (!StringUtils.hasLength(namePrefix)) {
-            return null;
-        }
-        return (namePrefix.endsWith(".") ? namePrefix : namePrefix + ".");
-    }
-
-    /**
-     * Flag to disable binding of nested properties (i.e. those with period
-     * separators in their paths). Can be useful to disable this if the name prefix
-     * is empty and you don't want to ignore unknown fields.
-     *
-     * @param ignoreNestedProperties the flag to set (default false)
-     */
-    public void setIgnoreNestedProperties(boolean ignoreNestedProperties) {
-        this.ignoreNestedProperties = ignoreNestedProperties;
-    }
-
-    /**
-     * Set name aliases.
-     *
-     * @param aliases a map of property name to aliases
-     */
-    public void setNameAliases(Map<String, List<String>> aliases) {
-        this.nameAliases = new LinkedMultiValueMap<String, String>(aliases);
-    }
-
-    /**
-     * Add aliases to the {@link DataBinder}.
-     *
-     * @param name the property name to alias
-     * @param alias aliases for the property names
-     * @return this instance
-     */
-    public RelaxedDataBinder withAlias(String name, String... alias) {
-        for (String value : alias) {
-            this.nameAliases.add(name, value);
-        }
-        return this;
-    }
-
-    @Override
-    protected void doBind(MutablePropertyValues propertyValues) {
-        super.doBind(modifyProperties(propertyValues, getTarget()));
-    }
-
-    /**
-     * Modify the property values so that period separated property paths are valid
-     * for map keys. Also creates new maps for properties of map type that are null
-     * (assuming all maps are potentially nested). The standard bracket {@code[...]}
-     * dereferencing is also accepted.
-     *
-     * @param propertyValues the property values
-     * @param target the target object
-     * @return modified property values
-     */
-    private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) {
-        propertyValues = getPropertyValuesForNamePrefix(propertyValues);
-        if (target instanceof RelaxedDataBinder.MapHolder) {
-            propertyValues = addMapPrefix(propertyValues);
-        }
-        BeanWrapper wrapper = new BeanWrapperImpl(target);
-        wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
-        wrapper.setAutoGrowNestedPaths(true);
-        List<PropertyValue> sortedValues = new ArrayList<PropertyValue>();
-        Set<String> modifiedNames = new HashSet<String>();
-        List<String> sortedNames = getSortedPropertyNames(propertyValues);
-        for (String name : sortedNames) {
-            PropertyValue propertyValue = propertyValues.getPropertyValue(name);
-            PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue);
-            if (modifiedNames.add(modifiedProperty.getName())) {
-                sortedValues.add(modifiedProperty);
-            }
-        }
-        return new MutablePropertyValues(sortedValues);
-    }
-
-    private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
-        List<String> names = new LinkedList<String>();
-        for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
-            names.add(propertyValue.getName());
-        }
-        sortPropertyNames(names);
-        return names;
-    }
-
-    /**
-     * Sort by name so that parent properties get processed first (e.g. 'foo.bar'
-     * before 'foo.bar.spam'). Don't use Collections.sort() because the order might
-     * be significant for other property names (it shouldn't be but who knows what
-     * people might be relying on, e.g. HSQL has a JDBCXADataSource where
-     * "databaseName" is a synonym for "url").
-     *
-     * @param names the names to sort
-     */
-    private void sortPropertyNames(List<String> names) {
-        for (String name : new ArrayList<String>(names)) {
-            int propertyIndex = names.indexOf(name);
-            RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
-            for (String prefix : path.prefixes()) {
-                int prefixIndex = names.indexOf(prefix);
-                if (prefixIndex >= propertyIndex) {
-                    // The child property has a parent in the list in the wrong order
-                    names.remove(name);
-                    names.add(prefixIndex, name);
-                }
-            }
-        }
-    }
-
-    private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) {
-        MutablePropertyValues rtn = new MutablePropertyValues();
-        for (PropertyValue pv : propertyValues.getPropertyValues()) {
-            rtn.add("map." + pv.getName(), pv.getValue());
-        }
-        return rtn;
-    }
-
-    private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) {
-        if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) {
-            return propertyValues;
-        }
-        MutablePropertyValues rtn = new MutablePropertyValues();
-        for (PropertyValue value : propertyValues.getPropertyValues()) {
-            String name = value.getName();
-            for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) {
-                for (String separator : new String[] { ".", "_" }) {
-                    String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix);
-                    if (name.startsWith(candidate)) {
-                        name = name.substring(candidate.length());
-                        if (!(this.ignoreNestedProperties && name.contains("."))) {
-                            PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value);
-                            rtn.addPropertyValue(
-                                new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin));
-                        }
-                    }
-                }
-            }
-        }
-        return rtn;
-    }
-
-    private String stripLastDot(String string) {
-        if (StringUtils.hasLength(string) && string.endsWith(".")) {
-            string = string.substring(0, string.length() - 1);
-        }
-        return string;
-    }
-
-    private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) {
-        String name = propertyValue.getName();
-        String normalizedName = normalizePath(target, name);
-        if (!normalizedName.equals(name)) {
-            return new PropertyValue(normalizedName, propertyValue.getValue());
-        }
-        return propertyValue;
-    }
-
-    /**
-     * Normalize a bean property path to a format understood by a BeanWrapper. This
-     * is used so that
-     * <ul>
-     * <li>Fuzzy matching can be employed for bean property names</li>
-     * <li>Period separators can be used instead of indexing ([...]) for map
-     * keys</li>
-     * </ul>
-     *
-     * @param wrapper a bean wrapper for the object to bind
-     * @param path the bean path to bind
-     * @return a transformed path with correct bean wrapper syntax
-     */
-    protected String normalizePath(BeanWrapper wrapper, String path) {
-        return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0);
-    }
-
-    @Override
-    protected AbstractPropertyBindingResult createBeanPropertyBindingResult() {
-        return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(),
-            getObjectName(),
-            isAutoGrowNestedPaths(),
-            getAutoGrowCollectionLimit(),
-            getConversionService());
-    }
-
-    private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
-        String prefix = path.prefix(index);
-        String key = path.name(index);
-        if (path.isProperty(index)) {
-            key = getActualPropertyName(wrapper, prefix, key);
-            path.rename(index, key);
-        }
-        if (path.name(++index) == null) {
-            return path.toString();
-        }
-        String name = path.prefix(index);
-        TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name);
-        if (descriptor == null || descriptor.isMap()) {
-            if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) {
-                path.collapseKeys(index);
-            }
-            path.mapIndex(index);
-            extendMapIfNecessary(wrapper, path, index);
-        } else if (descriptor.isCollection()) {
-            extendCollectionIfNecessary(wrapper, path, index);
-        } else if (descriptor.getType().equals(Object.class)) {
-            if (isBlanked(wrapper, name, path.name(index))) {
-                path.collapseKeys(index);
-            }
-            path.mapIndex(index);
-            if (path.isLastNode(index)) {
-                wrapper.setPropertyValue(path.toString(), BLANK);
-            } else {
-                String next = path.prefix(index + 1);
-                if (wrapper.getPropertyValue(next) == null) {
-                    wrapper.setPropertyValue(next, new LinkedHashMap<String, Object>());
-                }
-            }
-        }
-        return initializePath(wrapper, path, index);
-    }
-
-    private boolean isMapValueStringType(TypeDescriptor descriptor) {
-        if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) {
-            return false;
-        }
-        if (Properties.class.isAssignableFrom(descriptor.getObjectType())) {
-            // Properties is declared as Map<Object,Object> but we know it's really
-            // Map<String,String>
-            return true;
-        }
-        Class<?> valueType = descriptor.getMapValueTypeDescriptor().getObjectType();
-        return (valueType != null && CharSequence.class.isAssignableFrom(valueType));
-    }
-
-    @SuppressWarnings("rawtypes")
-    private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) {
-        Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null);
-        if (value instanceof Map) {
-            if (((Map) value).get(key) == BLANK) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
-        String name = path.prefix(index);
-        TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor();
-        if (!elementDescriptor.isMap() && !elementDescriptor.isCollection()
-            && !elementDescriptor.getType().equals(Object.class)) {
-            return;
-        }
-        Object extend = new LinkedHashMap<String, Object>();
-        if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
-            extend = new ArrayList<Object>();
-        }
-        wrapper.setPropertyValue(path.prefix(index + 1), extend);
-    }
-
-    private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) {
-        String name = path.prefix(index);
-        TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name);
-        if (parent == null) {
-            return;
-        }
-        TypeDescriptor descriptor = parent.getMapValueTypeDescriptor();
-        if (descriptor == null) {
-            descriptor = TypeDescriptor.valueOf(Object.class);
-        }
-        if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) {
-            return;
-        }
-        String extensionName = path.prefix(index + 1);
-        if (wrapper.isReadableProperty(extensionName)) {
-            Object currentValue = wrapper.getPropertyValue(extensionName);
-            if ((descriptor.isCollection() && currentValue instanceof Collection)
-                || (!descriptor.isCollection() && currentValue instanceof Map)) {
-                return;
-            }
-        }
-        Object extend = new LinkedHashMap<String, Object>();
-        if (descriptor.isCollection()) {
-            extend = new ArrayList<Object>();
-        }
-        if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
-            extend = BLANK;
-        }
-        wrapper.setPropertyValue(extensionName, extend);
-    }
-
-    private String getActualPropertyName(BeanWrapper target, String prefix, String name) {
-        String propertyName = resolvePropertyName(target, prefix, name);
-        if (propertyName == null) {
-            propertyName = resolveNestedPropertyName(target, prefix, name);
-        }
-        return (propertyName != null ? propertyName : name);
-    }
-
-    private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) {
-        StringBuilder candidate = new StringBuilder();
-        for (String field : name.split("[_\\-\\.]")) {
-            candidate.append(candidate.length() > 0 ? "." : "");
-            candidate.append(field);
-            String nested = resolvePropertyName(target, prefix, candidate.toString());
-            if (nested != null) {
-                Class<?> type = target.getPropertyType(nested);
-                if ((type != null) && Map.class.isAssignableFrom(type)) {
-                    // Special case for map property (gh-3836).
-                    return nested + "[" + name.substring(candidate.length() + 1) + "]";
-                }
-                String propertyName = resolvePropertyName(target,
-                    joinString(prefix, nested),
-                    name.substring(candidate.length() + 1));
-                if (propertyName != null) {
-                    return joinString(nested, propertyName);
-                }
-            }
-        }
-        return null;
-    }
-
-    private String resolvePropertyName(BeanWrapper target, String prefix, String name) {
-        Iterable<String> names = getNameAndAliases(name);
-        for (String nameOrAlias : names) {
-            for (String candidate : new RelaxedNames(nameOrAlias)) {
-                try {
-                    if (target.getPropertyType(joinString(prefix, candidate)) != null) {
-                        return candidate;
-                    }
-                } catch (InvalidPropertyException ex) {
-                    // swallow and continue
-                }
-            }
-        }
-        return null;
-    }
-
-    private String joinString(String prefix, String name) {
-        return (StringUtils.hasLength(prefix) ? prefix + "." + name : name);
-    }
-
-    private Iterable<String> getNameAndAliases(String name) {
-        List<String> aliases = this.nameAliases.get(name);
-        if (aliases == null) {
-            return Collections.singleton(name);
-        }
-        List<String> nameAndAliases = new ArrayList<String>(aliases.size() + 1);
-        nameAndAliases.add(name);
-        nameAndAliases.addAll(aliases);
-        return nameAndAliases;
-    }
-
-    private static Object wrapTarget(Object target) {
-        if (target instanceof Map) {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> map = (Map<String, Object>) target;
-            target = new RelaxedDataBinder.MapHolder(map);
-        }
-        return target;
-    }
-
-    @Override
-    public void registerCustomEditor(Class<?> requiredType, PropertyEditor propertyEditor) {
-        if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) {
-            super.registerCustomEditor(requiredType, propertyEditor);
-        }
-    }
-
-    @Override
-    public void registerCustomEditor(Class<?> requiredType, String field, PropertyEditor propertyEditor) {
-        if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) {
-            super.registerCustomEditor(requiredType, field, propertyEditor);
-        }
-    }
-
-    /**
-     * Holder to allow Map targets to be bound.
-     */
-    static class MapHolder {
-
-        private Map<String, Object> map;
-
-        MapHolder(Map<String, Object> map){
-            this.map = map;
-        }
-
-        public void setMap(Map<String, Object> map) {
-            this.map = map;
-        }
-
-        public Map<String, Object> getMap() {
-            return this.map;
-        }
-
-    }
-
-    /**
-     * A path though properties of a bean.
-     */
-    private static class BeanPath {
-
-        private List<PathNode> nodes;
-
-        BeanPath(String path){
-            this.nodes = splitPath(path);
-        }
-
-        public List<String> prefixes() {
-            List<String> prefixes = new ArrayList<String>();
-            for (int index = 1; index < this.nodes.size(); index++) {
-                prefixes.add(prefix(index));
-            }
-            return prefixes;
-        }
-
-        public boolean isLastNode(int index) {
-            return index >= this.nodes.size() - 1;
-        }
-
-        private List<PathNode> splitPath(String path) {
-            List<PathNode> nodes = new ArrayList<PathNode>();
-            String current = extractIndexedPaths(path, nodes);
-            for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
-                if (StringUtils.hasText(name)) {
-                    nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name));
-                }
-            }
-            return nodes;
-        }
-
-        private String extractIndexedPaths(String path, List<PathNode> nodes) {
-            int startRef = path.indexOf("[");
-            String current = path;
-            while (startRef >= 0) {
-                if (startRef > 0) {
-                    nodes.addAll(splitPath(current.substring(0, startRef)));
-                }
-                int endRef = current.indexOf("]", startRef);
-                if (endRef > 0) {
-                    String sub = current.substring(startRef + 1, endRef);
-                    if (sub.matches("[0-9]+")) {
-                        nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub));
-                    } else {
-                        nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub));
-                    }
-                }
-                current = current.substring(endRef + 1);
-                startRef = current.indexOf("[");
-            }
-            return current;
-        }
-
-        public void collapseKeys(int index) {
-            List<PathNode> revised = new ArrayList<PathNode>();
-            for (int i = 0; i < index; i++) {
-                revised.add(this.nodes.get(i));
-            }
-            StringBuilder builder = new StringBuilder();
-            for (int i = index; i < this.nodes.size(); i++) {
-                if (i > index) {
-                    builder.append(".");
-                }
-                builder.append(this.nodes.get(i).name);
-            }
-            revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString()));
-            this.nodes = revised;
-        }
-
-        public void mapIndex(int index) {
-            RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index);
-            if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) {
-                node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex();
-            }
-            this.nodes.set(index, node);
-        }
-
-        public String prefix(int index) {
-            return range(0, index);
-        }
-
-        public void rename(int index, String name) {
-            this.nodes.get(index).name = name;
-        }
-
-        public String name(int index) {
-            if (index < this.nodes.size()) {
-                return this.nodes.get(index).name;
-            }
-            return null;
-        }
-
-        private String range(int start, int end) {
-            StringBuilder builder = new StringBuilder();
-            for (int i = start; i < end; i++) {
-                RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i);
-                builder.append(node);
-            }
-            if (builder.toString().startsWith(("."))) {
-                builder.replace(0, 1, "");
-            }
-            return builder.toString();
-        }
-
-        public boolean isArrayIndex(int index) {
-            return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode;
-        }
-
-        public boolean isProperty(int index) {
-            return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode;
-        }
-
-        @Override
-        public String toString() {
-            return prefix(this.nodes.size());
-        }
-
-        private static class PathNode {
-
-            protected String name;
-
-            PathNode(String name){
-                this.name = name;
-            }
-
-        }
-
-        private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode {
-
-            ArrayIndexNode(String name){
-                super(name);
-            }
-
-            @Override
-            public String toString() {
-                return "[" + this.name + "]";
-            }
-
-        }
-
-        private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode {
-
-            MapIndexNode(String name){
-                super(name);
-            }
-
-            @Override
-            public String toString() {
-                return "[" + this.name + "]";
-            }
-
-        }
-
-        private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode {
-
-            PropertyNode(String name){
-                super(name);
-            }
-
-            public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() {
-                return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name);
-            }
-
-            @Override
-            public String toString() {
-                return "." + this.name;
-            }
-
-        }
-
-    }
-
-    /**
-     * Extended version of {@link BeanPropertyBindingResult} to support relaxed
-     * binding.
-     */
-    private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult {
-
-        private RelaxedConversionService conversionService;
-
-        RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths,
-                                         int autoGrowCollectionLimit, ConversionService conversionService){
-            super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit);
-            this.conversionService = new RelaxedConversionService(conversionService);
-        }
-
-        @Override
-        protected BeanWrapper createBeanWrapper() {
-            BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget());
-            beanWrapper.setConversionService(this.conversionService);
-            beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor());
-            return beanWrapper;
-        }
-
-    }
-
-    /**
-     * Extended version of {@link BeanWrapperImpl} to support relaxed binding.
-     */
-    private static class RelaxedBeanWrapper extends BeanWrapperImpl {
-
-        private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
-
-        static {
-            Set<String> names = new HashSet<String>();
-            names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
-            names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
-            BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);
-        }
-
-        RelaxedBeanWrapper(Object target){
-            super(target);
-        }
-
-        @Override
-        public void setPropertyValue(PropertyValue pv) throws BeansException {
-            try {
-                super.setPropertyValue(pv);
-            } catch (NotWritablePropertyException ex) {
-                PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv);
-                if (isBenign(origin)) {
-                    logger.debug("Ignoring benign property binding failure", ex);
-                    return;
-                }
-                if (origin == null) {
-                    throw ex;
-                }
-                throw new RelaxedBindingNotWritablePropertyException(ex, origin);
-            }
-        }
-
-        private boolean isBenign(PropertyOrigin origin) {
-            String name = (origin != null ? origin.getSource().getName() : null);
-            return BENIGN_PROPERTY_SOURCE_NAMES.contains(name);
-        }
-
-    }
-
-    public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException {
-
-        private final String         message;
-
-        private final PropertyOrigin propertyOrigin;
-
-        RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){
-            super(ex.getBeanClass(), ex.getPropertyName());
-            this.propertyOrigin = propertyOrigin;
-            this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '"
-                           + propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '"
-                           + ex.getBeanClass().getName() + "'";
-        }
-
-        @Override
-        public String getMessage() {
-            return this.message;
-        }
-
-        public PropertyOrigin getPropertyOrigin() {
-            return this.propertyOrigin;
-        }
-
-    }
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java

@@ -1,241 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Locale;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.springframework.util.StringUtils;
-
-/**
- * Generates relaxed name variations from a given source.
- *
- * @author Phillip Webb
- * @author Dave Syer
- * @see RelaxedDataBinder
- */
-public final class RelaxedNames implements Iterable<String> {
-
-    private static final Pattern CAMEL_CASE_PATTERN              = Pattern.compile("([^A-Z-])([A-Z])");
-
-    private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]");
-
-    private final String         name;
-
-    private final Set<String>    values                          = new LinkedHashSet<String>();
-
-    /**
-     * Create a new {@link RelaxedNames} instance.
-     *
-     * @param name the source name. For the maximum number of variations specify the
-     *     name using dashed notation (e.g. {@literal my-property-name}
-     */
-    public RelaxedNames(String name){
-        this.name = (name != null ? name : "");
-        initialize(RelaxedNames.this.name, this.values);
-    }
-
-    @Override
-    public Iterator<String> iterator() {
-        return this.values.iterator();
-    }
-
-    private void initialize(String name, Set<String> values) {
-        if (values.contains(name)) {
-            return;
-        }
-        for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) {
-            for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) {
-                String result = name;
-                result = manipulation.apply(result);
-                result = variation.apply(result);
-                values.add(result);
-                initialize(result, values);
-            }
-        }
-    }
-
-    /**
-     * Name variations.
-     */
-    enum Variation {
-
-                    NONE {
-
-                        @Override
-                        public String apply(String value) {
-                            return value;
-                        }
-
-                    },
-
-                    LOWERCASE {
-
-                        @Override
-                        public String apply(String value) {
-                            return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH));
-                        }
-
-                    },
-
-                    UPPERCASE {
-
-                        @Override
-                        public String apply(String value) {
-                            return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH));
-                        }
-
-                    };
-
-        public abstract String apply(String value);
-
-    }
-
-    /**
-     * Name manipulations.
-     */
-    enum Manipulation {
-
-                       NONE {
-
-                           @Override
-                           public String apply(String value) {
-                               return value;
-                           }
-
-                       },
-
-                       HYPHEN_TO_UNDERSCORE {
-
-                           @Override
-                           public String apply(String value) {
-                               return (value.indexOf('-') != -1 ? value.replace('-', '_') : value);
-                           }
-
-                       },
-
-                       UNDERSCORE_TO_PERIOD {
-
-                           @Override
-                           public String apply(String value) {
-                               return (value.indexOf('_') != -1 ? value.replace('_', '.') : value);
-                           }
-
-                       },
-
-                       PERIOD_TO_UNDERSCORE {
-
-                           @Override
-                           public String apply(String value) {
-                               return (value.indexOf('.') != -1 ? value.replace('.', '_') : value);
-                           }
-
-                       },
-
-                       CAMELCASE_TO_UNDERSCORE {
-
-                           @Override
-                           public String apply(String value) {
-                               if (value.isEmpty()) {
-                                   return value;
-                               }
-                               Matcher matcher = CAMEL_CASE_PATTERN.matcher(value);
-                               if (!matcher.find()) {
-                                   return value;
-                               }
-                               matcher = matcher.reset();
-                               StringBuffer result = new StringBuffer();
-                               while (matcher.find()) {
-                                   matcher.appendReplacement(result,
-                                       matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2)));
-                               }
-                               matcher.appendTail(result);
-                               return result.toString();
-                           }
-
-                       },
-
-                       CAMELCASE_TO_HYPHEN {
-
-                           @Override
-                           public String apply(String value) {
-                               if (value.isEmpty()) {
-                                   return value;
-                               }
-                               Matcher matcher = CAMEL_CASE_PATTERN.matcher(value);
-                               if (!matcher.find()) {
-                                   return value;
-                               }
-                               matcher = matcher.reset();
-                               StringBuffer result = new StringBuffer();
-                               while (matcher.find()) {
-                                   matcher.appendReplacement(result,
-                                       matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2)));
-                               }
-                               matcher.appendTail(result);
-                               return result.toString();
-                           }
-
-                       },
-
-                       SEPARATED_TO_CAMELCASE {
-
-                           @Override
-                           public String apply(String value) {
-                               return separatedToCamelCase(value, false);
-                           }
-
-                       },
-
-                       CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE {
-
-                           @Override
-                           public String apply(String value) {
-                               return separatedToCamelCase(value, true);
-                           }
-
-                       };
-
-        private static final char[] SUFFIXES = new char[] { '_', '-', '.' };
-
-        public abstract String apply(String value);
-
-        private static String separatedToCamelCase(String value, boolean caseInsensitive) {
-            if (value.isEmpty()) {
-                return value;
-            }
-            StringBuilder builder = new StringBuilder();
-            for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) {
-                field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field);
-                builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field);
-            }
-            char lastChar = value.charAt(value.length() - 1);
-            for (char suffix : SUFFIXES) {
-                if (lastChar == suffix) {
-                    builder.append(suffix);
-                    break;
-                }
-            }
-            return builder.toString();
-        }
-
-    }
-
-    /**
-     * Return a {@link RelaxedNames} for the given source camelCase source name.
-     *
-     * @param name the source name in camelCase
-     * @return the relaxed names
-     */
-    public static RelaxedNames forCamelCase(String name) {
-        StringBuilder result = new StringBuilder();
-        for (char c : name.toCharArray()) {
-            result.append(Character.isUpperCase(c) && result.length() > 0
-                          && result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c);
-        }
-        return new RelaxedNames(result.toString());
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java

@@ -1,17 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.bind;
-
-import org.springframework.core.convert.converter.Converter;
-
-/**
- * Converts a String to a Char Array.
- *
- * @author Phillip Webb
- */
-class StringToCharArrayConverter implements Converter<String, char[]> {
-
-    @Override
-    public char[] convert(String source) {
-        return source.toCharArray();
-    }
-
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java

@@ -1,203 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.common;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-import org.springframework.util.Assert;
-import org.springframework.util.ResourceUtils;
-
-/**
- * Convenience base class for {@link Resource} implementations, pre-implementing
- * typical behavior.
- * <p>
- * The "exists" method will check whether a File or InputStream can be opened;
- * "isOpen" will always return false; "getURL" and "getFile" throw an exception;
- * and "toString" will return the description.
- *
- * @author Juergen Hoeller
- * @since 28.12.2003
- */
-public abstract class AbstractResource implements Resource {
-
-    /**
-     * This implementation checks whether a File can be opened, falling back to
-     * whether an InputStream can be opened. This will cover both directories and
-     * content resources.
-     */
-    @Override
-    public boolean exists() {
-        // Try file existence: can we find the file in the file system?
-        try {
-            return getFile().exists();
-        } catch (IOException ex) {
-            // Fall back to stream existence: can we open the stream?
-            try {
-                InputStream is = getInputStream();
-                is.close();
-                return true;
-            } catch (Throwable isEx) {
-                return false;
-            }
-        }
-    }
-
-    /**
-     * This implementation always returns {@code true}.
-     */
-    @Override
-    public boolean isReadable() {
-        return true;
-    }
-
-    /**
-     * This implementation always returns {@code false}.
-     */
-    @Override
-    public boolean isOpen() {
-        return false;
-    }
-
-    /**
-     * This implementation throws a FileNotFoundException, assuming that the
-     * resource cannot be resolved to a URL.
-     */
-    @Override
-    public URL getURL() throws IOException {
-        throw new FileNotFoundException(getDescription() + " cannot be resolved to URL");
-    }
-
-    /**
-     * This implementation builds a URI based on the URL returned by
-     * {@link #getURL()}.
-     */
-    @Override
-    public URI getURI() throws IOException {
-        URL url = getURL();
-        try {
-            return ResourceUtils.toURI(url);
-        } catch (URISyntaxException ex) {
-            throw new RuntimeException("Invalid URI [" + url + "]", ex);
-        }
-    }
-
-    /**
-     * This implementation throws a FileNotFoundException, assuming that the
-     * resource cannot be resolved to an absolute file path.
-     */
-    @Override
-    public File getFile() throws IOException {
-        throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path");
-    }
-
-    /**
-     * This implementation reads the entire InputStream to calculate the content
-     * length. Subclasses will almost always be able to provide a more optimal
-     * version of this, e.g. checking a File length.
-     *
-     * @see #getInputStream()
-     */
-    @Override
-    public long contentLength() throws IOException {
-        InputStream is = getInputStream();
-        Assert.state(is != null, "Resource InputStream must not be null");
-        try {
-            long size = 0;
-            byte[] buf = new byte[255];
-            int read;
-            while ((read = is.read(buf)) != -1) {
-                size += read;
-            }
-            return size;
-        } finally {
-            try {
-                is.close();
-            } catch (IOException ex) {
-            }
-        }
-    }
-
-    /**
-     * This implementation checks the timestamp of the underlying File, if
-     * available.
-     *
-     * @see #getFileForLastModifiedCheck()
-     */
-    @Override
-    public long lastModified() throws IOException {
-        long lastModified = getFileForLastModifiedCheck().lastModified();
-        if (lastModified == 0L) {
-            throw new FileNotFoundException(
-                getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp");
-        }
-        return lastModified;
-    }
-
-    /**
-     * Determine the File to use for timestamp checking.
-     * <p>
-     * The default implementation delegates to {@link #getFile()}.
-     *
-     * @return the File to use for timestamp checking (never {@code null})
-     * @throws FileNotFoundException if the resource cannot be resolved as an
-     *     absolute file path, i.e. is not available in a file system
-     * @throws IOException in case of general resolution/reading failures
-     */
-    protected File getFileForLastModifiedCheck() throws IOException {
-        return getFile();
-    }
-
-    /**
-     * This implementation throws a FileNotFoundException, assuming that relative
-     * resources cannot be created for this resource.
-     */
-    @Override
-    public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException {
-        throw new FileNotFoundException("Cannot create a relative resource for " + getDescription());
-    }
-
-    /**
-     * This implementation always returns {@code null}, assuming that this resource
-     * type does not have a filename.
-     */
-    @Override
-    public String getFilename() {
-        return null;
-    }
-
-    /**
-     * This implementation returns the description of this resource.
-     *
-     * @see #getDescription()
-     */
-    @Override
-    public String toString() {
-        return getDescription();
-    }
-
-    /**
-     * This implementation compares description strings.
-     *
-     * @see #getDescription()
-     */
-    @Override
-    public boolean equals(Object obj) {
-        return (obj == this
-                || (obj instanceof org.springframework.core.io.Resource
-                    && ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription())));
-    }
-
-    /**
-     * This implementation returns the description's hash code.
-     *
-     * @see #getDescription()
-     */
-    @Override
-    public int hashCode() {
-        return getDescription().hashCode();
-    }
-}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java

@@ -1,117 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.config.common;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.core.io.Resource;
-import org.springframework.util.Assert;
-
-/**
- * {@link Resource} implementation for a given byte array.
- * <p>
- * Creates a {@link ByteArrayInputStream} for the given byte array.
- * <p>
- * Useful for loading content from any given byte array, without having to
- * resort to a single-use {@link InputStreamResource}. Particularly useful for
- * creating mail attachments from local content, where JavaMail needs to be able
- * to read the stream multiple times.
- *
- * @author Juergen Hoeller
- * @author Sam Brannen
- * @see ByteArrayInputStream
- * @see InputStreamResource
- * @since 1.2.3
- */
-public class ByteArrayResource extends AbstractResource {
-
-    private final byte[] byteArray;
-
-    private final String description;
-
-    /**
-     * Create a new {@code ByteArrayResource}.
-     *
-     * @param byteArray the byte array to wrap
-     */
-    public ByteArrayResource(byte[] byteArray){
-        this(byteArray, "resource loaded from byte array");
-    }
-
-    /**
-     * Create a new {@code ByteArrayResource} with a description.
-     *
-     * @param byteArray the byte array to wrap
-     * @param description where the byte array comes from
-     */
-    public ByteArrayResource(byte[] byteArray, String description){
-        Assert.notNull(byteArray, "Byte array must not be null");
-        this.byteArray = byteArray;
-        this.description = (description != null ? description : "");
-    }
-
-    /**
-     * Return the underlying byte array.
-     */
-    public final byte[] getByteArray() {
-        return this.byteArray;
-    }
-
-    /**
-     * This implementation always returns {@code true}.
-     */
-    @Override
-    public boolean exists() {
-        return true;
-    }
-
-    /**
-     * This implementation returns the length of the underlying byte array.
-     */
-    @Override
-    public long contentLength() {
-        return this.byteArray.length;
-    }
-
-    /**
-     * This implementation returns a ByteArrayInputStream for the underlying byte
-     * array.
-     *
-     * @see ByteArrayInputStream
-     */
-    @Override
-    public InputStream getInputStream() throws IOException {
-        return new ByteArrayInputStream(this.byteArray);
-    }
-
-    /**
-     * This implementation returns a description that includes the passed-in
-     * {@code description}, if any.
-     */
-    @Override
-    public String getDescription() {
-        return "Byte array resource [" + this.description + "]";
-    }
-
-    /**
-     * This implementation compares the underlying byte array.
-     *
-     * @see Arrays#equals(byte[], byte[])
-     */
-    @Override
-    public boolean equals(Object obj) {
-        return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource
-                                && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray)));
-    }
-
-    /**
-     * This implementation returns the hash code based on the underlying byte array.
-     */
-    @Override
-    public int hashCode() {
-        return (byte[].class.hashCode() * 29 * this.byteArray.length);
-    }
-
-}