fix(canal):fix endless local binlog mode with exception.

README.md

@@ -90,13 +90,10 @@ canal 作为 MySQL binlog 增量获取和解析工具，可将变更记录投递
 
 - 参考文档: [Canal Kafka/RocketMQ QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart)
 
-## 相关开源&产品
+## 相关开源
 
 - [canal 消费端开源项目: Otter](http://github.com/alibaba/otter)
 - [阿里巴巴去 Oracle 数据迁移同步工具: yugong](http://github.com/alibaba/yugong)
-- [阿里巴巴离线同步开源项目 DataX](https://github.com/alibaba/datax)
-- [阿里巴巴数据库连接池开源项目 Druid](https://github.com/alibaba/druid)
-- [阿里巴巴实时数据同步工具 DTS](https://www.aliyun.com/product/dts)
 
 ## 问题反馈
 - 报告 issue: [github issues](https://github.com/alibaba/canal/issues)

admin/admin-ui/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <artifactId>canal.admin</artifactId>
     <groupId>com.alibaba.otter</groupId>
-    <version>1.1.5</version>
+    <version>1.1.5-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

admin/admin-web/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.admin</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>canal.admin-web</artifactId>

admin/admin-web/src/main/bin/restart.sh

@@ -1,13 +1,5 @@
 #!/bin/bash
 
-case $(uname) in
+sh stop.sh
-Linux)
-  bin_abs_path=$(readlink -f $(dirname $0))
-  ;;
-*)
-  bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
-  ;;
-esac
 
-sh "$bin_abs_path"/stop.sh
+sh startup.sh
-sh "$bin_abs_path"/startup.sh

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/common/Threads.java

@@ -12,6 +12,6 @@ public class Threads {
                                                             pool,
                                                             0L,
                                                             TimeUnit.MILLISECONDS,
-                                                            new ArrayBlockingQueue<>(pool * 20),
+                                                            new ArrayBlockingQueue<Runnable>(pool * 20),
                                                             DaemonThreadFactory.daemonThreadFactory);
 }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/EbeanConfig.java

@@ -1,10 +1,5 @@
 package com.alibaba.otter.canal.admin.config;
 
-import io.ebean.EbeanServer;
-import io.ebean.EbeanServerFactory;
-import io.ebean.config.ServerConfig;
-import io.ebean.config.UnderscoreNamingConvention;
-
 import java.util.ArrayList;
 import java.util.List;
 
@@ -13,6 +8,11 @@ import javax.sql.DataSource;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import io.ebean.EbeanServer;
+import io.ebean.EbeanServerFactory;
+import io.ebean.config.ServerConfig;
+import io.ebean.config.UnderscoreNamingConvention;
+
 /**
  * Ebean 配置
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/config/WebConfig.java

@@ -84,9 +84,7 @@ public class WebConfig implements WebMvcConfigurer {
             }
         })
             .addPathPatterns("/api/**")
-            .excludePathPatterns("/api/**/config/server_polling")
+            .excludePathPatterns("/api/**/config/**")
-            .excludePathPatterns("/api/**/config/instances_polling")
-            .excludePathPatterns("/api/**/config/instance_polling/**")
             .excludePathPatterns("/api/**/user/login")
             .excludePathPatterns("/api/**/user/logout")
             .excludePathPatterns("/api/**/user/info");

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnector.java

@@ -108,7 +108,9 @@ public class SimpleAdminConnector implements AdminConnector {
             }
 
             connected = true;
-        } catch (IOException | NoSuchAlgorithmException e) {
+        } catch (IOException e) {
+            throw new ServiceException(e);
+        } catch (NoSuchAlgorithmException e) {
             throw new ServiceException(e);
         }
     }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java

@@ -2,12 +2,11 @@ package com.alibaba.otter.canal.admin.connector;
 
 import java.util.function.Function;
 
+import com.alibaba.otter.canal.admin.config.SpringContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.core.env.Environment;
 
-import com.alibaba.otter.canal.admin.config.SpringContext;
-
 public class SimpleAdminConnectors {
 
     private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnectors.class);
@@ -26,7 +25,7 @@ public class SimpleAdminConnectors {
             connector.connect();
             return function.apply(connector);
         } catch (Exception e) {
-            logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user, passwd);
+            logger.error("connect to ip:{},port:{},user:{},password:{}, failed", ip, port, user,passwd);
             logger.error(e.getMessage());
         } finally {
             connector.disconnect();

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalClusterController.java

@@ -4,14 +4,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.*;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.PutMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -29,7 +22,7 @@ public class CanalClusterController {
     CanalClusterService canalClusterServic;
 
     @Autowired
-    NodeServerService   nodeServerService;
+    NodeServerService  nodeServerService;
 
     @GetMapping(value = "/clusters")
     public BaseModel<List<CanalCluster>> clusters(CanalCluster canalCluster, @PathVariable String env) {

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalConfigController.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.admin.controller;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -8,7 +9,6 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalConfig;
 import com.alibaba.otter.canal.admin.service.CanalConfigService;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.admin.controller;
 import java.util.List;
 import java.util.Map;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -14,7 +15,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.model.BaseModel;
 import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 import com.alibaba.otter.canal.admin.model.Pager;
@@ -143,7 +143,8 @@ public class CanalInstanceController {
      * @return 是否成功
      */
     @PutMapping(value = "/instance/status/{id}")
-    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option, @PathVariable String env) {
+    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option,
+                                            @PathVariable String env) {
         return BaseModel.getInstance(canalInstanceConfigService.instanceOperation(id, option));
     }
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/PollingConfigController.java

@@ -46,15 +46,14 @@ public class PollingConfigController {
     public BaseModel<CanalConfig> canalConfigPoll(@RequestHeader String user, @RequestHeader String passwd,
                                                   @RequestParam String ip, @RequestParam Integer port,
                                                   @RequestParam String md5, @RequestParam boolean register,
-                                                  @RequestParam String cluster, @RequestParam String name,
+                                                  @RequestParam String cluster, @PathVariable String env) {
-                                                  @PathVariable String env) {
         if (!auth(user, passwd)) {
             throw new RuntimeException("auth :" + user + " is failed");
         }
 
         if (StringUtils.isEmpty(md5) && register) {
             // do something
-            pollingConfigService.autoRegister(ip, port, cluster, StringUtils.trimToNull(name));
+            pollingConfigService.autoRegister(ip, port, cluster);
         }
 
         CanalConfig canalConfig = pollingConfigService.getChangedConfig(ip, port, md5);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/controller/UserController.java

@@ -1,28 +1,20 @@
 package com.alibaba.otter.canal.admin.controller;
 
+import com.alibaba.otter.canal.admin.service.UserService;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+import com.alibaba.otter.canal.admin.model.BaseModel;
+import com.alibaba.otter.canal.admin.model.User;
+
+import javax.servlet.http.HttpServletRequest;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
-import javax.servlet.http.HttpServletRequest;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.PutMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.alibaba.otter.canal.admin.model.BaseModel;
-import com.alibaba.otter.canal.admin.model.User;
-import com.alibaba.otter.canal.admin.service.UserService;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-
 /**
  * 用户管理控制层
  *
@@ -34,9 +26,9 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
 public class UserController {
 
     public static final LoadingCache<String, User> loginUsers = Caffeine.newBuilder()
-                                                                  .maximumSize(10_000)
+        .maximumSize(10_000)
-                                                                  .expireAfterAccess(30, TimeUnit.MINUTES)
+        .expireAfterAccess(30, TimeUnit.MINUTES)
-                                                                  .build(key -> null); // 用户登录信息缓存
+        .build(key -> null);                                                         // 用户登录信息缓存
 
     @Autowired
     UserService                                    userService;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/handler/CustomExceptionHandler.java

@@ -1,5 +1,7 @@
 package com.alibaba.otter.canal.admin.handler;
 
+import com.alibaba.otter.canal.admin.common.exception.ServiceException;
+import com.alibaba.otter.canal.admin.model.BaseModel;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
@@ -8,9 +10,6 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
-import com.alibaba.otter.canal.admin.common.exception.ServiceException;
-import com.alibaba.otter.canal.admin.model.BaseModel;
-
 /**
  * MVC异常拦截器
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalCluster.java

@@ -3,11 +3,10 @@ package com.alibaba.otter.canal.admin.model;
 import io.ebean.Finder;
 import io.ebean.annotation.WhenModified;
 
-import java.util.Date;
-
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
+import java.util.Date;
 
 /**
  * Canal集群信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/CanalInstanceConfig.java

@@ -1,17 +1,11 @@
 package com.alibaba.otter.canal.admin.model;
 
-import io.ebean.Finder;
-import io.ebean.annotation.WhenModified;
-
 import java.util.Date;
 
-import javax.persistence.Column;
+import javax.persistence.*;
-import javax.persistence.Entity;
+
-import javax.persistence.FetchType;
+import io.ebean.Finder;
-import javax.persistence.Id;
+import io.ebean.annotation.WhenModified;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Transient;
 
 /**
  * Canal实例配置信息实体类
@@ -50,14 +44,14 @@ public class CanalInstanceConfig extends Model {
     private String       name;
     private String       content;
     private String       contentMd5;
-    private String       status;             // 1: 正常 0: 停止
+    private String       status;         // 1: 正常 0: 停止
     @WhenModified
     private Date         modifiedTime;
 
     @Transient
     private String       clusterServerId;
     @Transient
-    private String       runningStatus = "0"; // 1: 运行中 0: 停止
+    private String       runningStatus = "0";  // 1: 运行中 0: 停止
 
     public void init() {
         status = "1";

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Model.java

@@ -1,13 +1,13 @@
 package com.alibaba.otter.canal.admin.model;
 
-import io.ebean.Ebean;
-import io.ebean.EbeanServer;
-
 import javax.persistence.MappedSuperclass;
 import javax.persistence.OptimisticLockException;
 
 import org.apache.commons.beanutils.PropertyUtils;
 
+import io.ebean.Ebean;
+import io.ebean.EbeanServer;
+
 /**
  * EBean Model扩展类
  *

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/NodeServer.java

@@ -5,13 +5,7 @@ import io.ebean.annotation.WhenModified;
 
 import java.util.Date;
 
-import javax.persistence.Column;
+import javax.persistence.*;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
 
 /**
  * 节点信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Pager.java

@@ -9,7 +9,7 @@ public class Pager<T> implements Serializable {
     private static final long serialVersionUID = -986577815091763517L;
 
     private Long              count            = 0L;
-    private List<T>           items            = new ArrayList<>();
+    private List<T>           items            = new ArrayList<T>();
     private Integer           page             = 1;
     private Integer           size             = 20;
     private Long              offset           = 0L;

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/User.java

@@ -3,12 +3,11 @@ package com.alibaba.otter.canal.admin.model;
 import io.ebean.Finder;
 import io.ebean.annotation.WhenCreated;
 
-import java.util.Date;
-
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
 import javax.persistence.Transient;
+import java.util.Date;
 
 /**
  * 用户信息实体类

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/NodeServerService.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.admin.service;
 
-import java.util.List;
-
 import com.alibaba.otter.canal.admin.model.NodeServer;
 import com.alibaba.otter.canal.admin.model.Pager;
 
+import java.util.List;
+
 public interface NodeServerService {
 
     void save(NodeServer nodeServer);

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/PollingConfigService.java

@@ -5,7 +5,7 @@ import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 
 public interface PollingConfigService {
 
-    public boolean autoRegister(String ip, Integer adminPort, String cluster, String name);
+    public boolean autoRegister(String ip, Integer adminPort, String cluster);
 
     CanalConfig getChangedConfig(String ip, Integer port, String md5);
 

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java

@@ -109,12 +109,12 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
             }));
         }
 
-        for (Future<Void> f : futures) {
+        for (Future<Void> f :futures){
             try {
                 f.get(3, TimeUnit.SECONDS);
             } catch (InterruptedException | ExecutionException e) {
                 // ignore
-            } catch (TimeoutException e) {
+            } catch (TimeoutException e){
                 break;
             }
         }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java

@@ -154,12 +154,12 @@ public class NodeServerServiceImpl implements NodeServerService {
                 return !status;
             }));
         }
-        for (Future<Boolean> f : futures) {
+        for (Future<Boolean> f :futures){
             try {
                 f.get(3, TimeUnit.SECONDS);
             } catch (InterruptedException | ExecutionException e) {
                 // ignore
-            } catch (TimeoutException e) {
+            } catch (TimeoutException e){
                 break;
             }
         }

admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/service/impl/PollingConfigServiceImpl.java

@@ -2,7 +2,6 @@ package com.alibaba.otter.canal.admin.service.impl;
 
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang.StringUtils;
@@ -29,11 +28,11 @@ public class PollingConfigServiceImpl implements PollingConfigService {
     @Autowired
     CanalClusterService canalClusterService;
 
-    public boolean autoRegister(String ip, Integer adminPort, String cluster, String name) {
+    public boolean autoRegister(String ip, Integer adminPort, String cluster) {
         NodeServer server = NodeServer.find.query().where().eq("ip", ip).eq("adminPort", adminPort).findOne();
         if (server == null) {
             server = new NodeServer();
-            server.setName(Optional.ofNullable(name).orElse(ip));
+            server.setName(ip);
             server.setIp(ip);
             server.setAdminPort(adminPort);
             server.setTcpPort(adminPort + 1);

admin/admin-web/src/main/resources/canal-template.properties

@@ -8,24 +8,20 @@ canal.register.ip =
 canal.port = 11111
 canal.metrics.pull.port = 11112
 # canal instance user/passwd
-# canal.user = canal
+canal.user = canal
-# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
+canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
 
 # canal admin config
-#canal.admin.manager = 127.0.0.1:8089
+canal.admin.manager = 127.0.0.1:8089
 canal.admin.port = 11110
 canal.admin.user = admin
 canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
-# admin auto register
-#canal.admin.register.auto = true
-#canal.admin.register.cluster =
-#canal.admin.register.name =
 
 canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, rocketMQ, rabbitMQ
+# tcp, kafka, RocketMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -64,9 +60,6 @@ canal.instance.filter.query.ddl = false
 canal.instance.filter.table.error = false
 canal.instance.filter.rows = false
 canal.instance.filter.transaction.entry = false
-canal.instance.filter.dml.insert = false
-canal.instance.filter.dml.update = false
-canal.instance.filter.dml.delete = false
 
 # binlog format/image check
 canal.instance.binlog.format = ROW,STATEMENT,MIXED 
@@ -93,6 +86,10 @@ canal.instance.tsdb.snapshot.interval = 24
 # purge snapshot expire , default 360 hour(15 days)
 canal.instance.tsdb.snapshot.expire = 360
 
+# aliyun ak/sk , support rds/mq
+canal.aliyun.accessKey =
+canal.aliyun.secretKey =
+
 #################################################
 ######### 		destinations		#############
 #################################################
@@ -102,9 +99,6 @@ canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
 canal.auto.scan = true
 canal.auto.scan.interval = 5
-# set this value to 'true' means that when binlog pos not found, skip to latest.
-# WARN: pls keep 'false' in production env, or if you know what you want.
-canal.auto.reset.latest.pos.mode = false
 
 canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
 #canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
@@ -117,58 +111,29 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml
 
 ##################################################
-######### 	      MQ Properties      #############
+######### 		     MQ 		     #############
 ##################################################
-# aliyun ak/sk , support rds/mq
+canal.mq.servers = 127.0.0.1:6667
-canal.aliyun.accessKey =
+canal.mq.retries = 0
-canal.aliyun.secretKey =
+canal.mq.batchSize = 16384
-canal.aliyun.uid=
+canal.mq.maxRequestSize = 1048576
-
+canal.mq.lingerMs = 100
-canal.mq.flatMessage = true
+canal.mq.bufferMemory = 33554432
 canal.mq.canalBatchSize = 50
 canal.mq.canalGetTimeout = 100
+canal.mq.flatMessage = true
+canal.mq.compressionType = none
+canal.mq.acks = all
+#canal.mq.properties. =
+canal.mq.producerGroup = test
 # Set this value to "cloud", if you want open message trace feature in aliyun.
 canal.mq.accessChannel = local
-
+# aliyun mq namespace
-canal.mq.database.hash = true
+#canal.mq.namespace =
-canal.mq.send.thread.size = 30
-canal.mq.build.thread.size = 8
 
 ##################################################
-######### 		     Kafka 		     #############
+#########     Kafka Kerberos Info    #############
 ##################################################
-kafka.bootstrap.servers = 127.0.0.1:6667
+canal.mq.kafka.kerberos.enable = false
-kafka.acks = all
+canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
-kafka.compression.type = none
+canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
-kafka.batch.size = 16384
-kafka.linger.ms = 1
-kafka.max.request.size = 1048576
-kafka.buffer.memory = 33554432
-kafka.max.in.flight.requests.per.connection = 1
-kafka.retries = 0
-
-kafka.kerberos.enable = false
-kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
-kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
-
-##################################################
-######### 		    RocketMQ	     #############
-##################################################
-rocketmq.producer.group = test
-rocketmq.enable.message.trace = false
-rocketmq.customized.trace.topic =
-rocketmq.namespace =
-rocketmq.namesrv.addr = 127.0.0.1:9876
-rocketmq.retry.times.when.send.failed = 0
-rocketmq.vip.channel.enabled = false
-rocketmq.tag = 
-
-##################################################
-######### 		    RabbitMQ	     #############
-##################################################
-rabbitmq.host =
-rabbitmq.virtual.host =
-rabbitmq.exchange =
-rabbitmq.username =
-rabbitmq.password =
-rabbitmq.deliveryMode =

admin/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alibaba.otter</groupId>
     <artifactId>canal.admin</artifactId>
-    <version>1.1.5</version>
+    <version>1.1.5-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>canal admin module for otter ${project.version}</name>
     <modules>
@@ -69,7 +69,7 @@
             <dependency>
                 <groupId>mysql</groupId>
                 <artifactId>mysql-connector-java</artifactId>
-                <version>5.1.48</version>
+                <version>8.0.17</version>
             </dependency>
             <dependency>
                 <groupId>com.github.ben-manes.caffeine</groupId>

client-adapter/common/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -17,11 +17,6 @@
             <artifactId>canal.protocol</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>connector.core</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-core</artifactId>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java

@@ -140,7 +140,7 @@ public class YmlConfigBinder {
             MutablePropertySources propertySources = new MutablePropertySources();
             propertySources.addFirst(propertySource);
 
-            PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<>(target);
+            PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target);
             factory.setPropertySources(propertySources);
             factory.setIgnoreInvalidFields(true);
             factory.setIgnoreUnknownFields(true);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java

@@ -25,7 +25,7 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
     }
 
     protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
-        this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names)));
+        this(delimiters, ignoreCase, new HashSet<String>(Arrays.asList(names)));
     }
 
     DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){
@@ -70,8 +70,8 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
                 return false;
             }
         }
-        for (boolean b : match) {
+        for (int i = 0; i < match.length; i++) {
-            if (b) {
+            if (match[i]) {
                 return true;
             }
         }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java

@@ -277,7 +277,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
     }
 
     private Set<String> getNames(Iterable<String> prefixes) {
-        Set<String> names = new LinkedHashSet<>();
+        Set<String> names = new LinkedHashSet<String>();
         if (this.target != null) {
             PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
             for (PropertyDescriptor descriptor : descriptors) {
@@ -318,7 +318,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
             // We can filter properties to those starting with the target name, but
             // we can't do a complete filter since we need to trigger the
             // unknown fields check
-            Set<String> relaxedNames = new HashSet<>();
+            Set<String> relaxedNames = new HashSet<String>();
             for (String relaxedTargetName : relaxedTargetNames) {
                 relaxedNames.add(relaxedTargetName);
             }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java

@@ -9,9 +9,23 @@ package com.alibaba.otter.canal.client.adapter.config.bind;
  */
 interface PropertyNamePatternsMatcher {
 
-    PropertyNamePatternsMatcher ALL  = propertyName -> true;
+    PropertyNamePatternsMatcher ALL  = new PropertyNamePatternsMatcher() {
 
-    PropertyNamePatternsMatcher NONE = propertyName -> false;
+                                         @Override
+                                         public boolean matches(String propertyName) {
+                                             return true;
+                                         }
+
+                                     };
+
+    PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() {
+
+                                         @Override
+                                         public boolean matches(String propertyName) {
+                                             return false;
+                                         }
+
+                                     };
 
     /**
      * Return {@code true} of the property name matches.

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java

@@ -37,9 +37,9 @@ public class PropertySourcesPropertyValues implements PropertyValues {
 
     private final PropertyNamePatternsMatcher                  includes;
 
-    private final Map<String, PropertyValue>                   propertyValues      = new LinkedHashMap<>();
+    private final Map<String, PropertyValue>                   propertyValues      = new LinkedHashMap<String, PropertyValue>();
 
-    private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners    = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners    = new ConcurrentHashMap<String, PropertySource<?>>();
 
     private final boolean                                      resolvePlaceholders;
 

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java

@@ -32,7 +32,7 @@ public class RelaxedDataBinder extends DataBinder {
     private static final Set<Class<?>> EXCLUDED_EDITORS;
 
     static {
-        Set<Class<?>> excluded = new HashSet<>();
+        Set<Class<?>> excluded = new HashSet<Class<?>>();
         excluded.add(FileEditor.class);
         EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
     }
@@ -43,7 +43,7 @@ public class RelaxedDataBinder extends DataBinder {
 
     private boolean                       ignoreNestedProperties;
 
-    private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<>();
+    private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<String, String>();
 
     /**
      * Create a new {@link RelaxedDataBinder} instance.
@@ -89,7 +89,7 @@ public class RelaxedDataBinder extends DataBinder {
      * @param aliases a map of property name to aliases
      */
     public void setNameAliases(Map<String, List<String>> aliases) {
-        this.nameAliases = new LinkedMultiValueMap<>(aliases);
+        this.nameAliases = new LinkedMultiValueMap<String, String>(aliases);
     }
 
     /**
@@ -129,8 +129,8 @@ public class RelaxedDataBinder extends DataBinder {
         BeanWrapper wrapper = new BeanWrapperImpl(target);
         wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
         wrapper.setAutoGrowNestedPaths(true);
-        List<PropertyValue> sortedValues = new ArrayList<>();
+        List<PropertyValue> sortedValues = new ArrayList<PropertyValue>();
-        Set<String> modifiedNames = new HashSet<>();
+        Set<String> modifiedNames = new HashSet<String>();
         List<String> sortedNames = getSortedPropertyNames(propertyValues);
         for (String name : sortedNames) {
             PropertyValue propertyValue = propertyValues.getPropertyValue(name);
@@ -143,7 +143,7 @@ public class RelaxedDataBinder extends DataBinder {
     }
 
     private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
-        List<String> names = new LinkedList<>();
+        List<String> names = new LinkedList<String>();
         for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
             names.add(propertyValue.getName());
         }
@@ -161,7 +161,7 @@ public class RelaxedDataBinder extends DataBinder {
      * @param names the names to sort
      */
     private void sortPropertyNames(List<String> names) {
-        for (String name : new ArrayList<>(names)) {
+        for (String name : new ArrayList<String>(names)) {
             int propertyIndex = names.indexOf(name);
             RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
             for (String prefix : path.prefixes()) {
@@ -319,7 +319,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
         Object extend = new LinkedHashMap<String, Object>();
         if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
-            extend = new ArrayList<>();
+            extend = new ArrayList<Object>();
         }
         wrapper.setPropertyValue(path.prefix(index + 1), extend);
     }
@@ -347,7 +347,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
         Object extend = new LinkedHashMap<String, Object>();
         if (descriptor.isCollection()) {
-            extend = new ArrayList<>();
+            extend = new ArrayList<Object>();
         }
         if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
             extend = BLANK;
@@ -411,7 +411,7 @@ public class RelaxedDataBinder extends DataBinder {
         if (aliases == null) {
             return Collections.singleton(name);
         }
-        List<String> nameAndAliases = new ArrayList<>(aliases.size() + 1);
+        List<String> nameAndAliases = new ArrayList<String>(aliases.size() + 1);
         nameAndAliases.add(name);
         nameAndAliases.addAll(aliases);
         return nameAndAliases;
@@ -473,7 +473,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
 
         public List<String> prefixes() {
-            List<String> prefixes = new ArrayList<>();
+            List<String> prefixes = new ArrayList<String>();
             for (int index = 1; index < this.nodes.size(); index++) {
                 prefixes.add(prefix(index));
             }
@@ -485,7 +485,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
 
         private List<PathNode> splitPath(String path) {
-            List<PathNode> nodes = new ArrayList<>();
+            List<PathNode> nodes = new ArrayList<PathNode>();
             String current = extractIndexedPaths(path, nodes);
             for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
                 if (StringUtils.hasText(name)) {
@@ -518,7 +518,7 @@ public class RelaxedDataBinder extends DataBinder {
         }
 
         public void collapseKeys(int index) {
-            List<PathNode> revised = new ArrayList<>();
+            List<PathNode> revised = new ArrayList<PathNode>();
             for (int i = 0; i < index; i++) {
                 revised.add(this.nodes.get(i));
             }
@@ -668,7 +668,7 @@ public class RelaxedDataBinder extends DataBinder {
         private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
 
         static {
-            Set<String> names = new HashSet<>();
+            Set<String> names = new HashSet<String>();
             names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
             names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
             BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java

@@ -24,7 +24,7 @@ public final class RelaxedNames implements Iterable<String> {
 
     private final String         name;
 
-    private final Set<String>    values                          = new LinkedHashSet<>();
+    private final Set<String>    values                          = new LinkedHashSet<String>();
 
     /**
      * Create a new {@link RelaxedNames} instance.

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java

@@ -22,7 +22,7 @@ import org.springframework.util.StringUtils;
  */
 public class CompositePropertySource extends EnumerablePropertySource<Object> {
 
-    private final Set<PropertySource<?>> propertySources = new LinkedHashSet<>();
+    private final Set<PropertySource<?>> propertySources = new LinkedHashSet<PropertySource<?>>();
 
     /**
      * Create a new {@code CompositePropertySource}.
@@ -56,7 +56,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
 
     @Override
     public String[] getPropertyNames() {
-        Set<String> names = new LinkedHashSet<>();
+        Set<String> names = new LinkedHashSet<String>();
         for (PropertySource<?> propertySource : this.propertySources) {
             if (!(propertySource instanceof EnumerablePropertySource)) {
                 throw new IllegalStateException(
@@ -83,7 +83,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
      * @since 4.1
      */
     public void addFirstPropertySource(PropertySource<?> propertySource) {
-        List<PropertySource<?>> existing = new ArrayList<>(this.propertySources);
+        List<PropertySource<?>> existing = new ArrayList<PropertySource<?>>(this.propertySources);
         this.propertySources.clear();
         this.propertySources.add(propertySource);
         this.propertySources.addAll(existing);

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java

@@ -28,7 +28,7 @@ public class MutablePropertySources implements PropertySources {
 
     private final Log                     logger;
 
-    private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<>();
+    private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<PropertySource<?>>();
 
     /**
      * Create a new {@link MutablePropertySources}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java

@@ -31,7 +31,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
     }
 
     public void addActiveProfiles(String... profiles) {
-        LinkedHashSet<String> set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles));
+        LinkedHashSet<String> set = new LinkedHashSet<String>(Arrays.asList(this.activeProfiles));
         Collections.addAll(set, profiles);
         this.activeProfiles = set.toArray(new String[set.size()]);
     }
@@ -67,14 +67,14 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
 
     private ProfilesMatcher getProfilesMatcher() {
         return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher(
-                new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
+            new HashSet<String>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
     }
 
     private Set<String> extractProfiles(List<String> profiles, ProfileType type) {
         if (CollectionUtils.isEmpty(profiles)) {
             return null;
         }
-        Set<String> extractedProfiles = new HashSet<>();
+        Set<String> extractedProfiles = new HashSet<String>();
         for (String candidate : profiles) {
             ProfileType candidateType = ProfileType.POSITIVE;
             if (candidate.startsWith("!")) {
@@ -167,7 +167,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
      */
     static class SpringProperties {
 
-        private List<String> profiles = new ArrayList<>();
+        private List<String> profiles = new ArrayList<String>();
 
         public List<String> getProfiles() {
             return this.profiles;

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java

@@ -130,7 +130,8 @@ public abstract class YamlProcessor {
             if (logger.isDebugEnabled()) {
                 logger.debug("Loading from YAML: " + resource);
             }
-            try (Reader reader = new UnicodeReader(resource.getInputStream())) {
+            Reader reader = new UnicodeReader(resource.getInputStream());
+            try {
                 for (Object object : yaml.loadAll(reader)) {
                     if (object != null && process(asMap(object), callback)) {
                         count++;
@@ -141,8 +142,10 @@ public abstract class YamlProcessor {
                 }
                 if (logger.isDebugEnabled()) {
                     logger.debug(
-                            "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
+                        "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
                 }
+            } finally {
+                reader.close();
             }
         } catch (IOException ex) {
             handleProcessError(resource, ex);
@@ -163,7 +166,7 @@ public abstract class YamlProcessor {
     @SuppressWarnings("unchecked")
     private Map<String, Object> asMap(Object object) {
         // YAML can have numbers as keys
-        Map<String, Object> result = new LinkedHashMap<>();
+        Map<String, Object> result = new LinkedHashMap<String, Object>();
         if (!(object instanceof Map)) {
             // A document can be a text literal
             result.put("document", object);
@@ -246,7 +249,7 @@ public abstract class YamlProcessor {
      * @since 4.1.3
      */
     protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) {
-        Map<String, Object> result = new LinkedHashMap<>();
+        Map<String, Object> result = new LinkedHashMap<String, Object>();
         buildFlattenedMap(result, source, null);
         return result;
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.config.common;
 import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Properties;
 import java.util.regex.Pattern;
 
 import org.springframework.util.ClassUtils;
@@ -71,8 +72,14 @@ public class YamlPropertySourceLoader implements PropertySourceLoader {
         }
 
         public Map<String, Object> process() {
-            final Map<String, Object> result = new LinkedHashMap<>();
+            final Map<String, Object> result = new LinkedHashMap<String, Object>();
-            process((properties, map) -> result.putAll(getFlattenedMap(map)));
+            process(new MatchCallback() {
+
+                @Override
+                public void process(Properties properties, Map<String, Object> map) {
+                    result.putAll(getFlattenedMap(map));
+                }
+            });
             return result;
         }
 

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.client.adapter.support;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Properties;
 
 /**
  * 配置信息类
@@ -28,7 +27,7 @@ public class CanalClientConfig {
     // 重试次数
     private Integer            retries;
     // 消费超时时间
-    private Long               timeout       = 500L;
+    private Long               timeout;
     // 模式 tcp kafka rocketMQ
     private String             mode          = "tcp";
     // aliyun ak/sk
@@ -40,8 +39,6 @@ public class CanalClientConfig {
     // rabbitmq vhost
     private String             vhost         = "/";
 
-    private Properties         consumerProperties;
-
     private Long               resourceOwnerId;
     // 是否启用消息轨迹
     private boolean            enableMessageTrace;
@@ -166,14 +163,6 @@ public class CanalClientConfig {
         this.vhost = vhost;
     }
 
-    public Properties getConsumerProperties() {
-        return consumerProperties;
-    }
-
-    public void setConsumerProperties(Properties consumerProperties) {
-        this.consumerProperties = consumerProperties;
-    }
-
     public Long getResourceOwnerId() {
         return resourceOwnerId;
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java

@@ -6,7 +6,6 @@ import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
@@ -260,7 +259,7 @@ public class ExtensionLoader<T> {
             }
         }
 
-        Map<String, Class<?>> extensionClasses = new HashMap<>();
+        Map<String, Class<?>> extensionClasses = new HashMap<String, Class<?>>();
 
         // 1. plugin folder，customized extension classLoader （jar_dir/plugin）
         String dir = File.separator + this.getJarDirectoryPath() + File.separator + "plugin";
@@ -318,8 +317,10 @@ public class ExtensionLoader<T> {
                 while (urls.hasMoreElements()) {
                     URL url = urls.nextElement();
                     try {
-                        try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8))) {
+                        BufferedReader reader = null;
-                            String line;
+                        try {
+                            reader = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));
+                            String line = null;
                             while ((line = reader.readLine()) != null) {
                                 final int ci = line.indexOf('#');
                                 if (ci >= 0) line = line.substring(0, ci);
@@ -339,10 +340,10 @@ public class ExtensionLoader<T> {
                                             // classLoader);
                                             if (!type.isAssignableFrom(clazz)) {
                                                 throw new IllegalStateException(
-                                                        "Error when load extension class(interface: " + type
+                                                    "Error when load extension class(interface: " + type
-                                                                + ", class line: " + clazz.getName()
+                                                                                + ", class line: " + clazz.getName()
-                                                                + "), class " + clazz.getName()
+                                                                                + "), class " + clazz.getName()
-                                                                + "is not subtype of interface.");
+                                                                                + "is not subtype of interface.");
                                             } else {
                                                 try {
                                                     clazz.getConstructor(type);
@@ -360,10 +361,10 @@ public class ExtensionLoader<T> {
                                                             } else if (c != clazz) {
                                                                 cachedNames.remove(clazz);
                                                                 throw new IllegalStateException(
-                                                                        "Duplicate extension " + type.getName() + " name "
+                                                                    "Duplicate extension " + type.getName() + " name "
-                                                                                + n + " on "
+                                                                                                + n + " on "
-                                                                                + c.getName() + " and "
+                                                                                                + c.getName() + " and "
-                                                                                + clazz.getName());
+                                                                                                + clazz.getName());
                                                             }
                                                         }
                                                     }
@@ -372,15 +373,19 @@ public class ExtensionLoader<T> {
                                         }
                                     } catch (Throwable t) {
                                         IllegalStateException e = new IllegalStateException(
-                                                "Failed to load extension class(interface: " + type + ", class line: "
+                                            "Failed to load extension class(interface: " + type + ", class line: "
-                                                        + line + ") in " + url
+                                                                                            + line + ") in " + url
-                                                        + ", cause: "
+                                                                                            + ", cause: "
-                                                        + t.getMessage(),
+                                                                                            + t.getMessage(),
-                                                t);
+                                            t);
                                         exceptions.put(line, e);
                                     }
                                 }
                             } // end of while read lines
+                        } finally {
+                            if (reader != null) {
+                                reader.close();
+                            }
                         }
                     } catch (Throwable t) {
                         logger.error("Exception when load extension class(interface: " + type + ", class file: " + url

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java

@@ -2,8 +2,8 @@ package com.alibaba.otter.canal.client.adapter.support;
 
 import java.util.*;
 
-import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -19,7 +19,7 @@ public class MessageUtil {
             return null;
         }
         List<CanalEntry.Entry> entries = message.getEntries();
-        List<Dml> dmls = new ArrayList<>(entries.size());
+        List<Dml> dmls = new ArrayList<Dml>(entries.size());
         for (CanalEntry.Entry entry : entries) {
             if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
                 || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
@@ -131,13 +131,10 @@ public class MessageUtil {
         return dmls;
     }
 
-    public static List<Dml> flatMessage2Dml(String destination, String groupId, List<CommonMessage> commonMessages) {
+    public static List<Dml> flatMessage2Dml(String destination, String groupId, List<FlatMessage> flatMessages) {
-        if (commonMessages == null) {
+        List<Dml> dmls = new ArrayList<Dml>(flatMessages.size());
-            return new ArrayList<>();
+        for (FlatMessage flatMessage : flatMessages) {
-        }
+            Dml dml = flatMessage2Dml(destination, groupId, flatMessage);
-        List<Dml> dmls = new ArrayList<>(commonMessages.size());
-        for (CommonMessage commonMessage : commonMessages) {
-            Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
             if (dml != null) {
                 dmls.add(dml);
             }
@@ -146,31 +143,31 @@ public class MessageUtil {
         return dmls;
     }
 
-    public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) {
+    public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) {
-        if (commonMessage == null) {
+        if (flatMessage == null) {
             return null;
         }
         Dml dml = new Dml();
         dml.setDestination(destination);
         dml.setGroupId(groupId);
-        dml.setDatabase(commonMessage.getDatabase());
+        dml.setDatabase(flatMessage.getDatabase());
-        dml.setTable(commonMessage.getTable());
+        dml.setTable(flatMessage.getTable());
-        dml.setPkNames(commonMessage.getPkNames());
+        dml.setPkNames(flatMessage.getPkNames());
-        dml.setIsDdl(commonMessage.getIsDdl());
+        dml.setIsDdl(flatMessage.getIsDdl());
-        dml.setType(commonMessage.getType());
+        dml.setType(flatMessage.getType());
-        dml.setTs(commonMessage.getTs());
+        dml.setTs(flatMessage.getTs());
-        dml.setEs(commonMessage.getEs());
+        dml.setEs(flatMessage.getEs());
-        dml.setSql(commonMessage.getSql());
+        dml.setSql(flatMessage.getSql());
         // if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
         // throw new RuntimeException("SqlType or mysqlType is null");
         // }
-        List<Map<String, Object>> data = commonMessage.getData();
+        List<Map<String, String>> data = flatMessage.getData();
         if (data != null) {
-            dml.setData(data);
+            dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
         }
-        List<Map<String, Object>> old = commonMessage.getOld();
+        List<Map<String, String>> old = flatMessage.getOld();
         if (old != null) {
-            dml.setOld(old);
+            dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
         }
         return dml;
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/URLClassExtensionLoader.java

@@ -7,8 +7,7 @@ import java.util.Enumeration;
 import java.util.NoSuchElementException;
 
 public class URLClassExtensionLoader extends URLClassLoader {
-
+    public URLClassExtensionLoader(URL[] urls) {
-    public URLClassExtensionLoader(URL[] urls){
         super(urls);
     }
 
@@ -19,9 +18,9 @@ public class URLClassExtensionLoader extends URLClassLoader {
             return c;
         }
 
-        if (name.startsWith("java.") || name.startsWith("org.slf4j.") || name.startsWith("org.apache.logging")
+        if (name.startsWith("java.") || name.startsWith("org.slf4j.")
-            || name.startsWith("org.apache.zookeeper.") || name.startsWith("org.I0Itec.zkclient.")
+                || name.startsWith("org.apache.logging")
-            || name.startsWith("org.apache.commons.logging.")) {
+                || name.startsWith("org.apache.commons.logging.")) {
             // || name.startsWith("org.apache.hadoop."))
             // {
             c = super.loadClass(name);

client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtilTest.java

@@ -1,13 +1,24 @@
 package com.alibaba.otter.canal.client.adapter.support;
 
+import static org.mockito.AdditionalMatchers.or;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Matchers.isNull;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+
+import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
+import java.lang.reflect.Array;
+
 @RunWith(PowerMockRunner.class)
 public class JdbcTypeUtilTest {
 

client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/UtilTest.java

@@ -3,7 +3,9 @@ package com.alibaba.otter.canal.client.adapter.support;
 import static org.mockito.AdditionalMatchers.or;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Matchers.isNull;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
 
+import com.alibaba.otter.canal.client.adapter.support.Util;
 import com.diffblue.deeptestutils.mock.DTUMemberMatcher;
 import org.apache.commons.lang.StringUtils;
 import org.junit.Assert;

client-adapter/es6x/pom.xml

@@ -1,9 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java

@@ -5,10 +5,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import javax.sql.DataSource;
-
-import org.elasticsearch.action.search.SearchResponse;
-
 import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es6x.etl.ESEtlService;
@@ -18,6 +14,9 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
+import org.elasticsearch.action.search.SearchResponse;
+
+import javax.sql.DataSource;
 
 /**
  * ES 6.x 外部适配器

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java

@@ -10,6 +10,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import javax.sql.DataSource;
 
 import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -78,118 +79,119 @@ public class ESEtlService extends AbstractEtlService {
                             }
 
                             // 如果是主键字段则不插入
-                if (fieldItem.getFieldName().equals(mapping.get_id())) {
+                            if (fieldItem.getFieldName().equals(mapping.get_id())) {
-                    idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                                idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                } else {
+                            } else {
-                    Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
+                                Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                    esFieldData.put(Util.cleanColumn(fieldName), val);
+                                esFieldData.put(Util.cleanColumn(fieldName), val);
-                }
+                            }
-
-            }
-
-            if (!mapping.getRelations().isEmpty()) {
-                mapping.getRelations().forEach((relationField, relationMapping) -> {
-                    Map<String, Object> relations = new HashMap<>();
-                    relations.put("name", relationMapping.getName());
-                    if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                        FieldItem parentFieldItem = mapping.getSchemaItem()
-                            .getSelectFields()
-                            .get(relationMapping.getParent());
-                        Object parentVal;
-                        try {
-                            parentVal = esTemplate.getValFromRS(mapping,
-                                rs,
-                                parentFieldItem.getFieldName(),
-                                parentFieldItem.getFieldName());
-                        } catch (SQLException e) {
-                            throw new RuntimeException(e);
-                        }
-                        if (parentVal != null) {
-                            relations.put("parent", parentVal.toString());
-                            esFieldData.put("$parent_routing", parentVal.toString());
 
                         }
-                    }
-                    esFieldData.put(Util.cleanColumn(relationField), relations);
-                });
-            }
 
-            if (idVal != null) {
+                        if (!mapping.getRelations().isEmpty()) {
-                String parentVal = (String) esFieldData.remove("$parent_routing");
+                            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                if (mapping.isUpsert()) {
+                                Map<String, Object> relations = new HashMap<>();
-                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
+                                relations.put("name", relationMapping.getName());
-                        mapping.get_type(),
+                                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                        idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+                                    FieldItem parentFieldItem = mapping.getSchemaItem()
+                                        .getSelectFields()
+                                        .get(relationMapping.getParent());
+                                    Object parentVal;
+                                    try {
+                                        parentVal = esTemplate.getValFromRS(mapping,
+                                            rs,
+                                            parentFieldItem.getFieldName(),
+                                            parentFieldItem.getFieldName());
+                                    } catch (SQLException e) {
+                                        throw new RuntimeException(e);
+                                    }
+                                    if (parentVal != null) {
+                                        relations.put("parent", parentVal.toString());
+                                        esFieldData.put("$parent_routing", parentVal.toString());
 
-                    if (StringUtils.isNotEmpty(parentVal)) {
+                                    }
-                        esUpdateRequest.setRouting(parentVal);
+                                }
+                                esFieldData.put(Util.cleanColumn(relationField), relations);
+                            });
+                        }
+
+                        if (idVal != null) {
+                            String parentVal = (String) esFieldData.remove("$parent_routing");
+                            if (mapping.isUpsert()) {
+                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(
+                                    mapping.get_index(),
+                                    mapping.get_type(),
+                                    idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
+
+                                if (StringUtils.isNotEmpty(parentVal)) {
+                                    esUpdateRequest.setRouting(parentVal);
+                                }
+
+                                esBulkRequest.add(esUpdateRequest);
+                            } else {
+                                ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping
+                                    .get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
+                                if (StringUtils.isNotEmpty(parentVal)) {
+                                    esIndexRequest.setRouting(parentVal);
+                                }
+                                esBulkRequest.add(esIndexRequest);
+                            }
+                        } else {
+                            idVal = esFieldData.get(mapping.getPk());
+                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
+                                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
+                                    .size(10000);
+                            SearchResponse response = esSearchRequest.getResponse();
+                            for (SearchHit hit : response.getHits()) {
+                                ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping
+                                    .get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
+                                esBulkRequest.add(esUpdateRequest);
+                            }
+                        }
+
+                        if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
+                            && esBulkRequest.numberOfActions() > 0) {
+                            long esBatchBegin = System.currentTimeMillis();
+                            ESBulkResponse rp = esBulkRequest.bulk();
+                            if (rp.hasFailures()) {
+                                rp.processFailBulkResponse("全量数据 etl 异常 ");
+                            }
+
+                            if (logger.isTraceEnabled()) {
+                                logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                                    (System.currentTimeMillis() - batchBegin),
+                                    (System.currentTimeMillis() - esBatchBegin),
+                                    esBulkRequest.numberOfActions(),
+                                    mapping.get_index());
+                            }
+                            batchBegin = System.currentTimeMillis();
+                            esBulkRequest.resetBulk();
+                        }
+                        count++;
+                        impCount.incrementAndGet();
                     }
 
-                    esBulkRequest.add(esUpdateRequest);
+                    if (esBulkRequest.numberOfActions() > 0) {
-                } else {
+                        long esBatchBegin = System.currentTimeMillis();
-                    ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(),
+                        ESBulkResponse rp = esBulkRequest.bulk();
-                        mapping.get_type(),
+                        if (rp.hasFailures()) {
-                        idVal.toString()).setSource(esFieldData);
+                            rp.processFailBulkResponse("全量数据 etl 异常 ");
-                    if (StringUtils.isNotEmpty(parentVal)) {
+                        }
-                        esIndexRequest.setRouting(parentVal);
+                        if (logger.isTraceEnabled()) {
+                            logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
+                                (System.currentTimeMillis() - batchBegin),
+                                (System.currentTimeMillis() - esBatchBegin),
+                                esBulkRequest.numberOfActions(),
+                                mapping.get_index());
+                        }
                     }
-                    esBulkRequest.add(esIndexRequest);
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
+                    throw new RuntimeException(e);
                 }
-            } else {
+                return count;
-                idVal = esFieldData.get(mapping.getPk());
+            });
-                ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                    mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
-                SearchResponse response = esSearchRequest.getResponse();
-                for (SearchHit hit : response.getHits()) {
-                    ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
-                        mapping.get_type(),
-                        hit.getId()).setDoc(esFieldData);
-                    esBulkRequest.add(esUpdateRequest);
-                }
-            }
-
-            if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 && esBulkRequest.numberOfActions() > 0) {
-                long esBatchBegin = System.currentTimeMillis();
-                ESBulkResponse rp = esBulkRequest.bulk();
-                if (rp.hasFailures()) {
-                    rp.processFailBulkResponse("全量数据 etl 异常 ");
-                }
-
-                if (logger.isTraceEnabled()) {
-                    logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                        (System.currentTimeMillis() - batchBegin),
-                        (System.currentTimeMillis() - esBatchBegin),
-                        esBulkRequest.numberOfActions(),
-                        mapping.get_index());
-                }
-                batchBegin = System.currentTimeMillis();
-                esBulkRequest.resetBulk();
-            }
-            count++;
-            impCount.incrementAndGet();
-        }
-
-        if (esBulkRequest.numberOfActions() > 0) {
-            long esBatchBegin = System.currentTimeMillis();
-            ESBulkResponse rp = esBulkRequest.bulk();
-            if (rp.hasFailures()) {
-                rp.processFailBulkResponse("全量数据 etl 异常 ");
-            }
-            if (logger.isTraceEnabled()) {
-                logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                    (System.currentTimeMillis() - batchBegin),
-                    (System.currentTimeMillis() - esBatchBegin),
-                    esBulkRequest.numberOfActions(),
-                    mapping.get_index());
-            }
-        }
-    } catch (Exception e) {
-        logger.error(e.getMessage(), e);
-        errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
-        throw new RuntimeException(e);
-    }
-    return count;
-}           );
 
             return true;
         } catch (Exception e) {

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java

@@ -2,11 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es6x.support;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -24,8 +20,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest;

client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java

@@ -2,12 +2,10 @@ package com.alibaba.otter.canal.client.adapter.es6x.support;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.UnknownHostException;
-import java.util.Arrays;
 import java.util.Map;
 
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
 import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
@@ -29,11 +27,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
-import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.*;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.client.RestHighLevelClientExt;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -46,8 +40,6 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
-
 /**
  * ES 连接器, Transport Rest 两种方式
  *
@@ -59,7 +51,7 @@ public class ESConnection {
     private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
 
     public enum ESClientMode {
-        TRANSPORT, REST
+                              TRANSPORT, REST
     }
 
     private ESClientMode        mode;
@@ -81,15 +73,23 @@ public class ESConnection {
                     Integer.parseInt(host.substring(i + 1))));
             }
         } else {
-            HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
+            HttpHost[] httpHosts = new HttpHost[hosts.length];
+            for (int i = 0; i < hosts.length; i++) {
+                String host = hosts[i];
+                int j = host.indexOf(":");
+                HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
+                    Integer.parseInt(host.substring(j + 1)));
+                httpHosts[i] = httpHost;
+            }
             RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
             String nameAndPwd = properties.get("security.auth");
             if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
                 String[] nameAndPwdArr = nameAndPwd.split(":");
                 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
+                credentialsProvider.setCredentials(AuthScope.ANY,
-                    nameAndPwdArr[1]));
+                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-                restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+                restClientBuilder.setHttpClientConfigCallback(
+                    httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
             }
             restHighLevelClient = new RestHighLevelClient(restClientBuilder);
         }
@@ -418,7 +418,6 @@ public class ESConnection {
             }
         }
 
-        @SuppressWarnings("deprecation")
         public ESBulkResponse bulk() {
             if (mode == ESClientMode.TRANSPORT) {
                 BulkResponse responses = bulkRequestBuilder.execute().actionGet();
@@ -503,17 +502,4 @@ public class ESConnection {
     public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
     }
-
-    private HttpHost createHttpHost(String uriStr) {
-        URI uri = URI.create(uriStr);
-        if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
-            return HttpHost.create(uri.toString());
-        }
-        try {
-            return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(),
-                                           uri.getQuery(), uri.getFragment()).toString());
-        } catch (URISyntaxException ex) {
-            throw new IllegalStateException(ex);
-        }
-    }
 }

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.sql.DataSource;
 

client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java

@@ -1,10 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import org.elasticsearch.action.get.GetResponse;
 import org.junit.Assert;

client-adapter/es7x/pom.xml

@@ -1,9 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java

@@ -5,20 +5,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import javax.sql.DataSource;
+import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
-
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import org.elasticsearch.action.search.SearchResponse;
 
 import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
 import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate;
 import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
 import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
 
+import javax.sql.DataSource;
+
 /**
  * ES 7.x 外部适配器
  *

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java

@@ -2,11 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 

client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java

@@ -2,10 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.UnknownHostException;
-import java.util.Arrays;
 import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
@@ -57,12 +54,11 @@ public class ESConnection {
     private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
 
     public enum ESClientMode {
-        TRANSPORT, REST
+                              TRANSPORT, REST
     }
 
     private ESClientMode        mode;
 
-    @SuppressWarnings("deprecation")
     private TransportClient     transportClient;
 
     private RestHighLevelClient restHighLevelClient;
@@ -80,15 +76,23 @@ public class ESConnection {
                     Integer.parseInt(host.substring(i + 1))));
             }
         } else {
-            HttpHost[] httpHosts = Arrays.stream(hosts).map(this::createHttpHost).toArray(HttpHost[]::new);
+            HttpHost[] httpHosts = new HttpHost[hosts.length];
+            for (int i = 0; i < hosts.length; i++) {
+                String host = hosts[i];
+                int j = host.indexOf(":");
+                HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
+                    Integer.parseInt(host.substring(j + 1)));
+                httpHosts[i] = httpHost;
+            }
             RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
             String nameAndPwd = properties.get("security.auth");
             if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
                 String[] nameAndPwdArr = nameAndPwd.split(":");
                 final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
+                credentialsProvider.setCredentials(AuthScope.ANY,
-                    nameAndPwdArr[1]));
+                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-                restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+                restClientBuilder.setHttpClientConfigCallback(
+                    httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
             }
             restHighLevelClient = new RestHighLevelClient(restClientBuilder);
         }
@@ -497,17 +501,4 @@ public class ESConnection {
     public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
     }
-
-    private HttpHost createHttpHost(String uriStr) {
-        URI uri = URI.create(uriStr);
-        if (!org.springframework.util.StringUtils.hasLength(uri.getUserInfo())) {
-            return HttpHost.create(uri.toString());
-        }
-        try {
-            return HttpHost.create(new URI(uri.getScheme(), null, uri.getHost(), uri.getPort(), uri.getPath(),
-                                           uri.getQuery(), uri.getFragment()).toString());
-        } catch (URISyntaxException ex) {
-            throw new IllegalStateException(ex);
-        }
-    }
 }

client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java

@@ -24,7 +24,6 @@ import org.junit.Test;
 @Ignore
 public class ES7xTest {
 
-    @SuppressWarnings("deprecation")
     private TransportClient transportClient;
 
     @Before
@@ -66,9 +65,8 @@ public class ES7xTest {
         relations.put("parent", "2");
 
         BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
-        bulkRequestBuilder.add(transportClient.prepareIndex("test", "osm", "2_4")
+        bulkRequestBuilder
-            .setRouting("2")
+            .add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData));
-            .setSource(esFieldData));
         commit(bulkRequestBuilder);
     }
 

client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java

@@ -8,9 +8,9 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.springframework.util.Assert;
 
 import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
+import org.springframework.util.Assert;
 
 @Ignore
 public class ESConnectionTest {

client-adapter/escore/pom.xml

@@ -1,9 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -20,8 +22,8 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>com.alibaba</groupId>
+            <groupId>com.alibaba.fastsql</groupId>
-            <artifactId>druid</artifactId>
+            <artifactId>fastsql</artifactId>
         </dependency>
 
         <dependency>

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.client.adapter.es.core;
 
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -7,6 +8,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import org.apache.commons.lang.StringUtils;
 
 import com.alibaba.druid.pool.DruidDataSource;
@@ -17,7 +19,6 @@ import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor;
 import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService;
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
@@ -62,8 +63,8 @@ public abstract class ESAdapter implements OuterAdapter {
             // 过滤不匹配的key的配置
             esSyncConfigTmp.forEach((key, config) -> {
                 if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                    || (config.getOuterAdapterKey() != null
-                        .equalsIgnoreCase(configuration.getKey()))) {
+                        && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
                     esSyncConfig.put(key, config);
                 }
             });
@@ -103,12 +104,12 @@ public abstract class ESAdapter implements OuterAdapter {
         String table = dml.getTable();
         Map<String, ESSyncConfig> configMap;
         if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "-"
+            configMap = dbTableEsSyncConfig
-                                                + StringUtils.trimToEmpty(dml.getGroupId()) + "_" + database + "-"
+                .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
-                                                + table);
+                     + "_" + database + "-" + table);
         } else {
-            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-"
+            configMap = dbTableEsSyncConfig
-                                                + table);
+                .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
         }
 
         if (configMap != null && !configMap.values().isEmpty()) {
@@ -154,29 +155,23 @@ public abstract class ESAdapter implements OuterAdapter {
         }
         String schema = matcher.group(2);
 
-        schemaItem.getAliasTableItems()
+        schemaItem.getAliasTableItems().values().forEach(tableItem -> {
-            .values()
+            Map<String, ESSyncConfig> esSyncConfigMap;
-            .forEach(tableItem -> {
+            if (envProperties != null
-                Map<String, ESSyncConfig> esSyncConfigMap;
+                    && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                esSyncConfigMap = dbTableEsSyncConfig
-                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+                        .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
-                                                                          + "-"
+                                        + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-"
-                                                                          + StringUtils.trimToEmpty(config.getGroupId())
+                                        + tableItem.getTableName(),
-                                                                          + "_"
+                                k -> new ConcurrentHashMap<>());
-                                                                          + schema
+            } else {
-                                                                          + "-"
+                esSyncConfigMap = dbTableEsSyncConfig
-                                                                          + tableItem.getTableName(),
+                        .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-"
-                        k -> new ConcurrentHashMap<>());
+                                        + tableItem.getTableName(),
-                } else {
+                                k -> new ConcurrentHashMap<>());
-                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+            }
-                                                                          + "_"
-                                                                          + schema
-                                                                          + "-"
-                                                                          + tableItem.getTableName(),
-                        k -> new ConcurrentHashMap<>());
-                }
 
-                esSyncConfigMap.put(configName, config);
+            esSyncConfigMap.put(configName, config);
-            });
+        });
     }
 }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java

@@ -1,31 +1,20 @@
 package com.alibaba.otter.canal.client.adapter.es.core.config;
 
-import static com.alibaba.druid.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
+import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
-import static com.alibaba.druid.sql.ast.expr.SQLBinaryOperator.Equality;
+import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.fastsql.sql.SQLUtils;
-import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.fastsql.sql.ast.SQLExpr;
-import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr;
+import com.alibaba.fastsql.sql.ast.expr.*;
-import com.alibaba.druid.sql.ast.expr.SQLCaseExpr;
+import com.alibaba.fastsql.sql.ast.statement.*;
-import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
+import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr;
+import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
-import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.fastsql.sql.parser.ParserException;
-import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
+import com.alibaba.fastsql.sql.parser.SQLStatementParser;
-import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
-import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
-import com.alibaba.druid.sql.ast.statement.SQLSelectGroupByClause;
-import com.alibaba.druid.sql.ast.statement.SQLSelectItem;
-import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
-import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
-import com.alibaba.druid.sql.ast.statement.SQLTableSource;
-import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
-import com.alibaba.druid.sql.parser.ParserException;
-import com.alibaba.druid.sql.parser.SQLStatementParser;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.RelationFieldsPair;
@@ -230,7 +219,8 @@ public class SqlParser {
             }
             FieldItem rightFieldItem = new FieldItem();
             visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
-            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod() || rightFieldItem.isBinaryOp()) {
+            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
+                || rightFieldItem.isBinaryOp()) {
                 throw new UnsupportedOperationException("Unsupported for complex of on-condition");
             }
             tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java

@@ -1,9 +1,13 @@
 package com.alibaba.otter.canal.client.adapter.es.core.monitor;
 
 import java.io.File;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
 import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -11,9 +15,12 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
-import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
+import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
 
@@ -63,11 +70,8 @@ public class ESConfigMonitor {
             try {
                 // 加载新增的配置文件
                 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
+                ESSyncConfig config = YmlConfigBinder
-                    configContent,
+                    .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
-                    ESSyncConfig.class,
-                    null,
-                    envProperties);
                 if (config != null) {
                     config.validate();
                     addConfigToCache(file, config);
@@ -85,17 +89,14 @@ public class ESConfigMonitor {
             try {
                 if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
                     // 加载配置文件
-                    String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
+                    String configContent = MappingConfigsLoader
-                                                                           + file.getName());
+                        .loadConfig(adapterName + File.separator + file.getName());
                     if (configContent == null) {
                         onFileDelete(file);
                         return;
                     }
-                    ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
+                    ESSyncConfig config = YmlConfigBinder
-                        configContent,
+                        .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
-                        ESSyncConfig.class,
-                        null,
-                        envProperties);
                     if (config == null) {
                         return;
                     }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java

@@ -1,26 +1,22 @@
 package com.alibaba.otter.canal.client.adapter.es.core.service;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
 
 import javax.sql.DataSource;
 
+import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java

@@ -5,13 +5,7 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.sql.Blob;
 import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
 
 import org.apache.commons.codec.binary.Base64;
 import org.joda.time.DateTime;
@@ -63,187 +57,172 @@ public class ESSyncUtil {
             return val;
         }
         Object res = null;
-        switch (esType) {
+        if ("integer".equals(esType)) {
-            case "integer":
+            if (val instanceof Number) {
-                if (val instanceof Number) {
+                res = ((Number) val).intValue();
-                    res = ((Number) val).intValue();
+            } else {
+                res = Integer.parseInt(val.toString());
+            }
+        } else if ("long".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).longValue();
+            } else {
+                res = Long.parseLong(val.toString());
+            }
+        } else if ("short".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).shortValue();
+            } else {
+                res = Short.parseShort(val.toString());
+            }
+        } else if ("byte".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).byteValue();
+            } else {
+                res = Byte.parseByte(val.toString());
+            }
+        } else if ("double".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).doubleValue();
+            } else {
+                res = Double.parseDouble(val.toString());
+            }
+        } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) {
+            if (val instanceof Number) {
+                res = ((Number) val).floatValue();
+            } else {
+                res = Float.parseFloat(val.toString());
+            }
+        } else if ("boolean".equals(esType)) {
+            if (val instanceof Boolean) {
+                res = val;
+            } else if (val instanceof Number) {
+                int v = ((Number) val).intValue();
+                res = v != 0;
+            } else {
+                res = Boolean.parseBoolean(val.toString());
+            }
+        } else if ("date".equals(esType)) {
+            if (val instanceof java.sql.Time) {
+                DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
+                if (dateTime.getMillisOfSecond() != 0) {
+                    res = dateTime.toString("HH:mm:ss.SSS");
                 } else {
-                    res = Integer.parseInt(val.toString());
+                    res = dateTime.toString("HH:mm:ss");
                 }
-                break;
+            } else if (val instanceof java.sql.Timestamp) {
-            case "long":
+                DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
-                if (val instanceof Number) {
+                if (dateTime.getMillisOfSecond() != 0) {
-                    res = ((Number) val).longValue();
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
                 } else {
-                    res = Long.parseLong(val.toString());
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
                 }
-                break;
+            } else if (val instanceof java.sql.Date || val instanceof Date) {
-            case "short":
+                DateTime dateTime;
-                if (val instanceof Number) {
+                if (val instanceof java.sql.Date) {
-                    res = ((Number) val).shortValue();
+                    dateTime = new DateTime(((java.sql.Date) val).getTime());
                 } else {
-                    res = Short.parseShort(val.toString());
+                    dateTime = new DateTime(((Date) val).getTime());
                 }
-                break;
+                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
-            case "byte":
+                    && dateTime.getMillisOfSecond() == 0) {
-                if (val instanceof Number) {
+                    res = dateTime.toString("yyyy-MM-dd");
-                    res = ((Number) val).byteValue();
                 } else {
-                    res = Byte.parseByte(val.toString());
-                }
-                break;
-            case "double":
-                if (val instanceof Number) {
-                    res = ((Number) val).doubleValue();
-                } else {
-                    res = Double.parseDouble(val.toString());
-                }
-                break;
-            case "float":
-            case "half_float":
-            case "scaled_float":
-                if (val instanceof Number) {
-                    res = ((Number) val).floatValue();
-                } else {
-                    res = Float.parseFloat(val.toString());
-                }
-                break;
-            case "boolean":
-                if (val instanceof Boolean) {
-                    res = val;
-                } else if (val instanceof Number) {
-                    int v = ((Number) val).intValue();
-                    res = v != 0;
-                } else {
-                    res = Boolean.parseBoolean(val.toString());
-                }
-                break;
-            case "date":
-                if (val instanceof java.sql.Time) {
-                    DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
-                    if (dateTime.getMillisOfSecond() != 0) {
-                        res = dateTime.toString("HH:mm:ss.SSS");
-                    } else {
-                        res = dateTime.toString("HH:mm:ss");
-                    }
-                } else if (val instanceof java.sql.Timestamp) {
-                    DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
                     if (dateTime.getMillisOfSecond() != 0) {
                         res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
                     } else {
                         res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
                     }
-                } else if (val instanceof java.sql.Date || val instanceof Date) {
+                }
-                    DateTime dateTime;
+            } else if (val instanceof Long) {
-                    if (val instanceof java.sql.Date) {
+                DateTime dateTime = new DateTime(((Long) val).longValue());
-                        dateTime = new DateTime(((java.sql.Date) val).getTime());
+                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
-                    } else {
+                    && dateTime.getMillisOfSecond() == 0) {
-                        dateTime = new DateTime(((Date) val).getTime());
+                    res = dateTime.toString("yyyy-MM-dd");
-                    }
+                } else if (dateTime.getMillisOfSecond() != 0) {
-                    if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                            && dateTime.getMillisOfSecond() == 0) {
+                } else {
-                        res = dateTime.toString("yyyy-MM-dd");
+                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                    } else {
+                }
+            } else if (val instanceof String) {
+                String v = ((String) val).trim();
+                if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
+                    && v.charAt(13) == ':' && v.charAt(16) == ':') {
+                    String dt = v.substring(0, 10) + "T" + v.substring(11);
+                    Date date = Util.parseDate(dt);
+                    if (date != null) {
+                        DateTime dateTime = new DateTime(date);
                         if (dateTime.getMillisOfSecond() != 0) {
                             res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
                         } else {
                             res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
                         }
                     }
-                } else if (val instanceof Long) {
+                } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
-                    DateTime dateTime = new DateTime(((Long) val).longValue());
+                    Date date = Util.parseDate(v);
-                    if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
+                    if (date != null) {
-                            && dateTime.getMillisOfSecond() == 0) {
+                        DateTime dateTime = new DateTime(date);
                         res = dateTime.toString("yyyy-MM-dd");
-                    } else if (dateTime.getMillisOfSecond() != 0) {
-                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                    } else {
-                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                    }
-                } else if (val instanceof String) {
-                    String v = ((String) val).trim();
-                    if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
-                            && v.charAt(13) == ':' && v.charAt(16) == ':') {
-                        String dt = v.substring(0, 10) + "T" + v.substring(11);
-                        Date date = Util.parseDate(dt);
-                        if (date != null) {
-                            DateTime dateTime = new DateTime(date);
-                            if (dateTime.getMillisOfSecond() != 0) {
-                                res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                            } else {
-                                res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                            }
-                        }
-                    } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
-                        Date date = Util.parseDate(v);
-                        if (date != null) {
-                            DateTime dateTime = new DateTime(date);
-                            res = dateTime.toString("yyyy-MM-dd");
-                        }
                     }
                 }
-                break;
+            }
-            case "binary":
+        } else if ("binary".equals(esType)) {
-                if (val instanceof byte[]) {
+            if (val instanceof byte[]) {
-                    Base64 base64 = new Base64();
+                Base64 base64 = new Base64();
-                    res = base64.encodeAsString((byte[]) val);
+                res = base64.encodeAsString((byte[]) val);
-                } else if (val instanceof Blob) {
+            } else if (val instanceof Blob) {
-                    byte[] b = blobToBytes((Blob) val);
+                byte[] b = blobToBytes((Blob) val);
-                    Base64 base64 = new Base64();
+                Base64 base64 = new Base64();
-                    res = base64.encodeAsString(b);
+                res = base64.encodeAsString(b);
-                } else if (val instanceof String) {
+            } else if (val instanceof String) {
-                    // 对应canal中的单字节编码
+                // 对应canal中的单字节编码
-                    byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
+                byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
-                    Base64 base64 = new Base64();
+                Base64 base64 = new Base64();
-                    res = base64.encodeAsString(b);
+                res = base64.encodeAsString(b);
-                }
+            }
-                break;
+        } else if ("geo_point".equals(esType)) {
-            case "geo_point":
+            if (!(val instanceof String)) {
-                if (!(val instanceof String)) {
+                logger.error("es type is geo_point, but source type is not String");
-                    logger.error("es type is geo_point, but source type is not String");
+                return val;
-                    return val;
+            }
-                }
 
-                if (!((String) val).contains(",")) {
+            if (!((String) val).contains(",")) {
-                    logger.error("es type is geo_point, source value not contains ',' separator");
+                logger.error("es type is geo_point, source value not contains ',' separator");
-                    return val;
+                return val;
-                }
+            }
 
-                String[] point = ((String) val).split(",");
+            String[] point = ((String) val).split(",");
-                Map<String, Double> location = new HashMap<>();
+            Map<String, Double> location = new HashMap<>();
-                location.put("lat", Double.valueOf(point[0].trim()));
+            location.put("lat", Double.valueOf(point[0].trim()));
-                location.put("lon", Double.valueOf(point[1].trim()));
+            location.put("lon", Double.valueOf(point[1].trim()));
-                return location;
+            return location;
-            case "array":
+        } else if ("array".equals(esType)) {
-                if ("".equals(val.toString().trim())) {
+            if ("".equals(val.toString().trim())) {
-                    res = new ArrayList<>();
+                res = new ArrayList<>();
-                } else {
+            } else {
-                    String value = val.toString();
+                String value = val.toString();
-                    String separator = ",";
+                String separator = ",";
-                    if (!value.contains(",")) {
+                if (!value.contains(",")) {
-                        if (value.contains(";")) {
+                    if (value.contains(";")) {
-                            separator = ";";
+                        separator = ";";
-                        } else if (value.contains("|")) {
+                    } else if (value.contains("|")) {
-                            separator = "\\|";
+                        separator = "\\|";
-                        } else if (value.contains("-")) {
+                    } else if (value.contains("-")) {
-                            separator = "-";
+                        separator = "-";
-                        }
                     }
-                    String[] values = value.split(separator);
-                    return Arrays.asList(values);
                 }
-                break;
+                String[] values = value.split(separator);
-            case "object":
+                return Arrays.asList(values);
-                if ("".equals(val.toString().trim())) {
+            }
-                    res = new HashMap<>();
+        } else if ("object".equals(esType)) {
-                } else {
+            if ("".equals(val.toString().trim())) {
-                    res = JSON.parseObject(val.toString(), Map.class);
+                res = new HashMap<>();
-                }
+            } else {
-                break;
+                res = JSON.parseObject(val.toString(), Map.class);
-            default:
+            }
-                // 其他类全以字符串处理
+        } else {
-                res = val.toString();
+            // 其他类全以字符串处理
-                break;
+            res = val.toString();
         }
 
         return res;

client-adapter/hbase/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -20,7 +20,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
-            <artifactId>hbase-shaded-client</artifactId>
+            <artifactId>hbase-client</artifactId>
             <version>1.1.2</version>
             <exclusions>
                 <exclusion>

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/monitor/HbaseConfigMonitor.java

@@ -1,10 +1,9 @@
 package com.alibaba.otter.canal.client.adapter.hbase.monitor;
 
-import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import java.io.File;
-import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter;
+import java.util.HashMap;
-import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
+import java.util.Map;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import java.util.Properties;
-import com.alibaba.otter.canal.client.adapter.support.Util;
 
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
@@ -14,10 +13,11 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
+import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
-import java.util.HashMap;
+import com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter;
-import java.util.Map;
+import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
-import java.util.Properties;
+import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
+import com.alibaba.otter.canal.client.adapter.support.Util;
 
 public class HbaseConfigMonitor {
 
@@ -125,8 +125,8 @@ public class HbaseConfigMonitor {
         private void addConfigToCache(File file, MappingConfig config) {
             hbaseAdapter.getHbaseMapping().put(file.getName(), config);
             Map<String, MappingConfig> configMap = hbaseAdapter.getMappingConfigCache()
-                .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_"
+                .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "."
-                                 + config.getHbaseMapping().getDatabase() + "-" + config.getHbaseMapping().getTable(),
+                                 + config.getHbaseMapping().getDatabase() + "." + config.getHbaseMapping().getTable(),
                     k1 -> new HashMap<>());
             configMap.put(file.getName(), config);
         }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java

@@ -5,24 +5,19 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.sql.DataSource;
 
+import com.alibaba.otter.canal.client.adapter.support.*;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
-import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
+import com.alibaba.otter.canal.client.adapter.hbase.support.*;
-import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
-import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
-import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
-import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
-import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
-import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
-import com.alibaba.otter.canal.client.adapter.support.Util;
 import com.google.common.base.Joiner;
 
 /**
@@ -84,8 +79,7 @@ public class HbaseEtlService extends AbstractEtlService {
             createTable();
 
             // 拼接sql
-            String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable()
+            String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable() + "`";
-                         + "`";
 
             return super.importData(sql, params);
         } catch (Exception e) {
@@ -126,146 +120,146 @@ public class HbaseEtlService extends AbstractEtlService {
 
                         if (rowKeyColumns != null) {
                             // 取rowKey字段拼接
-                StringBuilder rowKeyVale = new StringBuilder();
+                            StringBuilder rowKeyVale = new StringBuilder();
-                for (String rowKeyColumnName : rowKeyColumns) {
+                            for (String rowKeyColumnName : rowKeyColumns) {
-                    Object obj = rs.getObject(rowKeyColumnName);
+                                Object obj = rs.getObject(rowKeyColumnName);
-                    if (obj != null) {
+                                if (obj != null) {
-                        rowKeyVale.append(obj.toString());
+                                    rowKeyVale.append(obj.toString());
-                    }
-                    rowKeyVale.append("|");
-                }
-                int len = rowKeyVale.length();
-                if (len > 0) {
-                    rowKeyVale.delete(len - 1, len);
-                }
-                row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
-            }
-
-            for (int j = 1; j <= cc; j++) {
-                String columnName = rs.getMetaData().getColumnName(j);
-
-                Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
-                if (val == null) {
-                    continue;
-                }
-
-                MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
-                // 没有配置映射
-                if (columnItem == null) {
-                    String family = hbaseMapping.getFamily();
-                    String qualifile = columnName;
-                    if (hbaseMapping.isUppercaseQualifier()) {
-                        qualifile = qualifile.toUpperCase();
-                    }
-                    if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
-                        if (hbaseMapping.getRowKey() == null && j == 1) {
-                            row.setRowKey(Bytes.toBytes(val.toString()));
-                        } else {
-                            row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
-                        }
-                    } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
-                        Type type = Type.getType(classes[j - 1]);
-                        if (hbaseMapping.getRowKey() == null && j == 1) {
-                            row.setRowKey(TypeUtil.toBytes(val, type));
-                        } else {
-                            row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
-                        }
-                    } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
-                        PhType phType = PhType.getType(classes[j - 1]);
-                        if (hbaseMapping.getRowKey() == null && j == 1) {
-                            row.setRowKey(PhTypeUtil.toBytes(val, phType));
-                        } else {
-                            row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
-                        }
-                    }
-                } else {
-                    // 如果不需要类型转换
-                    if (columnItem.getType() == null || "".equals(columnItem.getType())) {
-                        if (val instanceof java.sql.Date) {
-                            SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
-                            val = dateFmt.format((Date) val);
-                        } else if (val instanceof Timestamp) {
-                            SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-                            val = datetimeFmt.format((Date) val);
-                        }
-
-                        byte[] valBytes = Bytes.toBytes(val.toString());
-                        if (columnItem.isRowKey()) {
-                            if (columnItem.getRowKeyLen() != null) {
-                                valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
-                                row.setRowKey(valBytes);
-                            } else {
-                                row.setRowKey(valBytes);
-                            }
-                        } else {
-                            row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
-                        }
-                    } else {
-                        if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
-                            byte[] valBytes = Bytes.toBytes(val.toString());
-                            if (columnItem.isRowKey()) {
-                                if (columnItem.getRowKeyLen() != null) {
-                                    valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
                                 }
-                                row.setRowKey(valBytes);
+                                rowKeyVale.append("|");
-                            } else {
-                                row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
                             }
-                        } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
+                            int len = rowKeyVale.length();
-                            Type type = Type.getType(columnItem.getType());
+                            if (len > 0) {
-                            if (columnItem.isRowKey()) {
+                                rowKeyVale.delete(len - 1, len);
-                                if (columnItem.getRowKeyLen() != null) {
+                            }
-                                    String v = limitLenNum(columnItem.getRowKeyLen(), val);
+                            row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
-                                    row.setRowKey(Bytes.toBytes(v));
+                        }
+
+                        for (int j = 1; j <= cc; j++) {
+                            String columnName = rs.getMetaData().getColumnName(j);
+
+                            Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
+                            if (val == null) {
+                                continue;
+                            }
+
+                            MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
+                            // 没有配置映射
+                            if (columnItem == null) {
+                                String family = hbaseMapping.getFamily();
+                                String qualifile = columnName;
+                                if (hbaseMapping.isUppercaseQualifier()) {
+                                    qualifile = qualifile.toUpperCase();
+                                }
+                                if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
+                                    if (hbaseMapping.getRowKey() == null && j == 1) {
+                                        row.setRowKey(Bytes.toBytes(val.toString()));
+                                    } else {
+                                        row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
+                                    }
+                                } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
+                                    Type type = Type.getType(classes[j - 1]);
+                                    if (hbaseMapping.getRowKey() == null && j == 1) {
+                                        row.setRowKey(TypeUtil.toBytes(val, type));
+                                    } else {
+                                        row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
+                                    }
+                                } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
+                                    PhType phType = PhType.getType(classes[j - 1]);
+                                    if (hbaseMapping.getRowKey() == null && j == 1) {
+                                        row.setRowKey(PhTypeUtil.toBytes(val, phType));
+                                    } else {
+                                        row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
+                                    }
+                                }
+                            } else {
+                                // 如果不需要类型转换
+                                if (columnItem.getType() == null || "".equals(columnItem.getType())) {
+                                    if (val instanceof java.sql.Date) {
+                                        SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
+                                        val = dateFmt.format((Date) val);
+                                    } else if (val instanceof Timestamp) {
+                                        SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+                                        val = datetimeFmt.format((Date) val);
+                                    }
+
+                                    byte[] valBytes = Bytes.toBytes(val.toString());
+                                    if (columnItem.isRowKey()) {
+                                        if (columnItem.getRowKeyLen() != null) {
+                                            valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
+                                            row.setRowKey(valBytes);
+                                        } else {
+                                            row.setRowKey(valBytes);
+                                        }
+                                    } else {
+                                        row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
+                                    }
                                 } else {
-                                    row.setRowKey(TypeUtil.toBytes(val, type));
+                                    if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
+                                        byte[] valBytes = Bytes.toBytes(val.toString());
+                                        if (columnItem.isRowKey()) {
+                                            if (columnItem.getRowKeyLen() != null) {
+                                                valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
+                                            }
+                                            row.setRowKey(valBytes);
+                                        } else {
+                                            row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
+                                        }
+                                    } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
+                                        Type type = Type.getType(columnItem.getType());
+                                        if (columnItem.isRowKey()) {
+                                            if (columnItem.getRowKeyLen() != null) {
+                                                String v = limitLenNum(columnItem.getRowKeyLen(), val);
+                                                row.setRowKey(Bytes.toBytes(v));
+                                            } else {
+                                                row.setRowKey(TypeUtil.toBytes(val, type));
+                                            }
+                                        } else {
+                                            row.addCell(columnItem.getFamily(),
+                                                columnItem.getQualifier(),
+                                                TypeUtil.toBytes(val, type));
+                                        }
+                                    } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
+                                        PhType phType = PhType.getType(columnItem.getType());
+                                        if (columnItem.isRowKey()) {
+                                            row.setRowKey(PhTypeUtil.toBytes(val, phType));
+                                        } else {
+                                            row.addCell(columnItem.getFamily(),
+                                                columnItem.getQualifier(),
+                                                PhTypeUtil.toBytes(val, phType));
+                                        }
+                                    }
                                 }
-                            } else {
-                                row.addCell(columnItem.getFamily(),
-                                    columnItem.getQualifier(),
-                                    TypeUtil.toBytes(val, type));
-                            }
-                        } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
-                            PhType phType = PhType.getType(columnItem.getType());
-                            if (columnItem.isRowKey()) {
-                                row.setRowKey(PhTypeUtil.toBytes(val, phType));
-                            } else {
-                                row.addCell(columnItem.getFamily(),
-                                    columnItem.getQualifier(),
-                                    PhTypeUtil.toBytes(val, phType));
                             }
                         }
+
+                        if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
+
+                        rows.add(row);
+                        complete = false;
+                        if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
+                            hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
+                            rows.clear();
+                            complete = true;
+                        }
+                        i++;
+                        impCount.incrementAndGet();
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("successful import count:" + impCount.get());
+                        }
                     }
+
+                    if (!complete && !rows.isEmpty()) {
+                        hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
+                    }
+
+                } catch (Exception e) {
+                    logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
+                    errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
+                    // throw new RuntimeException(e);
                 }
-            }
+                return i;
-
+            });
-            if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
-
-            rows.add(row);
-            complete = false;
-            if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
-                hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
-                rows.clear();
-                complete = true;
-            }
-            i++;
-            impCount.incrementAndGet();
-            if (logger.isDebugEnabled()) {
-                logger.debug("successful import count:" + impCount.get());
-            }
-        }
-
-        if (!complete && !rows.isEmpty()) {
-            hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
-        }
-
-    } catch (Exception e) {
-        logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
-        errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
-        // throw new RuntimeException(e);
-    }
-    return i;
-}           );
             return true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);

client-adapter/kudu/pom.xml

@@ -1,85 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>canal.client-adapter</artifactId>
-        <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>com.alibaba.otter</groupId>
-    <artifactId>client-adapter.kudu</artifactId>
-    <packaging>jar</packaging>
-    <name>canal client adapter kudu module for otter ${project.version}</name>
-    <dependencies>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>client-adapter.common</artifactId>
-            <version>${project.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kudu</groupId>
-            <artifactId>kudu-client</artifactId>
-            <version>1.6.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-            <version>5.1.48</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <configuration>
-                            <tasks>
-                                <copy todir="${project.basedir}/../launcher/target/classes/kudu" overwrite="true">
-                                    <fileset dir="${project.basedir}/target/classes/kudu" erroronmissingdir="true">
-                                        <include name="*.yml" />
-                                    </fileset>
-                                </copy>
-                            </tasks>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/KuduAdapter.java

@@ -1,206 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
-import com.alibaba.otter.canal.client.adapter.kudu.monitor.KuduConfigMonitor;
-import com.alibaba.otter.canal.client.adapter.kudu.service.KuduEtlService;
-import com.alibaba.otter.canal.client.adapter.kudu.service.KuduSyncService;
-import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.SPI;
-
-/**
- * @author liuyadong
- * @description kudu适配器主类
- */
-@SPI("kudu")
-public class KuduAdapter implements OuterAdapter {
-
-    private static Logger                               logger             = LoggerFactory.getLogger(KuduAdapter.class);
-
-    private Map<String, KuduMappingConfig>              kuduMapping        = new ConcurrentHashMap<>();                 // 文件名对应配置
-    private Map<String, Map<String, KuduMappingConfig>> mappingConfigCache = new ConcurrentHashMap<>();                 // 库名-表名对应配置
-
-    private String                                      dataSourceKey;
-
-    private KuduTemplate                                kuduTemplate;
-
-    private KuduSyncService                             kuduSyncService;
-
-    private KuduConfigMonitor                           kuduConfigMonitor;
-
-    private Properties                                  envProperties;
-
-    public Map<String, KuduMappingConfig> getKuduMapping() {
-        return kuduMapping;
-    }
-
-    public Map<String, Map<String, KuduMappingConfig>> getMappingConfigCache() {
-        return mappingConfigCache;
-    }
-
-    @Override
-    public void init(OuterAdapterConfig configuration, Properties envProperties) {
-        this.envProperties = envProperties;
-        Map<String, KuduMappingConfig> kuduMappingTmp = KuduMappingConfigLoader.load(envProperties);
-        // 过滤不匹配的key的配置,获取连接key，key为配置文件名称
-        kuduMappingTmp.forEach((key, mappingConfig) -> {
-            if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
-                || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey()
-                    .equalsIgnoreCase(configuration.getKey()))) {
-                kuduMapping.put(key, mappingConfig);
-                dataSourceKey = mappingConfig.getDataSourceKey();
-            }
-        });
-        // 判断目标字段是否为空
-        if (kuduMapping.isEmpty()) {
-            throw new RuntimeException("No kudu adapter found for config key: " + configuration.getKey());
-        }
-        for (Map.Entry<String, KuduMappingConfig> entry : kuduMapping.entrySet()) {
-            String configName = entry.getKey();
-            KuduMappingConfig mappingConfig = entry.getValue();
-            String k;
-            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
-                    + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
-                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
-            } else {
-                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
-                    + mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
-            }
-            Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
-                k1 -> new ConcurrentHashMap<>());
-            configMap.put(configName, mappingConfig);
-        }
-        Map<String, String> properties = configuration.getProperties();
-
-        String kudu_master = properties.get("kudu.master.address");
-        kuduTemplate = new KuduTemplate(kudu_master);
-        kuduSyncService = new KuduSyncService(kuduTemplate);
-
-        kuduConfigMonitor = new KuduConfigMonitor();
-        kuduConfigMonitor.init(this, envProperties);
-    }
-
-    @Override
-    public void sync(List<Dml> dmls) {
-        if (dmls == null || dmls.isEmpty()) {
-            return;
-        }
-        for (Dml dml : dmls) {
-            if (dml == null) {
-                return;
-            }
-            String destination = StringUtils.trimToEmpty(dml.getDestination());
-            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
-            String database = dml.getDatabase();
-            String table = dml.getTable();
-            Map<String, KuduMappingConfig> configMap;
-            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table);
-            } else {
-                configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
-            }
-            if (configMap != null) {
-                List<KuduMappingConfig> configs = new ArrayList<>();
-                configMap.values().forEach(config -> {
-                    if (StringUtils.isNotEmpty(config.getGroupId())) {
-                        if (config.getGroupId().equals(dml.getGroupId())) {
-                            configs.add(config);
-                        }
-                    } else {
-                        configs.add(config);
-                    }
-                });
-                if (!configs.isEmpty()) {
-                    configs.forEach(config -> kuduSyncService.sync(config, dml));
-                } else {
-                    logger.error("groupID didn't mach,please check your gruopId ");
-                }
-            } else {
-                logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-"
-                                                                                + table);
-            }
-        }
-    }
-
-    @Override
-    public void destroy() {
-        if (kuduConfigMonitor != null) {
-            kuduConfigMonitor.destroy();
-        }
-        // 加入kudu client 关闭钩子
-        kuduTemplate.closeKuduClient();
-    }
-
-    @Override
-    public EtlResult etl(String task, List<String> params) {
-        EtlResult etlResult = new EtlResult();
-        KuduMappingConfig config = kuduMapping.get(task);
-        KuduEtlService hbaseEtlService = new KuduEtlService(kuduTemplate, config);
-        if (config != null) {
-            return hbaseEtlService.importData(params);
-        } else {
-            StringBuilder resultMsg = new StringBuilder();
-            boolean resSucc = true;
-            for (KuduMappingConfig configTmp : kuduMapping.values()) {
-                // 取所有的destination为task的配置
-                if (configTmp.getDestination().equals(task)) {
-                    EtlResult etlRes = hbaseEtlService.importData(params);
-                    if (!etlRes.getSucceeded()) {
-                        resSucc = false;
-                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
-                    } else {
-                        resultMsg.append(etlRes.getResultMessage()).append("\n");
-                    }
-                }
-            }
-            if (resultMsg.length() > 0) {
-                etlResult.setSucceeded(resSucc);
-                if (resSucc) {
-                    etlResult.setResultMessage(resultMsg.toString());
-                } else {
-                    etlResult.setErrorMessage(resultMsg.toString());
-                }
-                return etlResult;
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Map<String, Object> count(String task) {
-        Map<String, Object> res = new LinkedHashMap<>();
-        KuduMappingConfig config = kuduMapping.get(task);
-        if (config != null && config.getKuduMapping() != null) {
-            String tableName = config.getKuduMapping().getTargetTable();
-            long rowCount = kuduTemplate.countRow(tableName);
-            res.put("kuduTable", tableName);
-            res.put("count", rowCount);
-        }
-        return res;
-    }
-
-    @Override
-    public String getDestination(String task) {
-        KuduMappingConfig config = kuduMapping.get(task);
-        if (config != null && config.getKuduMapping() != null) {
-            return config.getDestination();
-        }
-        return null;
-    }
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfig.java

@@ -1,200 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.config;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-
-/**
- * @author liuyadong
- * @description kudu配置文件映射
- */
-public class KuduMappingConfig implements AdapterConfig {
-
-    private String      dataSourceKey;     // 数据源key
-
-    private String      destination;       // canal实例或MQ的topic
-
-    private String      groupId;           // groupId
-
-    private String      outerAdapterKey;   // 对应适配器的key
-
-    private boolean     concurrent = false; // 是否并行同步
-
-    private KuduMapping kuduMapping;       // db映射配置
-
-    @Override
-    public String getDataSourceKey() {
-        return dataSourceKey;
-    }
-
-    @Override
-    public AdapterMapping getMapping() {
-        return kuduMapping;
-    }
-
-    public void setDataSourceKey(String dataSourceKey) {
-        this.dataSourceKey = dataSourceKey;
-    }
-
-    public String getGroupId() {
-        return groupId;
-    }
-
-    public void setGroupId(String groupId) {
-        this.groupId = groupId;
-    }
-
-    public String getOuterAdapterKey() {
-        return outerAdapterKey;
-    }
-
-    public void setOuterAdapterKey(String outerAdapterKey) {
-        this.outerAdapterKey = outerAdapterKey;
-    }
-
-    public boolean getConcurrent() {
-        return concurrent;
-    }
-
-    public void setConcurrent(boolean concurrent) {
-        this.concurrent = concurrent;
-    }
-
-    public KuduMapping getKuduMapping() {
-        return kuduMapping;
-    }
-
-    public void setKuduMapping(KuduMapping kuduMapping) {
-        this.kuduMapping = kuduMapping;
-    }
-
-    public String getDestination() {
-        return destination;
-    }
-
-    public void setDestination(String destination) {
-        this.destination = destination;
-    }
-
-    public void validate() {
-        if (kuduMapping.database == null || kuduMapping.database.isEmpty()) {
-            throw new NullPointerException("KuduMapping.database");
-        }
-    }
-
-    public static class KuduMapping implements AdapterMapping {
-
-        private String              database;                           // 数据库名或schema名
-        private String              table;                              // 表名
-        private Map<String, String> targetPk    = new LinkedHashMap<>(); // 目标表主键字段
-        private boolean             mapAll      = false;                // 映射所有字段
-        private String              targetDb;                           // 目标库名
-        private String              targetTable;                        // 目标表名
-        private Map<String, String> targetColumns;                      // 目标表字段映射
-
-        private String              etlCondition;                       // etl条件sql
-
-        private int                 readBatch   = 5000;
-        private int                 commitBatch = 5000;                 // etl等批量提交大小
-
-        private Map<String, String> allMapColumns;
-
-        public String getDatabase() {
-            return database;
-        }
-
-        public void setDatabase(String database) {
-            this.database = database;
-        }
-
-        public String getTable() {
-            return table;
-        }
-
-        public void setTable(String table) {
-            this.table = table;
-        }
-
-        public Map<String, String> getTargetPk() {
-            return targetPk;
-        }
-
-        public void setTargetPk(Map<String, String> targetPk) {
-            this.targetPk = targetPk;
-        }
-
-        public Boolean getMapAll() {
-            return mapAll;
-        }
-
-        public void setMapAll(Boolean mapAll) {
-            this.mapAll = mapAll;
-        }
-
-        public String getTargetDb() {
-            return targetDb;
-        }
-
-        public void setTargetDb(String targetDb) {
-            this.targetDb = targetDb;
-        }
-
-        public String getTargetTable() {
-            return targetTable;
-        }
-
-        public void setTargetTable(String targetTable) {
-            this.targetTable = targetTable;
-        }
-
-        public Map<String, String> getTargetColumns() {
-            if (targetColumns != null) {
-                targetColumns.forEach((key, value) -> {
-                    if (StringUtils.isEmpty(value)) {
-                        targetColumns.put(key, key);
-                    }
-                });
-            }
-            return targetColumns;
-        }
-
-        public void setTargetColumns(Map<String, String> targetColumns) {
-            this.targetColumns = targetColumns;
-        }
-
-        public String getEtlCondition() {
-            return etlCondition;
-        }
-
-        public void setEtlCondition(String etlCondition) {
-            this.etlCondition = etlCondition;
-        }
-
-        public int getReadBatch() {
-            return readBatch;
-        }
-
-        public void setReadBatch(int readBatch) {
-            this.readBatch = readBatch;
-        }
-
-        public int getCommitBatch() {
-            return commitBatch;
-        }
-
-        public void setCommitBatch(int commitBatch) {
-            this.commitBatch = commitBatch;
-        }
-
-        public Map<String, String> getAllMapColumns() {
-            return allMapColumns;
-        }
-
-        public void setAllMapColumns(Map<String, String> allMapColumns) {
-            this.allMapColumns = allMapColumns;
-        }
-    }
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/config/KuduMappingConfigLoader.java

@@ -1,52 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.config;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-
-/**
- * @author liuyadong
- * @description kudu表信息加载
- */
-public class KuduMappingConfigLoader {
-
-    private static Logger logger = LoggerFactory.getLogger(KuduMappingConfigLoader.class);
-
-    /**
-     * 加载HBase表映射配置
-     *
-     * @return 配置名/配置文件名--对象
-     */
-    public static Map<String, KuduMappingConfig> load(Properties envProperties) {
-        logger.info("## Start loading kudu mapping config ... ");
-
-        Map<String, KuduMappingConfig> result = new LinkedHashMap<>();
-
-        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
-        configContentMap.forEach((fileName, content) -> {
-            KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
-                content,
-                KuduMappingConfig.class,
-                null,
-                envProperties);
-            if (config == null) {
-                return;
-            }
-            try {
-                config.validate();
-            } catch (Exception e) {
-                throw new RuntimeException("ERROR load Config: " + fileName + " " + e.getMessage(), e);
-            }
-            result.put(fileName, config);
-        });
-
-        logger.info("## kudu mapping config loaded");
-        return result;
-    }
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/monitor/KuduConfigMonitor.java

@@ -1,170 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.monitor;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
-import org.apache.commons.io.monitor.FileAlterationMonitor;
-import org.apache.commons.io.monitor.FileAlterationObserver;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
-import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * @author liuyadong
- * @description 配置文件监听
- */
-public class KuduConfigMonitor {
-
-    private static final Logger   logger      = LoggerFactory.getLogger(KuduConfigMonitor.class);
-
-    private static final String   adapterName = "kudu";
-
-    private KuduAdapter           kuduAdapter;
-
-    private Properties            envProperties;
-
-    private FileAlterationMonitor fileMonitor;
-
-    public void init(KuduAdapter kuduAdapter, Properties envProperties) {
-        this.kuduAdapter = kuduAdapter;
-        this.envProperties = envProperties;
-        File confDir = Util.getConfDirPath(adapterName);
-        try {
-            FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
-            FileListener listener = new FileListener();
-            observer.addListener(listener);
-            fileMonitor = new FileAlterationMonitor(3000, observer);
-            fileMonitor.start();
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    /**
-     * 停止监听配置文件
-     */
-    public void destroy() {
-        try {
-            fileMonitor.stop();
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    /**
-     * 配置文件监听
-     */
-    private class FileListener extends FileAlterationListenerAdaptor {
-
-        @Override
-        public void onFileCreate(File file) {
-            super.onFileCreate(file);
-
-            try {
-                // 加载新增的配置文件
-                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
-                    configContent,
-                    KuduMappingConfig.class,
-                    null,
-                    envProperties);
-                if (config == null) {
-                    return;
-                }
-                config.validate();
-                addConfigToCache(file, config);
-
-                logger.info("Add a new kudu mapping config: {} to canal adapter", file.getName());
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        @Override
-        public void onFileChange(File file) {
-            super.onFileChange(file);
-
-            try {
-                if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
-                    // 加载配置文件
-                    String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
-                                                                           + file.getName());
-                    if (configContent == null) {
-                        onFileDelete(file);
-                        return;
-                    }
-                    KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
-                        configContent,
-                        KuduMappingConfig.class,
-                        null,
-                        envProperties);
-                    if (config == null) {
-                        return;
-                    }
-                    config.validate();
-                    if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
-                        deleteConfigFromCache(file);
-                    }
-                    addConfigToCache(file, config);
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        @Override
-        public void onFileDelete(File file) {
-            super.onFileDelete(file);
-
-            try {
-                if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
-                    deleteConfigFromCache(file);
-                    logger.info("Delete a hbase mapping config: {} of canal adapter", file.getName());
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        /**
-         * 添加配置文件信息到缓存
-         *
-         * @param file
-         * @param config
-         */
-        private void addConfigToCache(File file, KuduMappingConfig config) {
-            kuduAdapter.getKuduMapping().put(file.getName(), config);
-            Map<String, KuduMappingConfig> configMap = kuduAdapter.getMappingConfigCache()
-                .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "."
-                                 + config.getKuduMapping().getDatabase() + "." + config.getKuduMapping().getTable(),
-                    k1 -> new HashMap<>());
-            configMap.put(file.getName(), config);
-        }
-
-        /**
-         * 从缓存中删除配置
-         *
-         * @param file 文件
-         */
-        private void deleteConfigFromCache(File file) {
-            kuduAdapter.getKuduMapping().remove(file.getName());
-            for (Map<String, KuduMappingConfig> configMap : kuduAdapter.getMappingConfigCache().values()) {
-                if (configMap != null) {
-                    configMap.remove(file.getName());
-                }
-            }
-        }
-
-    }
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduEtlService.java

@@ -1,132 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.service;
-
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.sql.DataSource;
-
-import org.apache.kudu.client.KuduException;
-
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
-import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
-import com.alibaba.otter.canal.client.adapter.kudu.support.SyncUtil;
-import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.google.common.base.Joiner;
-
-/**
- * @author liuyadong
- * @description kudu 拉取历史数据
- */
-public class KuduEtlService extends AbstractEtlService {
-
-    private KuduTemplate      kuduTemplate;
-    private KuduMappingConfig config;
-
-    public KuduEtlService(KuduTemplate kuduTemplate, KuduMappingConfig config){
-        super("kudu", config);
-        this.kuduTemplate = kuduTemplate;
-        this.config = config;
-    }
-
-    public EtlResult importData(List<String> params) {
-        EtlResult etlResult = new EtlResult();
-        List<String> errMsg = new ArrayList<>();
-
-        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
-        boolean flag = kuduTemplate.tableExists(kuduMapping.getTargetTable());
-        // 表不存在，停止导入
-        if (!flag) {
-            logger.info("{} is don't hava,please check your kudu table !", kuduMapping.getTargetTable());
-            errMsg.add(kuduMapping.getTargetTable() + " is don't hava,please check your kudu table !");
-            etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
-            return etlResult;
-        }
-        logger.info("{} etl is starting!", kuduMapping.getTargetTable());
-        String sql = "SELECT * FROM " + kuduMapping.getDatabase() + "." + kuduMapping.getTable();
-        return importData(sql, params);
-    }
-
-    @Override
-    protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
-                                       AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
-        KuduMappingConfig.KuduMapping kuduMapping = (KuduMappingConfig.KuduMapping) mapping;
-        // 获取字段元数据
-        Map<String, String> columnsMap = new LinkedHashMap<>();// 需要同步的字段
-
-        try {
-            Util.sqlRS(ds, "SELECT * FROM " + SyncUtil.getDbTableName(kuduMapping) + " LIMIT 1", rs -> {
-                try {
-                    ResultSetMetaData rsd = rs.getMetaData();
-                    int columnCount = rsd.getColumnCount();
-                    List<String> columns = new ArrayList<>();
-                    for (int i = 1; i <= columnCount; i++) {
-                        columns.add(rsd.getColumnName(i).toLowerCase());
-                    }
-                    columnsMap.putAll(SyncUtil.getColumnsMap(kuduMapping, columns));
-                    return true;
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    return false;
-                }
-            });
-            // 写入数据
-            logger.info("etl select data sql is :{}", sql);
-            Util.sqlRS(ds, sql, values, rs -> {
-                int idx = 1;
-                try {
-                    List<Map<String, Object>> dataList = new ArrayList<>();
-                    while (rs.next()) {
-                        Map<String, Object> data = new HashMap<>();
-                        for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
-                            String mysqlColumnName = entry.getKey();// mysql字段名
-                            String kuduColumnName = entry.getValue();// kudu字段名
-                            if (kuduColumnName == null) {
-                                kuduColumnName = mysqlColumnName;
-                            }
-                            Object value = rs.getObject(kuduColumnName);
-                            if (value != null) {
-                                data.put(kuduColumnName, value);
-                            } else {
-                                data.put(kuduColumnName, null);
-                            }
-                        }
-                        dataList.add(data);
-                        idx++;
-                        impCount.incrementAndGet();
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("successful import count:" + impCount.get());
-                        }
-                        if (idx % kuduMapping.getCommitBatch() == 0) {
-                            kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
-                            dataList.clear();
-                        }
-                    }
-                    if (!dataList.isEmpty()) {
-                        kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
-                    }
-                    return true;
-
-                } catch (SQLException | KuduException e) {
-                    e.printStackTrace();
-                    logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
-                    errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
-                    return false;
-                }
-            });
-            return true;
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-            return false;
-        }
-
-    }
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduSyncService.java

@@ -1,209 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.service;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kudu.client.KuduException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
-import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-/**
- * @author liuyadong
- * @description kudu实时同步
- */
-public class KuduSyncService {
-
-    private static Logger logger = LoggerFactory.getLogger(KuduSyncService.class);
-
-    private KuduTemplate  kuduTemplate;
-
-    // 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
-    // private Map<String, Map<String, Integer>> columnsTypeCache = new
-    // ConcurrentHashMap<>();
-
-    public KuduSyncService(KuduTemplate kuduTemplate){
-        this.kuduTemplate = kuduTemplate;
-    }
-
-    // public Map<String, Map<String, Integer>> getColumnsTypeCache() {
-    // return columnsTypeCache;
-    // }
-
-    /**
-     * 同步事件处理
-     *
-     * @param config
-     * @param dml
-     */
-    public void sync(KuduMappingConfig config, Dml dml) {
-        if (config != null) {
-            String type = dml.getType();
-            if (type != null && type.equalsIgnoreCase("INSERT")) {
-                insert(config, dml);
-            } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
-                upsert(config, dml);
-            } else if (type != null && type.equalsIgnoreCase("DELETE")) {
-                delete(config, dml);
-            }
-            if (logger.isDebugEnabled()) {
-                logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
-            }
-        }
-    }
-
-    /**
-     * 删除事件
-     *
-     * @param config
-     * @param dml
-     */
-    private void delete(KuduMappingConfig config, Dml dml) {
-        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
-        String configTable = kuduMapping.getTable();
-        String configDatabase = kuduMapping.getDatabase();
-        String table = dml.getTable();
-        String database = dml.getDatabase();
-        if (configTable.equals(table) && configDatabase.equals(database)) {
-            List<Map<String, Object>> data = dml.getData();
-            if (data == null || data.isEmpty()) {
-                return;
-            }
-            // 判定主键映射
-            String pkId = "";
-            Map<String, String> targetPk = kuduMapping.getTargetPk();
-            for (Map.Entry<String, String> entry : targetPk.entrySet()) {
-                String mysqlID = entry.getKey().toLowerCase();
-                String kuduID = entry.getValue();
-                if (kuduID == null) {
-                    pkId = mysqlID;
-                } else {
-                    pkId = kuduID;
-                }
-            }
-            // 切割联合主键
-            List<String> pkIds = Arrays.asList(pkId.split(","));
-            try {
-                int idx = 1;
-                boolean completed = false;
-                List<Map<String, Object>> dataList = new ArrayList<>();
-
-                for (Map<String, Object> item : data) {
-                    Map<String, Object> primaryKeyMap = new HashMap<>();
-                    for (Map.Entry<String, Object> entry : item.entrySet()) {
-                        String columnName = entry.getKey().toLowerCase();
-                        Object value = entry.getValue();
-                        if (pkIds.contains(columnName)) {
-                            primaryKeyMap.put(columnName, value);
-                        }
-                    }
-                    dataList.add(primaryKeyMap);
-                    idx++;
-                    if (idx % kuduMapping.getCommitBatch() == 0) {
-                        kuduTemplate.delete(kuduMapping.getTargetTable(), dataList);
-                        dataList.clear();
-                        completed = true;
-                    }
-                }
-                if (!completed) {
-                    kuduTemplate.delete(kuduMapping.getTargetTable(), dataList);
-                }
-            } catch (KuduException e) {
-                logger.error(e.getMessage());
-                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
-            }
-        }
-    }
-
-    /**
-     * 更新插入事件
-     *
-     * @param config
-     * @param dml
-     */
-    private void upsert(KuduMappingConfig config, Dml dml) {
-        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
-        String configTable = kuduMapping.getTable();
-        String configDatabase = kuduMapping.getDatabase();
-        String table = dml.getTable();
-        String database = dml.getDatabase();
-        if (configTable.equals(table) && configDatabase.equals(database)) {
-            List<Map<String, Object>> data = dml.getData();
-            if (data == null || data.isEmpty()) {
-                return;
-            }
-            try {
-                int idx = 1;
-                boolean completed = false;
-                List<Map<String, Object>> dataList = new ArrayList<>();
-
-                for (Map<String, Object> entry : data) {
-                    dataList.add(entry);
-                    idx++;
-                    if (idx % kuduMapping.getCommitBatch() == 0) {
-                        kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
-                        dataList.clear();
-                        completed = true;
-                    }
-                }
-                if (!completed) {
-                    kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
-                }
-            } catch (KuduException e) {
-                logger.error(e.getMessage());
-                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
-            }
-        }
-
-    }
-
-    /**
-     * 插入事件
-     *
-     * @param config
-     * @param dml
-     */
-    private void insert(KuduMappingConfig config, Dml dml) {
-        KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
-        String configTable = kuduMapping.getTable();
-        String configDatabase = kuduMapping.getDatabase();
-        String table = dml.getTable();
-        String database = dml.getDatabase();
-        if (configTable.equals(table) && configDatabase.equals(database)) {
-            List<Map<String, Object>> data = dml.getData();
-            if (data == null || data.isEmpty()) {
-                return;
-            }
-            try {
-                int idx = 1;
-                boolean completed = false;
-                List<Map<String, Object>> dataList = new ArrayList<>();
-
-                for (Map<String, Object> entry : data) {
-                    dataList.add(entry);
-                    idx++;
-                    if (idx % kuduMapping.getCommitBatch() == 0) {
-                        kuduTemplate.insert(kuduMapping.getTargetTable(), dataList);
-                        dataList.clear();
-                        completed = true;
-                    }
-                }
-                if (!completed) {
-                    kuduTemplate.insert(kuduMapping.getTargetTable(), dataList);
-                }
-            } catch (KuduException e) {
-                logger.error(e.getMessage());
-                logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
-            }
-        }
-    }
-
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/support/KuduTemplate.java

@@ -1,424 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.support;
-
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kudu.ColumnSchema;
-import org.apache.kudu.Schema;
-import org.apache.kudu.Type;
-import org.apache.kudu.client.Delete;
-import org.apache.kudu.client.Insert;
-import org.apache.kudu.client.KuduClient;
-import org.apache.kudu.client.KuduException;
-import org.apache.kudu.client.KuduScanner;
-import org.apache.kudu.client.KuduSession;
-import org.apache.kudu.client.KuduTable;
-import org.apache.kudu.client.OperationResponse;
-import org.apache.kudu.client.PartialRow;
-import org.apache.kudu.client.SessionConfiguration;
-import org.apache.kudu.client.Upsert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author liuyadong
- * @description kudu 操作工具类
- */
-public class KuduTemplate {
-
-    private Logger           logger          = LoggerFactory.getLogger(this.getClass());
-
-    private KuduClient       kuduClient;
-    private String           masters;
-
-    private final static int OPERATION_BATCH = 500;
-
-    private SimpleDateFormat sdf             = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-
-    public KuduTemplate(String master_str){
-        this.masters = master_str;
-        checkClient();
-    }
-
-    /**
-     * 检车连接
-     */
-    private void checkClient() {
-        if (kuduClient == null) {
-            // kudu master 以逗号分隔
-            List<String> masterList = Arrays.asList(masters.split(","));
-            kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
-                .defaultSocketReadTimeoutMs(60000)
-                .defaultAdminOperationTimeoutMs(60000)
-                .build();
-        }
-    }
-
-    /**
-     * 查询表是否存在
-     *
-     * @param tableName
-     * @return
-     */
-    public boolean tableExists(String tableName) {
-        this.checkClient();
-        try {
-            return kuduClient.tableExists(tableName);
-        } catch (KuduException e) {
-            logger.error("kudu table exists check fail,message :{}", e.getMessage());
-            return true;
-        }
-    }
-
-    /**
-     * 删除行
-     *
-     * @param tableName
-     * @param dataList
-     * @throws KuduException
-     */
-    public void delete(String tableName, List<Map<String, Object>> dataList) throws KuduException {
-        this.checkClient();
-        KuduTable kuduTable = kuduClient.openTable(tableName);
-        KuduSession session = kuduClient.newSession();
-        try {
-            session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
-            session.setMutationBufferSpace(OPERATION_BATCH);
-            // 获取元数据结构
-            Map<String, Type> metaMap = new HashMap<>();
-            Schema schema = kuduTable.getSchema();
-            for (ColumnSchema columnSchema : schema.getColumns()) {
-                String colName = columnSchema.getName().toLowerCase();
-                Type type = columnSchema.getType();
-                metaMap.put(colName, type);
-            }
-            int uncommit = 0;
-            for (Map<String, Object> data : dataList) {
-                Delete delete = kuduTable.newDelete();
-                PartialRow row = delete.getRow();
-                for (Map.Entry<String, Object> entry : data.entrySet()) {
-                    String name = entry.getKey().toLowerCase();
-                    Type type = metaMap.get(name);
-                    Object value = entry.getValue();
-                    fillRow(row, name, value, type); // 填充行数据
-                }
-                session.apply(delete);
-                // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
-                uncommit = uncommit + 1;
-                if (uncommit > OPERATION_BATCH / 3 * 2) {
-                    List<OperationResponse> delete_option = session.flush();
-                    if (delete_option.size() > 0) {
-                        OperationResponse response = delete_option.get(0);
-                        if (response.hasRowError()) {
-                            logger.error("delete row fail table name is :{} ", tableName);
-                            logger.error("error list is :{}", response.getRowError().getMessage());
-                        }
-                    }
-                    uncommit = 0;
-                }
-            }
-            List<OperationResponse> delete_option = session.flush();
-            if (delete_option.size() > 0) {
-                OperationResponse response = delete_option.get(0);
-                if (response.hasRowError()) {
-                    logger.error("delete row fail table name is :{} ", tableName);
-                    logger.error("error list is :{}", response.getRowError().getMessage());
-                }
-            }
-
-        } catch (KuduException e) {
-            logger.error("error message is :{}", dataList.toString());
-            throw e;
-        } finally {
-            if (!session.isClosed()) {
-                session.close();
-            }
-        }
-    }
-
-    /**
-     * 更新/插入字段
-     *
-     * @param tableName
-     * @param dataList
-     * @throws KuduException
-     */
-    public void upsert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
-        this.checkClient();
-        KuduTable kuduTable = kuduClient.openTable(tableName);
-        KuduSession session = kuduClient.newSession();
-        try {
-            session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
-            session.setMutationBufferSpace(OPERATION_BATCH);
-            // 获取元数据结构
-            Map<String, Type> metaMap = new HashMap<>();
-            Schema schema = kuduTable.getSchema();
-            for (ColumnSchema columnSchema : schema.getColumns()) {
-                String colName = columnSchema.getName().toLowerCase();
-                Type type = columnSchema.getType();
-                metaMap.put(colName, type);
-            }
-            int uncommit = 0;
-            for (Map<String, Object> data : dataList) {
-                Upsert upsert = kuduTable.newUpsert();
-                PartialRow row = upsert.getRow();
-                for (Map.Entry<String, Object> entry : data.entrySet()) {
-                    String name = entry.getKey().toLowerCase();
-                    Type type = metaMap.get(name);
-                    Object value = entry.getValue();
-                    fillRow(row, name, value, type); // 填充行数据
-                }
-                session.apply(upsert);
-                // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
-                uncommit = uncommit + 1;
-                if (uncommit > OPERATION_BATCH / 3 * 2) {
-                    List<OperationResponse> update_option = session.flush();
-                    if (update_option.size() > 0) {
-                        OperationResponse response = update_option.get(0);
-                        if (response.hasRowError()) {
-                            logger.error("update row fail table name is :{} ", tableName);
-                            logger.error("update list is :{}", response.getRowError().getMessage());
-                        }
-                    }
-                    uncommit = 0;
-                }
-            }
-            List<OperationResponse> update_option = session.flush();
-            if (update_option.size() > 0) {
-                OperationResponse response = update_option.get(0);
-                if (response.hasRowError()) {
-                    logger.error("update row fail table name is :{} ", tableName);
-                    logger.error("update list is :{}", response.getRowError().getMessage());
-                }
-            }
-        } catch (KuduException e) {
-            logger.error("error message is :{}", dataList.toString());
-            throw e;
-        } finally {
-            if (!session.isClosed()) {
-                session.close();
-            }
-        }
-
-    }
-
-    /**
-     * 插入数据
-     *
-     * @param tableName
-     * @param dataList
-     * @throws KuduException
-     */
-    public void insert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
-        this.checkClient();
-        KuduTable kuduTable = kuduClient.openTable(tableName);// 打开表
-        KuduSession session = kuduClient.newSession(); // 创建写session,kudu必须通过session写入
-        try {
-            session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // 采取Flush方式
-                                                                               // 手动刷新
-            session.setMutationBufferSpace(OPERATION_BATCH);
-            // 获取元数据结构
-            Map<String, Type> metaMap = new HashMap<>();
-            Schema schema = kuduTable.getSchema();
-            for (ColumnSchema columnSchema : schema.getColumns()) {
-                String colName = columnSchema.getName().toLowerCase();
-                Type type = columnSchema.getType();
-                metaMap.put(colName, type);
-            }
-            int uncommit = 0;
-            for (Map<String, Object> data : dataList) {
-                Insert insert = kuduTable.newInsert();
-                PartialRow row = insert.getRow();
-                for (Map.Entry<String, Object> entry : data.entrySet()) {
-                    String name = entry.getKey().toLowerCase();
-                    Type type = metaMap.get(name);
-                    Object value = entry.getValue();
-                    fillRow(row, name, value, type); // 填充行数据
-                }
-                session.apply(insert);
-                // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
-                uncommit = uncommit + 1;
-                if (uncommit > OPERATION_BATCH / 3 * 2) {
-                    List<OperationResponse> insert_option = session.flush();
-                    if (insert_option.size() > 0) {
-                        OperationResponse response = insert_option.get(0);
-                        if (response.hasRowError()) {
-                            logger.error("insert row fail table name is :{} ", tableName);
-                            logger.error("insert list is :{}", response.getRowError().getMessage());
-                        }
-                    }
-                    uncommit = 0;
-                }
-            }
-            List<OperationResponse> insert_option = session.flush();
-            if (insert_option.size() > 0) {
-                OperationResponse response = insert_option.get(0);
-                if (response.hasRowError()) {
-                    logger.error("insert row fail table name is :{} ", tableName);
-                    logger.error("insert list is :{}", response.getRowError().getMessage());
-                }
-            }
-        } catch (KuduException e) {
-            logger.error("error message is :{}", dataList.toString());
-            throw e;
-        } finally {
-            if (!session.isClosed()) {
-                session.close();
-            }
-        }
-
-    }
-
-    /**
-     * 统计kudu表数据
-     *
-     * @param tableName
-     * @return
-     */
-    public long countRow(String tableName) {
-        this.checkClient();
-        long rowCount = 0L;
-        try {
-            KuduTable kuduTable = kuduClient.openTable(tableName);
-            // 创建scanner扫描
-            KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
-            // 遍历数据
-            while (scanner.hasMoreRows()) {
-                while (scanner.nextRows().hasNext()) {
-                    rowCount++;
-                }
-            }
-        } catch (KuduException e) {
-            e.printStackTrace();
-        }
-        return rowCount;
-    }
-
-    /**
-     * 关闭钩子
-     *
-     * @throws IOException
-     */
-    public void closeKuduClient() {
-        if (kuduClient != null) {
-            try {
-                kuduClient.close();
-            } catch (Exception e) {
-                logger.error("ShutdownHook Close KuduClient Error! error message {}", e.getMessage());
-            }
-        }
-    }
-
-    /**
-     * 封装kudu行数据
-     *
-     * @param row
-     * @param rawVal
-     * @param type
-     */
-    private void fillRow(PartialRow row, String colName, Object rawVal, Type type) {
-        String rowValue = "0";
-        if (!(rawVal == null || "".equals(rawVal))) {
-            rowValue = rawVal + "";
-        } else {
-            return;
-        }
-        if (type == null) {
-            logger.warn("got unknown type {} for column '{}'-- ignoring this column", type, colName);
-            return;
-        }
-        try {
-            switch (type) {
-                case INT8:
-                    row.addByte(colName, Byte.parseByte(rowValue));
-                    break;
-                case INT16:
-                    row.addShort(colName, Short.parseShort(rowValue));
-                    break;
-                case INT32:
-                    row.addInt(colName, Integer.parseInt(rowValue));
-                    break;
-                case INT64:
-                    row.addLong(colName, Long.parseLong(rowValue));
-                    break;
-                case BINARY:
-                    row.addBinary(colName, rowValue.getBytes());
-                    break;
-                case STRING:
-                    row.addString(colName, rowValue);
-                    break;
-                case BOOL:
-                    if (!("true".equalsIgnoreCase(rowValue) || "false".equalsIgnoreCase(rowValue))) {
-                        return;
-                    }
-                    row.addBoolean(colName, Boolean.parseBoolean(rowValue));
-                    break;
-                case FLOAT:
-                    row.addFloat(colName, Float.parseFloat(rowValue));
-                    break;
-                case DOUBLE:
-                    row.addDouble(colName, Double.parseDouble(rowValue));
-                    break;
-                case UNIXTIME_MICROS:
-                    if ("0".equals(rowValue)) {
-                        try {
-                            Date parse = sdf.parse("2099-11-11 11:11:11");
-                            row.addLong(colName, parse.getTime());
-                        } catch (ParseException e) {
-                            logger.warn("date column is null");
-                        }
-                    } else {
-                        try {
-                            Date parse = rowValue.length() > 19 ? sdf.parse(rowValue.substring(0, 19)) : sdf.parse(rowValue);
-                            row.addLong(colName, parse.getTime());
-                        } catch (ParseException e) {
-                            logger.warn("date format error, error data is :{}", rowValue);
-                            try {
-                                Date parse = sdf.parse("2099-11-11 11:11:11");
-                                row.addLong(colName, parse.getTime());
-                            } catch (ParseException ie) {
-                                logger.warn("date column is null");
-                            }
-                        }
-                    }
-                    break;
-                default:
-                    logger.warn("got unknown type {} for column '{}'-- ignoring this column", type, colName);
-            }
-        } catch (NumberFormatException e) {
-            logger.error(e.getMessage());
-            logger.error("column parse fail==> column name=>{},column type=>{},column value=>{}", colName, type, rawVal);
-        }
-    }
-
-    /**
-     * kudu数据类型映射
-     *
-     * @param
-     */
-    private Type toKuduType(String mysqlType) throws IllegalArgumentException {
-
-        switch (mysqlType) {
-            case "varchar":
-                return Type.STRING;
-            case "int":
-                return Type.INT8;
-            case "decimal":
-                return Type.DOUBLE;
-            case "double":
-                return Type.DOUBLE;
-            case "datetime":
-                return Type.STRING;
-            case "timestamp":
-                return Type.STRING;
-            default:
-                throw new IllegalArgumentException("The provided data type doesn't map to know any known one.");
-        }
-    }
-}

client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/support/SyncUtil.java

@@ -1,68 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.support;
-
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
-
-/**
- * @author liuyadong
- * @description 工具
- */
-public class SyncUtil {
-
-    /**
-     * 获取源数据库 元数据结构
-     *
-     * @param kuduMapping
-     * @return
-     */
-    public static String getDbTableName(KuduMappingConfig.KuduMapping kuduMapping) {
-        String result = "";
-        if (StringUtils.isNotEmpty(kuduMapping.getTable())) {
-            result += kuduMapping.getDatabase() + ".";
-        }
-        result += kuduMapping.getTable();
-        return result;
-    }
-
-    /**
-     * 过滤转换字段
-     *
-     * @param kuduMapping
-     * @param columns
-     * @return
-     */
-    public static Map<String, String> getColumnsMap(KuduMappingConfig.KuduMapping kuduMapping, List<String> columns) {
-        Map<String, String> columnsMap;
-        if (kuduMapping.getMapAll()) {
-            if (kuduMapping.getAllMapColumns() != null) {
-                return kuduMapping.getAllMapColumns();
-            }
-            columnsMap = new LinkedHashMap<>();
-            for (String srcColumn : columns) {
-                boolean flag = true;
-                if (kuduMapping.getTargetColumns() != null) {
-                    for (Map.Entry<String, String> entry : kuduMapping.getTargetColumns().entrySet()) {
-                        if (srcColumn.equals(entry.getValue())) {
-                            columnsMap.put(entry.getKey(), srcColumn);
-                            flag = false;
-                            break;
-                        }
-                    }
-                }
-                if (flag) {
-                    columnsMap.put(srcColumn, srcColumn);
-                }
-            }
-            kuduMapping.setAllMapColumns(columnsMap);
-        } else {
-            columnsMap = kuduMapping.getTargetColumns();
-        }
-        return columnsMap;
-    }
-
-}

client-adapter/kudu/src/main/resources/META-INF.canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -1 +0,0 @@
-kudu=com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter

client-adapter/kudu/src/main/resources/kudu/kudutest_user.yml

@@ -1,19 +0,0 @@
-dataSourceKey: defaultDS
-destination: test_instance
-outerAdapterKey: kudu
-groupId: g1
-kuduMapping:
-  database: test
-  table: test
-  targetTable: test
-  targetPk:
-    id: id
-  mapAll: true
-  targetColumns:
-    id:
-    name:
-    role_id:
-    c_time:
-    test1:
-  etlCondition:
-  commitBatch: 3000 # 批量提交的大小

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/KuduConnectionTest.java

@@ -1,29 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.test;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.kudu.client.KuduClient;
-import org.apache.kudu.client.KuduException;
-import org.apache.kudu.client.KuduTable;
-import org.junit.Test;
-
-public class KuduConnectionTest {
-
-    @Test
-    public void test01() {
-        List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
-        KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
-            .defaultSocketReadTimeoutMs(30000)
-            .defaultAdminOperationTimeoutMs(60000)
-            .build();
-        try {
-            List<String> tablesList = kuduClient.getTablesList().getTablesList();
-            System.out.println(tablesList.toString());
-            KuduTable web_white_list = kuduClient.openTable("web_white_list");
-
-        } catch (KuduException e) {
-            e.printStackTrace();
-        }
-    }
-}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/TestConfig.java

@@ -1,29 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.test;
-
-import java.util.Map;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
-import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-
-/**
- * @description
- */
-public class TestConfig {
-
-    @Before
-    public void before() {
-        // 加载数据源连接池
-        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
-    }
-
-    @Test
-    public void testLoad() {
-        Map<String, KuduMappingConfig> configMap = KuduMappingConfigLoader.load(null);
-        Assert.assertFalse(configMap.isEmpty());
-    }
-}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/TestConstant.java

@@ -1,37 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.test;
-
-import java.sql.SQLException;
-
-import com.alibaba.druid.pool.DruidDataSource;
-
-public class TestConstant {
-
-    public final static String          jdbcUrl      = "jdbc:mysql://10.0.9.5:3306/canal_manager?useUnicode=true";
-    public final static String          jdbcUser     = "axtest";
-    public final static String          jdbcPassword = "axtest123";
-
-    public final static DruidDataSource dataSource;
-
-    static {
-        dataSource = new DruidDataSource();
-        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
-        dataSource.setUrl(jdbcUrl);
-        dataSource.setUsername(jdbcUser);
-        dataSource.setPassword(jdbcPassword);
-        dataSource.setInitialSize(1);
-        dataSource.setMinIdle(1);
-        dataSource.setMaxActive(1);
-        dataSource.setMaxWait(60000);
-        dataSource.setTimeBetweenEvictionRunsMillis(60000);
-        dataSource.setMinEvictableIdleTimeMillis(300000);
-        dataSource.setPoolPreparedStatements(false);
-        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
-        dataSource.setValidationQuery("select 1");
-        try {
-            dataSource.init();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-    }
-
-}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/sync/Common.java

@@ -1,30 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
-import com.alibaba.otter.canal.client.adapter.kudu.test.TestConstant;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-
-/**
- * @description
- */
-public class Common {
-
-    public static KuduAdapter init() {
-        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
-
-        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
-        outerAdapterConfig.setName("kudu");
-        outerAdapterConfig.setKey("kudu");
-        Map<String, String> properties = new HashMap<>();
-        properties.put("kudu.master.address", "10.6.36.102,10.6.36.187,10.6.36.229");
-        outerAdapterConfig.setProperties(properties);
-
-        KuduAdapter adapter = new KuduAdapter();
-        adapter.init(outerAdapterConfig, null);
-        return adapter;
-    }
-}

client-adapter/kudu/src/test/java/com/alibaba/otter/canal/client/adapter/kudu/test/sync/TestSyncKudu.java

@@ -1,59 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-/**
- * @description
- */
-public class TestSyncKudu {
-
-    private KuduAdapter kuduAdapter;
-
-    @Before
-    public void init() {
-        kuduAdapter = Common.init();
-    }
-
-    @Test
-    public void testEtl() {
-        List<String> param = new ArrayList<>();
-        kuduAdapter.etl("kudutest_user.yml", param);
-    }
-
-    @Test
-    public void testCount() {
-        kuduAdapter.count("kudutest_user.yml");
-    }
-
-    @Test
-    public void testSync() {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setGroupId("g1");
-        dml.setTs(new Date().getTime());
-        dml.setType("DELETE");
-        dml.setDatabase("test1");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "liuyadong");
-        data.put("role_id", 1L);
-        data.put("c_time", new Date());
-        dml.setData(dataList);
-
-        kuduAdapter.sync(Collections.singletonList(dml));
-    }
-}

client-adapter/launcher/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5</version>
+        <version>1.1.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -30,6 +30,11 @@
             <artifactId>client-adapter.common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.client</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
@@ -51,6 +56,25 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
         </dependency>
+        <!-- 单独引入rocketmq依赖 -->
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-client</artifactId>
+        </dependency>
+        <!-- 单独引入kafka依赖 -->
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.rabbitmq</groupId>
+            <artifactId>amqp-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.mq-amqp</groupId>
+            <artifactId>mq-amqp-client</artifactId>
+        </dependency>
         <!-- jdbc -->
         <dependency>
             <groupId>mysql</groupId>
@@ -69,12 +93,6 @@
             <artifactId>mssql-jdbc</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.hbase</groupId>
-            <artifactId>hbase-shaded-client</artifactId>
-            <scope>runtime</scope>
-        </dependency>
-
         <!-- outer adapter jar with dependencies-->
         <dependency>
             <groupId>com.alibaba.otter</groupId>
@@ -141,59 +159,6 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
-        <!-- connector plugin -->
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>connector.tcp</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <artifactId>*</artifactId>
-                    <groupId>*</groupId>
-                </exclusion>
-            </exclusions>
-            <classifier>jar-with-dependencies</classifier>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>connector.kafka</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <artifactId>*</artifactId>
-                    <groupId>*</groupId>
-                </exclusion>
-            </exclusions>
-            <classifier>jar-with-dependencies</classifier>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>connector.rocketmq</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <artifactId>*</artifactId>
-                    <groupId>*</groupId>
-                </exclusion>
-            </exclusions>
-            <classifier>jar-with-dependencies</classifier>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>connector.rabbitmq</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <artifactId>*</artifactId>
-                    <groupId>*</groupId>
-                </exclusion>
-            </exclusions>
-            <classifier>jar-with-dependencies</classifier>
-            <scope>provided</scope>
-        </dependency>
     </dependencies>
 
     <build>

client-adapter/launcher/src/main/assembly/dev.xml

@@ -50,13 +50,6 @@
                 <include>**/*</include>
             </includes>
         </fileSet>
-        <fileSet>
-            <directory>../kudu/src/main/resources/kudu</directory>
-            <outputDirectory>/conf/kudu</outputDirectory>
-            <includes>
-                <include>**/*</include>
-            </includes>
-        </fileSet>
         <fileSet>
             <directory>../rdb/src/main/resources/</directory>
             <outputDirectory>/conf</outputDirectory>

client-adapter/launcher/src/main/assembly/release.xml

@@ -51,13 +51,6 @@
                 <include>**/*</include>
             </includes>
         </fileSet>
-        <fileSet>
-            <directory>../kudu/src/main/resources/kudu</directory>
-            <outputDirectory>/conf/kudu</outputDirectory>
-            <includes>
-                <include>**/*</include>
-            </includes>
-        </fileSet>
         <fileSet>
             <directory>../rdb/src/main/resources/</directory>
             <outputDirectory>/conf</outputDirectory>

client-adapter/launcher/src/main/bin/restart.sh

@@ -1,13 +1,5 @@
 #!/bin/bash
 
-case $(uname) in
+sh stop.sh
-Linux)
-  bin_abs_path=$(readlink -f $(dirname $0))
-  ;;
-*)
-  bin_abs_path=$(cd $(dirname $0) ||exit ; pwd)
-  ;;
-esac
 
-sh "$bin_abs_path"/stop.sh
+sh startup.sh
-sh "$bin_abs_path"/startup.sh

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigHolder.java

@@ -1,10 +1,10 @@
 package com.alibaba.otter.canal.adapter.launcher.config;
 
+import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
+
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
-
 /**
  * @author L.J.R @ 2019-09-03
  * @version 1.1.4

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.adapter.launcher.config;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
@@ -25,7 +24,7 @@ public class CuratorClient {
 
     @PostConstruct
     public void init() {
-        if (StringUtils.isNotEmpty(adapterCanalConfig.getZookeeperHosts())) {
+        if (adapterCanalConfig.getZookeeperHosts() != null) {
             curator = CuratorFrameworkFactory.builder()
                 .connectString(adapterCanalConfig.getZookeeperHosts())
                 .retryPolicy(new ExponentialBackoffRetry(1000, 3))

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java

@@ -0,0 +1,263 @@
+package com.alibaba.otter.canal.adapter.launcher.loader;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
+import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
+import com.alibaba.otter.canal.client.CanalMQConnector;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * 适配器工作线程抽象类
+ *
+ * @author rewerma 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public abstract class AbstractCanalAdapterWorker {
+
+    protected final Logger                    logger  = LoggerFactory.getLogger(this.getClass());
+
+    protected String                          canalDestination;                                                // canal实例
+    protected String                          groupId = null;                                                  // groupId
+    protected List<List<OuterAdapter>>        canalOuterAdapters;                                              // 外部适配器
+    protected CanalClientConfig               canalClientConfig;                                               // 配置
+    protected ExecutorService                 groupInnerExecutorService;                                       // 组内工作线程池
+    protected volatile boolean                running = false;                                                 // 是否运行中
+    protected Thread                          thread  = null;
+    protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
+
+    protected SyncSwitch                      syncSwitch;
+
+    public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){
+        this.canalOuterAdapters = canalOuterAdapters;
+        this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
+        syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
+    }
+
+    protected void writeOut(final Message message) {
+        List<Future<Boolean>> futures = new ArrayList<>();
+        // 组间适配器并行运行
+        canalOuterAdapters.forEach(outerAdapters -> {
+            final List<OuterAdapter> adapters = outerAdapters;
+            futures.add(groupInnerExecutorService.submit(() -> {
+                try {
+                    // 组内适配器穿行运行，尽量不要配置组内适配器
+                    adapters.forEach(adapter -> {
+                        long begin = System.currentTimeMillis();
+                        List<Dml> dmls = MessageUtil.parse4Dml(canalDestination, groupId, message);
+                        if (dmls != null) {
+                            batchSync(dmls, adapter);
+
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("{} elapsed time: {}",
+                                    adapter.getClass().getName(),
+                                    (System.currentTimeMillis() - begin));
+                            }
+                        }
+                    });
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            }));
+
+            // 等待所有适配器写入完成
+            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
+            RuntimeException exception = null;
+            for (Future<Boolean> future : futures) {
+                try {
+                    if (!future.get()) {
+                        exception = new RuntimeException("Outer adapter sync failed! ");
+                    }
+                } catch (Exception e) {
+                    exception = new RuntimeException(e);
+                }
+            }
+            if (exception != null) {
+                throw exception;
+            }
+        });
+    }
+
+    protected void writeOut(final List<FlatMessage> flatMessages) {
+        List<Future<Boolean>> futures = new ArrayList<>();
+        // 组间适配器并行运行
+        canalOuterAdapters.forEach(outerAdapters -> {
+            futures.add(groupInnerExecutorService.submit(() -> {
+                try {
+                    // 组内适配器穿行运行，尽量不要配置组内适配器
+                    outerAdapters.forEach(adapter -> {
+                        long begin = System.currentTimeMillis();
+                        List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages);
+                        batchSync(dmls, adapter);
+
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("{} elapsed time: {}",
+                                adapter.getClass().getName(),
+                                (System.currentTimeMillis() - begin));
+                        }
+                    });
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            }));
+
+            // 等待所有适配器写入完成
+            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
+            RuntimeException exception = null;
+            for (Future<Boolean> future : futures) {
+                try {
+                    if (!future.get()) {
+                        exception = new RuntimeException("Outer adapter sync failed! ");
+                    }
+                } catch (Exception e) {
+                    exception = new RuntimeException(e);
+                }
+            }
+            if (exception != null) {
+                throw exception;
+            }
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
+                                     CanalMQConnector connector) {
+        ExecutorService workerExecutor =  Util.newSingleDaemonThreadExecutor(5000);
+        try {
+            List<?> messages;
+            if (!flatMessage) {
+                messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
+            } else {
+                messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
+            }
+            if (messages != null && !messages.isEmpty()) {
+                Future<Boolean> future = workerExecutor.submit(() -> {
+                    if (flatMessage) {
+                        // batch write
+                        writeOut((List<FlatMessage>) messages);
+                        // FIXME xxx
+                        // messages.forEach((message ->
+                        // System.out.println(JSON.toJSONString(message))));
+                    } else {
+                        for (final Object message : messages) {
+                            writeOut((Message) message);
+                        }
+                    }
+                    return true;
+                });
+
+                try {
+                    future.get(timeout, TimeUnit.MILLISECONDS);
+                } catch (Exception e) {
+                    future.cancel(true);
+                    throw e;
+                }
+                connector.ack();
+            }
+            return true;
+        } catch (Throwable e) {
+            if (i == retry - 1) {
+                connector.ack();
+                logger.error(e.getMessage() + " Error sync but ACK!");
+                return true;
+            } else {
+                connector.rollback();
+                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
+            }
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException e1) {
+                // ignore
+            }
+        } finally {
+            workerExecutor.shutdown();
+        }
+        return false;
+    }
+
+    /**
+     * 分批同步
+     *
+     * @param dmls
+     * @param adapter
+     */
+    private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
+        // 分批同步
+        if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
+            adapter.sync(dmls);
+        } else {
+            int len = 0;
+            List<Dml> dmlsBatch = new ArrayList<>();
+            for (Dml dml : dmls) {
+                dmlsBatch.add(dml);
+                if (dml.getData() == null || dml.getData().isEmpty()) {
+                    len += 1;
+                } else {
+                    len += dml.getData().size();
+                }
+                if (len >= canalClientConfig.getSyncBatchSize()) {
+                    adapter.sync(dmlsBatch);
+                    dmlsBatch.clear();
+                    len = 0;
+                }
+            }
+            if (!dmlsBatch.isEmpty()) {
+                adapter.sync(dmlsBatch);
+            }
+        }
+    }
+
+    public void start() {
+        if (!running) {
+            thread = new Thread(this::process);
+            thread.setUncaughtExceptionHandler(handler);
+            thread.start();
+            running = true;
+        }
+    }
+
+    protected abstract void process();
+
+    public void stop() {
+        try {
+            if (!running) {
+                return;
+            }
+
+            running = false;
+
+            syncSwitch.release(canalDestination);
+
+            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
+            if (thread != null) {
+                try {
+                    thread.join();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            groupInnerExecutorService.shutdown();
+            logger.info("destination {} adapters worker thread dead!", canalDestination);
+            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
+            logger.info("destination {} all adapters destroyed!", canalDestination);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -1,269 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
-import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.connector.core.config.CanalConstants;
-import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
-import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
-import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
-
-/**
- * 适配处理器
- * 
- * @author rewerma 2020-02-01
- * @version 1.0.0
- */
-public class AdapterProcessor {
-
-    private static final Logger             logger                    = LoggerFactory.getLogger(AdapterProcessor.class);
-
-    private static final String             CONNECTOR_SPI_DIR         = "/plugin";
-    private static final String             CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
-
-    private CanalMsgConsumer                canalMsgConsumer;
-
-    private String                          canalDestination;                                                           // canal实例
-    private String                          groupId                   = null;                                           // groupId
-    private List<List<OuterAdapter>>        canalOuterAdapters;                                                         // 外部适配器
-    private CanalClientConfig               canalClientConfig;                                                          // 配置
-    private ExecutorService                 groupInnerExecutorService;                                                  // 组内工作线程池
-    private volatile boolean                running                   = false;                                          // 是否运行中
-    private Thread                          thread                    = null;
-    private Thread.UncaughtExceptionHandler handler                   = (t, e) -> logger
-        .error("parse events has an error", e);
-
-    private SyncSwitch                      syncSwitch;
-
-    public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
-                            List<List<OuterAdapter>> canalOuterAdapters){
-        this.canalClientConfig = canalClientConfig;
-        this.canalDestination = destination;
-        this.groupId = groupId;
-        this.canalOuterAdapters = canalOuterAdapters;
-
-        this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
-        syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
-
-        // load connector consumer
-        ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
-        canalMsgConsumer = loader
-            .getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
-
-        Properties properties = canalClientConfig.getConsumerProperties();
-        properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
-        properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey());
-        properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey());
-        ClassLoader cl = Thread.currentThread().getContextClassLoader();
-        Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
-        canalMsgConsumer.init(properties, canalDestination, groupId);
-        Thread.currentThread().setContextClassLoader(cl);
-    }
-
-    public void start() {
-        if (!running) {
-            thread = new Thread(this::process);
-            thread.setUncaughtExceptionHandler(handler);
-            thread.start();
-            running = true;
-        }
-    }
-
-    public void writeOut(final List<CommonMessage> commonMessages) {
-        List<Future<Boolean>> futures = new ArrayList<>();
-        // 组间适配器并行运行
-        canalOuterAdapters.forEach(outerAdapters -> {
-            futures.add(groupInnerExecutorService.submit(() -> {
-                try {
-                    // 组内适配器穿行运行，尽量不要配置组内适配器
-                    outerAdapters.forEach(adapter -> {
-                        long begin = System.currentTimeMillis();
-                        List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, commonMessages);
-                        batchSync(dmls, adapter);
-
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("{} elapsed time: {}",
-                                adapter.getClass().getName(),
-                                (System.currentTimeMillis() - begin));
-                        }
-                    });
-                    return true;
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    return false;
-                }
-            }));
-
-            // 等待所有适配器写入完成
-            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            RuntimeException exception = null;
-            for (Future<Boolean> future : futures) {
-                try {
-                    if (!future.get()) {
-                        exception = new RuntimeException("Outer adapter sync failed! ");
-                    }
-                } catch (Exception e) {
-                    exception = new RuntimeException(e);
-                }
-            }
-            if (exception != null) {
-                throw exception;
-            }
-        });
-    }
-
-    /**
-     * 分批同步
-     *
-     * @param dmls
-     * @param adapter
-     */
-    private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
-        // 分批同步
-        if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
-            adapter.sync(dmls);
-        } else {
-            int len = 0;
-            List<Dml> dmlsBatch = new ArrayList<>();
-            for (Dml dml : dmls) {
-                dmlsBatch.add(dml);
-                if (dml.getData() == null || dml.getData().isEmpty()) {
-                    len += 1;
-                } else {
-                    len += dml.getData().size();
-                }
-                if (len >= canalClientConfig.getSyncBatchSize()) {
-                    adapter.sync(dmlsBatch);
-                    dmlsBatch.clear();
-                    len = 0;
-                }
-            }
-            if (!dmlsBatch.isEmpty()) {
-                adapter.sync(dmlsBatch);
-            }
-        }
-    }
-
-    private void process() {
-        while (!running) { // waiting until running == true
-            while (!running) {
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                }
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        if (retry == -1) {
-            // 重试次数-1代表异常时一直阻塞重试
-            retry = Integer.MAX_VALUE;
-        }
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-
-                logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
-                canalMsgConsumer.connect();
-                logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
-                while (running) {
-                    try {
-                        syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
-                    } catch (TimeoutException e) {
-                        break;
-                    }
-                    if (!running) {
-                        break;
-                    }
-
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        try {
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("destination: {} ", canalDestination);
-                            }
-                            long begin = System.currentTimeMillis();
-                            List<CommonMessage> commonMessages = canalMsgConsumer
-                                .getMessage(this.canalClientConfig.getTimeout(), TimeUnit.MILLISECONDS);
-                            writeOut(commonMessages);
-                            canalMsgConsumer.ack();
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("destination: {} elapsed time: {} ms",
-                                    canalDestination,
-                                    System.currentTimeMillis() - begin);
-                            }
-                            break;
-                        } catch (Exception e) {
-                            if (i != retry - 1) {
-                                canalMsgConsumer.rollback(); // 处理失败, 回滚数据
-                                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
-                            } else {
-                                canalMsgConsumer.ack();
-                                logger.error(e.getMessage() + " Error sync but ACK!");
-                            }
-                            Thread.sleep(500);
-                        }
-                    }
-                }
-
-                canalMsgConsumer.disconnect();
-            } catch (Throwable e) {
-                logger.error("process error!", e);
-            }
-
-            if (running) { // is reconnect
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-
-    public void stop() {
-        try {
-            if (!running) {
-                return;
-            }
-
-            running = false;
-
-            syncSwitch.release(canalDestination);
-
-            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
-            if (thread != null) {
-                try {
-                    thread.join();
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-            groupInnerExecutorService.shutdown();
-            logger.info("destination {} adapters worker thread dead!", canalDestination);
-            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
-            logger.info("destination {} all adapters destroyed!", canalDestination);
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java

@@ -0,0 +1,96 @@
+package com.alibaba.otter.canal.adapter.launcher.loader;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.common.errors.WakeupException;
+
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
+
+/**
+ * kafka对应的client适配器工作线程
+ *
+ * @author rewerma 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
+
+    private KafkaCanalConnector connector;
+    private String              topic;
+    private boolean             flatMessage;
+
+    public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
+                                   String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
+        super(canalOuterAdapters);
+        this.canalClientConfig = canalClientConfig;
+        this.topic = topic;
+        super.canalDestination = topic;
+        super.groupId = groupId;
+        this.flatMessage = flatMessage;
+        this.connector = new KafkaCanalConnector(bootstrapServers,
+            topic,
+            null,
+            groupId,
+            canalClientConfig.getBatchSize(),
+            flatMessage);
+        connector.setSessionTimeout(30L, TimeUnit.SECONDS);
+    }
+
+    @Override
+    protected void process() {
+        while (!running) {
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+        int retry = canalClientConfig.getRetries() == null
+                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
+
+        while (running) {
+            try {
+                syncSwitch.get(canalDestination);
+                logger.info("=============> Start to connect topic: {} <=============", this.topic);
+                connector.connect();
+                logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
+                connector.subscribe();
+                logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
+                while (running) {
+                    boolean status = syncSwitch.status(canalDestination);
+                    if (!status) {
+                        connector.disconnect();
+                        break;
+                    }
+                    if (retry == -1) {
+                        retry = Integer.MAX_VALUE;
+                    }
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
+                            break;
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+
+        try {
+            connector.unsubscribe();
+        } catch (WakeupException e) {
+            // No-op. Continue process
+        }
+        connector.disconnect();
+        logger.info("=============> Disconnect topic: {} <=============", this.topic);
+    }
+}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -1,5 +1,8 @@
 package com.alibaba.otter.canal.adapter.launcher.loader;
 
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -30,13 +33,15 @@ import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
  */
 public class CanalAdapterLoader {
 
-    private static final Logger           logger                 = LoggerFactory.getLogger(CanalAdapterLoader.class);
+    private static final Logger                     logger        = LoggerFactory.getLogger(CanalAdapterLoader.class);
 
-    private CanalClientConfig             canalClientConfig;
+    private CanalClientConfig                       canalClientConfig;
 
-    private Map<String, AdapterProcessor> canalAdapterProcessors = new HashMap<>();
+    private Map<String, CanalAdapterWorker>         canalWorkers  = new HashMap<>();
 
-    private ExtensionLoader<OuterAdapter> loader;
+    private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
+
+    private ExtensionLoader<OuterAdapter>           loader;
 
     public CanalAdapterLoader(CanalClientConfig canalClientConfig){
         this.canalClientConfig = canalClientConfig;
@@ -48,154 +53,120 @@ public class CanalAdapterLoader {
     public void init() {
         loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
 
-        for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+        String canalServerHost = this.canalClientConfig.getCanalServerHost();
-            for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+        SocketAddress sa = null;
-                List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+        if (canalServerHost != null) {
-                List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+            String[] ipPort = canalServerHost.split(":");
-                for (OuterAdapterConfig config : group.getOuterAdapters()) {
+            sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
-                    loadAdapter(config, canalOuterAdapters);
+        }
-                }
+        String zkHosts = this.canalClientConfig.getZookeeperHosts();
-                canalOuterAdapterGroups.add(canalOuterAdapters);
 
-                AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()
+        if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
-                                                                                           + "|"
+            // 初始化canal-client的适配器
-                                                                                           + StringUtils.trimToEmpty(group.getGroupId()),
+            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                    f -> new AdapterProcessor(canalClientConfig,
+                List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+
+                for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
+                    List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
+                    for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
+                        loadAdapter(c, canalOutConnectors);
+                    }
+                    canalOuterAdapterGroups.add(canalOutConnectors);
+                }
+                CanalAdapterWorker worker;
+                if (sa != null) {
+                    worker = new CanalAdapterWorker(canalClientConfig,
+                        canalAdapter.getInstance(),
+                        sa,
+                        canalOuterAdapterGroups);
+                } else if (zkHosts != null) {
+                    worker = new CanalAdapterWorker(canalClientConfig,
+                        canalAdapter.getInstance(),
+                        zkHosts,
+                        canalOuterAdapterGroups);
+                } else {
+                    throw new RuntimeException("No canal server connector found");
+                }
+                canalWorkers.put(canalAdapter.getInstance(), worker);
+                worker.start();
+                logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
+            }
+        } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
+            // 初始化canal-client-kafka的适配器
+            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                        loadAdapter(config, canalOuterAdapters);
+                    }
+                    canalOuterAdapterGroups.add(canalOuterAdapters);
+
+                    CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
+                        canalClientConfig.getMqServers(),
                         canalAdapter.getInstance(),
                         group.getGroupId(),
-                        canalOuterAdapterGroups));
+                        canalOuterAdapterGroups,
-                adapterProcessor.start();
+                        canalClientConfig.getFlatMessage());
-
+                    canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
-                logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
+                    canalKafkaWorker.start();
-                                                                                   + group.getGroupId());
+                    logger.info("Start adapter for canal-client mq topic: {} succeed",
+                        canalAdapter.getInstance() + "-" + group.getGroupId());
+                }
             }
-        }
+        } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
+            // 初始化canal-client-rocketMQ的适配器
+            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                        loadAdapter(config, canalOuterAdapters);
+                    }
+                    canalOuterAdapterGroups.add(canalOuterAdapters);
+                    CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
+                        canalClientConfig.getMqServers(),
+                        canalAdapter.getInstance(),
+                        group.getGroupId(),
+                        canalOuterAdapterGroups,
+                        canalClientConfig.getAccessKey(),
+                        canalClientConfig.getSecretKey(),
+                        canalClientConfig.getFlatMessage(),
+                        canalClientConfig.isEnableMessageTrace(),
+                        canalClientConfig.getCustomizedTraceTopic(),
+                        canalClientConfig.getAccessChannel(),
+                        canalClientConfig.getNamespace());
+                    canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
+                    rocketMQWorker.start();
 
-        // if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
+                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // // 初始化canal-client的适配器
+                        canalAdapter.getInstance() + "-" + group.getGroupId());
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
+                }
-        // canalClientConfig.getCanalAdapters()) {
+            }
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
+        } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
-        // CopyOnWriteArrayList<>();
+            // 初始化canal-client-rabbitMQ的适配器
-        //
+            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group connectorGroup :
+                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // canalAdapter.getGroups()) {
+                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
+                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
+                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(c, canalOutConnectors);
+                        loadAdapter(config, canalOuterAdapters);
-        // }
+                    }
-        // canalOuterAdapterGroups.add(canalOutConnectors);
+                    canalOuterAdapterGroups.add(canalOuterAdapters);
-        // }
+                    CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
-        // CanalAdapterWorker worker;
+                        canalOuterAdapterGroups,
-        // if (StringUtils.isNotEmpty(canalServerHost)) {
+                        canalAdapter.getInstance(),
-        // worker = new CanalAdapterWorker(canalClientConfig,
+                        group.getGroupId(),
-        // canalAdapter.getInstance(),
+                        canalClientConfig.getFlatMessage());
-        // canalServerHost,
+                    canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
-        // zkHosts,
+                    rabbitMQWork.start();
-        // canalOuterAdapterGroups);
+
-        // } else if (zkHosts != null) {
+                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // worker = new CanalAdapterWorker(canalClientConfig,
+                        canalAdapter.getInstance() + "-" + group.getGroupId());
-        // canalAdapter.getInstance(),
+                }
-        // zkHosts,
+            }
-        // canalOuterAdapterGroups);
+            // CanalAdapterRabbitMQWork
-        // } else {
+        }
-        // throw new RuntimeException("No canal server connector found");
-        // }
-        // canalWorkers.put(canalAdapter.getInstance(), worker);
-        // worker.start();
-        // logger.info("Start adapter for canal instance: {} succeed",
-        // canalAdapter.getInstance());
-        // }
-        // } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
-        // // 初始化canal-client-kafka的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(config, canalOuterAdapters);
-        // }
-        // canalOuterAdapterGroups.add(canalOuterAdapters);
-        //
-        // CanalAdapterKafkaWorker canalKafkaWorker = new
-        // CanalAdapterKafkaWorker(canalClientConfig,
-        // canalClientConfig.getMqServers(),
-        // canalAdapter.getInstance(),
-        // group.getGroupId(),
-        // canalOuterAdapterGroups,
-        // canalClientConfig.getFlatMessage());
-        // canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" +
-        // group.getGroupId(), canalKafkaWorker);
-        // canalKafkaWorker.start();
-        // logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // canalAdapter.getInstance() + "-" + group.getGroupId());
-        // }
-        // }
-        // } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode()))
-        // {
-        // // 初始化canal-client-rocketMQ的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(config, canalOuterAdapters);
-        // }
-        // canalOuterAdapterGroups.add(canalOuterAdapters);
-        // CanalAdapterRocketMQWorker rocketMQWorker = new
-        // CanalAdapterRocketMQWorker(canalClientConfig,
-        // canalClientConfig.getMqServers(),
-        // canalAdapter.getInstance(),
-        // group.getGroupId(),
-        // canalOuterAdapterGroups,
-        // canalClientConfig.getAccessKey(),
-        // canalClientConfig.getSecretKey(),
-        // canalClientConfig.getFlatMessage(),
-        // canalClientConfig.isEnableMessageTrace(),
-        // canalClientConfig.getCustomizedTraceTopic(),
-        // canalClientConfig.getAccessChannel(),
-        // canalClientConfig.getNamespace());
-        // canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" +
-        // group.getGroupId(), rocketMQWorker);
-        // rocketMQWorker.start();
-        //
-        // logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // canalAdapter.getInstance() + "-" + group.getGroupId());
-        // }
-        // }
-        // } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode()))
-        // {
-        // // 初始化canal-client-rabbitMQ的适配器
-        // for (CanalClientConfig.CanalAdapter canalAdapter :
-        // canalClientConfig.getCanalAdapters()) {
-        // for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-        // List<List<OuterAdapter>> canalOuterAdapterGroups = new
-        // CopyOnWriteArrayList<>();
-        // List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-        // for (OuterAdapterConfig config : group.getOuterAdapters()) {
-        // loadAdapter(config, canalOuterAdapters);
-        // }
-        // canalOuterAdapterGroups.add(canalOuterAdapters);
-        // CanalAdapterRabbitMQWorker rabbitMQWork = new
-        // CanalAdapterRabbitMQWorker(canalClientConfig,
-        // canalOuterAdapterGroups,
-        // canalAdapter.getInstance(),
-        // group.getGroupId(),
-        // canalClientConfig.getFlatMessage());
-        // canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" +
-        // group.getGroupId(), rabbitMQWork);
-        // rabbitMQWork.start();
-        //
-        // logger.info("Start adapter for canal-client mq topic: {} succeed",
-        // canalAdapter.getInstance() + "-" + group.getGroupId());
-        // }
-        // }
-        // // CanalAdapterRabbitMQWork
-        // }
     }
 
     private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
@@ -235,10 +206,10 @@ public class CanalAdapterLoader {
      * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
      */
     public void destroy() {
-        if (!canalAdapterProcessors.isEmpty()) {
+        if (!canalWorkers.isEmpty()) {
-            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalAdapterProcessors.size());
+            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
-            for (AdapterProcessor adapterProcessor : canalAdapterProcessors.values()) {
+            for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
-                stopExecutorService.execute(adapterProcessor::stop);
+                stopExecutorService.execute(canalAdapterWorker::stop);
             }
             stopExecutorService.shutdown();
             try {
@@ -249,6 +220,21 @@ public class CanalAdapterLoader {
                 // ignore
             }
         }
+
+        if (!canalMQWorker.isEmpty()) {
+            ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
+            for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
+                stopMQWorkerService.execute(canalAdapterMQWorker::stop);
+            }
+            stopMQWorkerService.shutdown();
+            try {
+                while (!stopMQWorkerService.awaitTermination(1, TimeUnit.SECONDS)) {
+                    // ignore
+                }
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
         logger.info("All canal adapters destroyed");
     }
 }

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java

@@ -0,0 +1,90 @@
+package com.alibaba.otter.canal.adapter.launcher.loader;
+
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector;
+import org.apache.kafka.common.errors.WakeupException;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker {
+
+    private RabbitMQCanalConnector connector;
+    private String                 topic;
+    private boolean                flatMessage;
+
+    public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List<List<OuterAdapter>> canalOuterAdapters,
+                                      String topic, String groupId, boolean flatMessage){
+        super(canalOuterAdapters);
+        this.canalClientConfig = canalClientConfig;
+        this.topic = topic;
+        this.flatMessage = flatMessage;
+        this.canalDestination = topic;
+        this.groupId = groupId;
+        this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(),
+            canalClientConfig.getVhost(),
+            topic,
+            canalClientConfig.getAccessKey(),
+            canalClientConfig.getSecretKey(),
+            canalClientConfig.getUsername(),
+            canalClientConfig.getPassword(),
+            canalClientConfig.getResourceOwnerId(),
+            flatMessage);
+    }
+
+    @Override
+    protected void process() {
+        while (!running) {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException ignored) {
+            }
+        }
+
+        int retry = canalClientConfig.getRetries() == null
+                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
+        while (running) {
+            try {
+                syncSwitch.get(canalDestination);
+                logger.info("=============> Start to connect topic: {} <=============", this.topic);
+                connector.connect();
+                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
+                connector.subscribe();
+                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
+                while (running) {
+                    boolean status = syncSwitch.status(canalDestination);
+                    if (!status) {
+                        connector.disconnect();
+                        break;
+                    }
+                    if (retry == -1) {
+                        retry = Integer.MAX_VALUE;
+                    }
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
+                            break;
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+
+        try {
+            connector.unsubscribe();
+        } catch (WakeupException e) {
+            // No-op. Continue process
+        }
+        connector.disconnect();
+        logger.info("=============> Disconnect topic: {} <=============", this.topic);
+
+    }
+}