fixed issue #483 , show slave hosts

kafka server端代码调整

.gitignore

@@ -13,3 +13,6 @@ otter.properties
 jtester.properties
 .idea/
 *.iml
+.DS_Store
+*.tar.gz
+*.rpm

README.md

@@ -5,11 +5,13 @@
 <ol>
 <li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
 <li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
+<li>Canal已在阿里云推出商业化版本 <a href="https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7">数据传输服务DTS</a>， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。<a href="https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E">DTS产品使用文档</a></li>
+DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，<a href="https://common-buy.aliyun.com/?commodityCode=dtspre&request=%7b%22dts_function%22%3a%22data_subscribe%22%7d">立即体验>>></a>
 </ol>
 
 <h1>背景</h1>
 <p style="font-size: 14px;">   早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&amp;消费的业务，从此开启了一段新纪元。</p>
-<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.6及以下的版本(阿里内部mysql 5.6.10, mysql 5.5.18和5.1.40/48)</p>
+<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.7及以下的版本(阿里内部mysql 5.7.13, 5.6.10, mysql 5.5.18和5.1.40/48)</p>
 <p style="font-size: 14px;"> </p>
 <p style="font-size: 14px;">基于日志增量订阅&amp;消费支持的业务：</p>
 <ol style="font-size: 14px;">
@@ -62,6 +64,25 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
 </ul>
 
+<h1>相关资料</h1>
+
+* ADC阿里技术嘉年华分享ppt (放在google docs上，可能需要翻墙): <a href="https://docs.google.com/presentation/d/1MkszUPYRDkfVPz9IqOT1LLT5d9tuwde_WC8GZvjaDRg/edit?usp=sharing">ppt下载</href>  
+* [与阿里巴巴的RocketMQ配合使用](https://github.com/alibaba/RocketMQ)
+
+<h1>相关开源</h1>
+<ol>
+<li>阿里巴巴分布式数据库同步系统(解决中美异地机房)：<a href="http://github.com/alibaba/otter">http://github.com/alibaba/otter</a></li>
+<li>阿里巴巴去Oracle数据迁移同步工具(目标支持MySQL/DRDS)：<a href="http://github.com/alibaba/yugong">http://github.com/alibaba/yugong</a></li>
+</ol>
+
+<h1>相关产品</h1>
+<ol>
+<li><a href="https://www.aliyun.com/product/drds?spm=5176.55326.cloudEssentials.71.69fd227dRPZj9K">阿里云分布式数据库DRDS</a></li>
+<li><a href="https://www.aliyun.com/product/dts?spm=5176.7947010.cloudEssentials.80.33f734f4JOAxSP">阿里云数据传输服务DTS</a></li>
+<li><a href="https://www.aliyun.com/product/dbs?spm=5176.54487.cloudEssentials.83.34b851a8GmVZg6">阿里云数据库备份服务DBS</a></li>
+<li><a href="https://www.aliyun.com/product/dms?spm=5176.169464.cloudEssentials.81.2e1066feC1sBBL">阿里云数据管理服务DMS</a></li>
+</ol>
+
 <h1>问题反馈</h1>
 <ol>
 <li>qq交流群： 161559791 </li>
@@ -69,3 +90,10 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li>新浪微博： agapple0002 </li>
 <li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
 </ol>
+
+<pre>
+【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
+岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
+工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊. 
+具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
+</pre>

client-adapter/common/pom.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.common</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter common module for otter ${project.version}</name>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.12</version>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+    </dependencies>
+
+</project>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java

@@ -0,0 +1,34 @@
+package com.alibaba.otter.canal.client.adapter;
+
+import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * 外部适配器接口
+ *
+ * @author machengyuan 2018-8-18 下午10:14:02
+ * @version 1.0.0
+ */
+@SPI("logger")
+public interface CanalOuterAdapter {
+
+    /**
+     * 外部适配器初始化接口
+     *
+     * @param configuration 外部适配器配置信息
+     */
+    void init(CanalOuterAdapterConfiguration configuration);
+
+    /**
+     * 往适配器中写入数据
+     *
+     * @param message message数据包
+     */
+    void writeOut(Message message);
+
+    /**
+     * 外部适配器销毁接口
+     */
+    void destroy();
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -0,0 +1,181 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * 配置信息类
+ *
+ * @author machengyuan 2018-8-18 下午10:40:12
+ * @version 1.0.0
+ */
+public class CanalClientConfig {
+
+    private String              canalServerHost;
+
+    private String              zookeeperHosts;
+
+    private Properties          properties;
+
+    private String              bootstrapServers;
+
+    private List<KafkaTopic>    kafkaTopics;
+
+    private List<CanalInstance> canalInstances;
+
+    public String getCanalServerHost() {
+        return canalServerHost;
+    }
+
+    public void setCanalServerHost(String canalServerHost) {
+        this.canalServerHost = canalServerHost;
+    }
+
+    public String getZookeeperHosts() {
+        return zookeeperHosts;
+    }
+
+    public void setZookeeperHosts(String zookeeperHosts) {
+        this.zookeeperHosts = zookeeperHosts;
+    }
+
+    public Properties getProperties() {
+        return properties;
+    }
+
+    public void setProperties(Properties properties) {
+        this.properties = properties;
+    }
+
+    public String getBootstrapServers() {
+        return bootstrapServers;
+    }
+
+    public void setBootstrapServers(String bootstrapServers) {
+        this.bootstrapServers = bootstrapServers;
+    }
+
+    public List<KafkaTopic> getKafkaTopics() {
+        return kafkaTopics;
+    }
+
+    public void setKafkaTopics(List<KafkaTopic> kafkaTopics) {
+        this.kafkaTopics = kafkaTopics;
+    }
+
+    public List<CanalInstance> getCanalInstances() {
+        return canalInstances;
+    }
+
+    public void setCanalInstances(List<CanalInstance> canalInstances) {
+        this.canalInstances = canalInstances;
+    }
+
+    public static class CanalInstance {
+
+        private String             instance;
+
+        private List<AdapterGroup> adapterGroups;
+
+        public String getInstance() {
+            return instance;
+        }
+
+        public void setInstance(String instance) {
+            if (instance != null) {
+                this.instance = instance.trim();
+            }
+        }
+
+        public List<AdapterGroup> getAdapterGroups() {
+            return adapterGroups;
+        }
+
+        public void setAdapterGroups(List<AdapterGroup> adapterGroups) {
+            this.adapterGroups = adapterGroups;
+        }
+    }
+
+    public static class AdapterGroup {
+
+        private List<CanalOuterAdapterConfiguration> outAdapters;
+
+        public List<CanalOuterAdapterConfiguration> getOutAdapters() {
+            return outAdapters;
+        }
+
+        public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters) {
+            this.outAdapters = outAdapters;
+        }
+    }
+
+    public static class KafkaTopic {
+
+        private String      topic;
+
+        private List<Group> groups = new ArrayList<>();
+
+        public String getTopic() {
+            return topic;
+        }
+
+        public void setTopic(String topic) {
+            this.topic = topic;
+        }
+
+        public List<Group> getGroups() {
+            return groups;
+        }
+
+        public void setGroups(List<Group> groups) {
+            this.groups = groups;
+        }
+    }
+
+    public static class Group {
+
+        private String                               groupId;
+
+        // private List<Adaptor> adapters = new ArrayList<>();
+
+        private List<CanalOuterAdapterConfiguration> outAdapters;
+
+        public String getGroupId() {
+            return groupId;
+        }
+
+        public void setGroupId(String groupId) {
+            this.groupId = groupId;
+        }
+
+        public List<CanalOuterAdapterConfiguration> getOutAdapters() {
+            return outAdapters;
+        }
+
+        public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters) {
+            this.outAdapters = outAdapters;
+        }
+
+        // public List<Adaptor> getAdapters() {
+        // return adapters;
+        // }
+        //
+        // public void setAdapters(List<Adaptor> adapters) {
+        // this.adapters = adapters;
+        // }
+    }
+
+    // public static class Adaptor {
+    // private List<CanalOuterAdapterConfiguration> outAdapters;
+    //
+    // public List<CanalOuterAdapterConfiguration> getOutAdapters() {
+    // return outAdapters;
+    // }
+    //
+    // public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters)
+    // {
+    // this.outAdapters = outAdapters;
+    // }
+    // }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java

@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.util.Properties;
+
+/**
+ * 外部适配器配置信息类
+ *
+ * @author machengyuan 2018-8-18 下午10:15:12
+ * @version 1.0.0
+ */
+public class CanalOuterAdapterConfiguration {
+
+    private String     name;       // 适配器名称, 如: logger, hbase, es
+
+    private String     hosts;      // 适配器内部的地址, 比如对应es该参数可以填写es的server地址
+
+    private String     zkHosts;    // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址
+
+    private Properties properties; // 其余参数, 可填写适配器中的所需的配置信息
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getHosts() {
+        return hosts;
+    }
+
+    public void setHosts(String hosts) {
+        this.hosts = hosts;
+    }
+
+    public Properties getProperties() {
+        return properties;
+    }
+
+    public void setProperties(Properties properties) {
+        this.properties = properties;
+    }
+
+    public String getZkHosts() {
+        return zkHosts;
+    }
+
+    public void setZkHosts(String zkHosts) {
+        this.zkHosts = zkHosts;
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java

@@ -0,0 +1,96 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * DML操作转换对象
+ *
+ * @author machengyuan 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class Dml implements Serializable {
+
+    private static final long         serialVersionUID = 2611556444074013268L;
+
+    private String                    database;
+    private String                    table;
+    private String                    type;
+    private Long                      ts;
+    private String                    sql;
+    private List<Map<String, Object>> data;
+    private List<Map<String, Object>> old;
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Long getTs() {
+        return ts;
+    }
+
+    public void setTs(Long ts) {
+        this.ts = ts;
+    }
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    public List<Map<String, Object>> getData() {
+        return data;
+    }
+
+    public void setData(List<Map<String, Object>> data) {
+        this.data = data;
+    }
+
+    public List<Map<String, Object>> getOld() {
+        return old;
+    }
+
+    public void setOld(List<Map<String, Object>> old) {
+        this.old = old;
+    }
+
+    public void clear() {
+        database = null;
+        table = null;
+        type = null;
+        ts = null;
+        data = null;
+        old = null;
+        sql = null;
+    }
+
+    @Override
+    public String toString() {
+        return "Dml{" + "database='" + database + '\'' + ", table='" + table + '\'' + ", type='" + type + '\'' + ", ts="
+               + ts + ", sql='" + sql + '\'' + ", data=" + data + ", old=" + old + '}';
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java

@@ -0,0 +1,563 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SPI 类加载器
+ *
+ * @author machengyuan 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class ExtensionLoader<T> {
+
+    private static final Logger                                      logger                     = LoggerFactory
+        .getLogger(ExtensionLoader.class);
+
+    private static final String                                      SERVICES_DIRECTORY         = "META-INF/services/";
+
+    private static final String                                      CANAL_DIRECTORY            = "META-INF/canal/";
+
+    private static final String                                      DEFAULT_CLASSLOADER_POLICY = "internal";
+
+    private static final Pattern                                     NAME_SEPARATOR             = Pattern
+        .compile("\\s*[,]+\\s*");
+
+    private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS          = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<Class<?>, Object>             EXTENSION_INSTANCES        = new ConcurrentHashMap<>();
+
+    private final Class<?>                                           type;
+
+    private final String                                             classLoaderPolicy;
+
+    private final ConcurrentMap<Class<?>, String>                    cachedNames                = new ConcurrentHashMap<>();
+
+    private final Holder<Map<String, Class<?>>>                      cachedClasses              = new Holder<>();
+
+    private final ConcurrentMap<String, Holder<Object>>              cachedInstances            = new ConcurrentHashMap<>();
+
+    private String                                                   cachedDefaultName;
+
+    private ConcurrentHashMap<String, IllegalStateException>         exceptions                 = new ConcurrentHashMap<>();
+
+    private static <T> boolean withExtensionAnnotation(Class<T> type) {
+        return type.isAnnotationPresent(SPI.class);
+    }
+
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
+        return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
+        if (type == null) throw new IllegalArgumentException("Extension type == null");
+        if (!type.isInterface()) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
+        }
+        if (!withExtensionAnnotation(type)) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+                                               + SPI.class.getSimpleName() + " Annotation!");
+        }
+
+        ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        if (loader == null) {
+            EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
+            loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        }
+        return loader;
+    }
+
+    private ExtensionLoader(Class<?> type, String classLoaderPolicy){
+        this.type = type;
+        this.classLoaderPolicy = classLoaderPolicy;
+    }
+
+    public String getExtensionName(T extensionInstance) {
+        return getExtensionName(extensionInstance.getClass());
+    }
+
+    public String getExtensionName(Class<?> extensionClass) {
+        return cachedNames.get(extensionClass);
+    }
+
+    public ConcurrentHashMap<String, IllegalStateException> getExceptions() {
+        return exceptions;
+    }
+
+    /**
+     * 返回扩展点实例，如果没有指定的扩展点或是还没加载（即实例化）则返回<code>null</code>注意：此方法不会触发扩展点的加载
+     * <p/>
+     * 一般应该调用{@link #getExtension(String)}方法获得扩展，这个方法会触发扩展点加载
+     *
+     * @see #getExtension(String)
+     */
+    @SuppressWarnings("unchecked")
+    public T getLoadedExtension(String name) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        Holder<Object> holder = cachedInstances.get(name);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(name, new Holder<>());
+            holder = cachedInstances.get(name);
+        }
+        return (T) holder.get();
+    }
+
+    /**
+     * 返回已经加载的扩展点的名字
+     * <p/>
+     * 一般应该调用{@link #getSupportedExtensions()}方法获得扩展，这个方法会返回所有的扩展点
+     *
+     * @see #getSupportedExtensions()
+     */
+    public Set<String> getLoadedExtensions() {
+        return Collections.unmodifiableSet(new TreeSet<>(cachedInstances.keySet()));
+    }
+
+    /**
+     * 返回指定名字的扩展
+     *
+     * @param name
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension();
+        }
+        Holder<Object> holder = cachedInstances.get(name);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(name, new Holder<>());
+            holder = cachedInstances.get(name);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    /**
+     * 返回缺省的扩展，如果没有设置则返回<code>null</code>
+     */
+    public T getDefaultExtension() {
+        getExtensionClasses();
+        if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
+            return null;
+        }
+        return getExtension(cachedDefaultName);
+    }
+
+    public boolean hasExtension(String name) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        try {
+            return getExtensionClass(name) != null;
+        } catch (Throwable t) {
+            return false;
+        }
+    }
+
+    public Set<String> getSupportedExtensions() {
+        Map<String, Class<?>> clazzes = getExtensionClasses();
+        return Collections.unmodifiableSet(new TreeSet<String>(clazzes.keySet()));
+    }
+
+    /**
+     * 返回缺省的扩展点名，如果没有设置缺省则返回<code>null</code>
+     */
+    public String getDefaultExtensionName() {
+        getExtensionClasses();
+        return cachedDefaultName;
+    }
+
+    /**
+     * 编程方式添加新扩展点
+     *
+     * @param name 扩展点名
+     * @param clazz 扩展点类
+     * @throws IllegalStateException 要添加扩展点名已经存在
+     */
+    public void addExtension(String name, Class<?> clazz) {
+        getExtensionClasses(); // load classes
+
+        if (!type.isAssignableFrom(clazz)) {
+            throw new IllegalStateException("Input type " + clazz + "not implement Extension " + type);
+        }
+        if (clazz.isInterface()) {
+            throw new IllegalStateException("Input type " + clazz + "can not be interface!");
+        }
+
+        if (name == null || "".equals(name)) {
+            throw new IllegalStateException("Extension name is blank (Extension " + type + ")!");
+        }
+        if (cachedClasses.get().containsKey(name)) {
+            throw new IllegalStateException("Extension name " + name + " already existed(Extension " + type + ")!");
+        }
+
+        cachedNames.put(clazz, name);
+        cachedClasses.get().put(name, clazz);
+    }
+
+    /**
+     * 编程方式添加替换已有扩展点
+     *
+     * @param name 扩展点名
+     * @param clazz 扩展点类
+     * @throws IllegalStateException 要添加扩展点名已经存在
+     * @deprecated 不推荐应用使用，一般只在测试时可以使用
+     */
+    @Deprecated
+    public void replaceExtension(String name, Class<?> clazz) {
+        getExtensionClasses(); // load classes
+
+        if (!type.isAssignableFrom(clazz)) {
+            throw new IllegalStateException("Input type " + clazz + "not implement Extension " + type);
+        }
+        if (clazz.isInterface()) {
+            throw new IllegalStateException("Input type " + clazz + "can not be interface!");
+        }
+
+        if (name == null || "".equals(name)) {
+            throw new IllegalStateException("Extension name is blank (Extension " + type + ")!");
+        }
+        if (!cachedClasses.get().containsKey(name)) {
+            throw new IllegalStateException("Extension name " + name + " not existed(Extension " + type + ")!");
+        }
+
+        cachedNames.put(clazz, name);
+        cachedClasses.get().put(name, clazz);
+        cachedInstances.remove(name);
+    }
+
+    @SuppressWarnings("unchecked")
+    private T createExtension(String name) {
+        Class<?> clazz = getExtensionClasses().get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_INSTANCES.get(clazz);
+            if (instance == null) {
+                EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
+                instance = (T) EXTENSION_INSTANCES.get(clazz);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(),
+                t);
+        }
+    }
+
+    private Class<?> getExtensionClass(String name) {
+        if (type == null) throw new IllegalArgumentException("Extension type == null");
+        if (name == null) throw new IllegalArgumentException("Extension name == null");
+        Class<?> clazz = getExtensionClasses().get(name);
+        if (clazz == null)
+            throw new IllegalStateException("No such extension \"" + name + "\" for " + type.getName() + "!");
+        return clazz;
+    }
+
+    private Map<String, Class<?>> getExtensionClasses() {
+        Map<String, Class<?>> classes = cachedClasses.get();
+        if (classes == null) {
+            synchronized (cachedClasses) {
+                classes = cachedClasses.get();
+                if (classes == null) {
+                    classes = loadExtensionClasses();
+                    cachedClasses.set(classes);
+                }
+            }
+        }
+        return classes;
+    }
+
+    private String getJarDirectoryPath() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("");
+        if (url == null) {
+            throw new IllegalStateException("failed to get class loader resource");
+        }
+        String dirtyPath = url.toString();
+        String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes file:/ and everything before it
+        jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything after .jar, if .jar exists in dirtyPath
+        jarPath = jarPath.replaceAll("%20", " "); // necessary if path has spaces within
+        if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run the app using Spring Tools Suit play
+                                         // button.
+            jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
+        }
+        return Paths.get(jarPath).getParent().toString(); // Paths - from java 8
+    }
+
+    private Map<String, Class<?>> loadExtensionClasses() {
+        final SPI defaultAnnotation = type.getAnnotation(SPI.class);
+        if (defaultAnnotation != null) {
+            String value = defaultAnnotation.value();
+            if ((value = value.trim()).length() > 0) {
+                String[] names = NAME_SEPARATOR.split(value);
+                if (names.length > 1) {
+                    throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+                                                    + ": " + Arrays.toString(names));
+                }
+                if (names.length == 1) cachedDefaultName = names[0];
+            }
+        }
+
+        Map<String, Class<?>> extensionClasses = new HashMap<String, Class<?>>();
+
+        // 1. lib folder，customized extension classLoader （jar_dir/lib）
+        String dir = File.separator + this.getJarDirectoryPath() + File.separator + "lib";
+        logger.info("extension classpath dir: " + dir);
+        File externalLibDir = new File(dir);
+        if (!externalLibDir.exists()) {
+            externalLibDir = new File(
+                File.separator + this.getJarDirectoryPath() + File.separator + "canal_client" + File.separator + "lib");
+        }
+        if (externalLibDir.exists()) {
+            File[] files = externalLibDir.listFiles(new FilenameFilter() {
+
+                @Override
+                public boolean accept(File dir, String name) {
+                    return name.endsWith(".jar");
+                }
+            });
+            if (files != null) {
+                for (File f : files) {
+                    URL url = null;
+                    try {
+                        url = f.toURI().toURL();
+                    } catch (MalformedURLException e) {
+                        throw new RuntimeException("load extension jar failed!", e);
+                    }
+
+                    ClassLoader parent = Thread.currentThread().getContextClassLoader();
+                    URLClassLoader localClassLoader;
+                    if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
+                        || DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
+                        localClassLoader = new URLClassLoader(new URL[] { url }, parent) {
+
+                            @Override
+                            public Class<?> loadClass(String name) throws ClassNotFoundException {
+                                Class<?> c = findLoadedClass(name);
+                                if (c != null) {
+                                    return c;
+                                }
+
+                                if (name.startsWith("java.") || name.startsWith("org.slf4j.")
+                                    || name.startsWith("org.apache.logging")
+                                    || name.startsWith("org.apache.commons.logging.")) {
+                                    // || name.startsWith("org.apache.hadoop.")) {
+                                    c = super.loadClass(name);
+                                }
+                                if (c != null) return c;
+
+                                try {
+                                    // 先加载jar内的class，可避免jar冲突
+                                    c = findClass(name);
+                                } catch (ClassNotFoundException e) {
+                                    c = null;
+                                }
+                                if (c != null) {
+                                    return c;
+                                }
+
+                                return super.loadClass(name);
+                            }
+
+                            @Override
+                            public Enumeration<URL> getResources(String name) throws IOException {
+                                @SuppressWarnings("unchecked")
+                                Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
+
+                                tmp[0] = findResources(name); // local class path first
+                                // tmp[1] = super.getResources(name);
+
+                                return new CompoundEnumeration<>(tmp);
+                            }
+                        };
+                    } else {
+                        localClassLoader = new URLClassLoader(new URL[] { url }, parent);
+                    }
+
+                    loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
+                    loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
+                }
+            }
+        }
+        // 2. load inner extension class with default classLoader
+        ClassLoader classLoader = findClassLoader();
+        loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
+        loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
+
+        return extensionClasses;
+    }
+
+    public static class CompoundEnumeration<E> implements Enumeration<E> {
+
+        private Enumeration<E>[] enums;
+        private int              index = 0;
+
+        public CompoundEnumeration(Enumeration<E>[] enums){
+            this.enums = enums;
+        }
+
+        private boolean next() {
+            while (this.index < this.enums.length) {
+                if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
+                    return true;
+                }
+
+                ++this.index;
+            }
+
+            return false;
+        }
+
+        public boolean hasMoreElements() {
+            return this.next();
+        }
+
+        public E nextElement() {
+            if (!this.next()) {
+                throw new NoSuchElementException();
+            } else {
+                return this.enums[this.index].nextElement();
+            }
+        }
+    }
+
+    private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
+        String fileName = dir + type.getName();
+        try {
+            Enumeration<URL> urls;
+            if (classLoader != null) {
+                urls = classLoader.getResources(fileName);
+            } else {
+                urls = ClassLoader.getSystemResources(fileName);
+            }
+            if (urls != null) {
+                while (urls.hasMoreElements()) {
+                    URL url = urls.nextElement();
+                    try {
+                        BufferedReader reader = null;
+                        try {
+                            reader = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));
+                            String line = null;
+                            while ((line = reader.readLine()) != null) {
+                                final int ci = line.indexOf('#');
+                                if (ci >= 0) line = line.substring(0, ci);
+                                line = line.trim();
+                                if (line.length() > 0) {
+                                    try {
+                                        String name = null;
+                                        int i = line.indexOf('=');
+                                        if (i > 0) {
+                                            name = line.substring(0, i).trim();
+                                            line = line.substring(i + 1).trim();
+                                        }
+                                        if (line.length() > 0) {
+                                            Class<?> clazz = classLoader.loadClass(line);
+                                            // Class<?> clazz = Class.forName(line, true, classLoader);
+                                            if (!type.isAssignableFrom(clazz)) {
+                                                throw new IllegalStateException(
+                                                    "Error when load extension class(interface: " + type
+                                                                                + ", class line: " + clazz.getName()
+                                                                                + "), class " + clazz.getName()
+                                                                                + "is not subtype of interface.");
+                                            } else {
+                                                try {
+                                                    clazz.getConstructor(type);
+                                                } catch (NoSuchMethodException e) {
+                                                    clazz.getConstructor();
+                                                    String[] names = NAME_SEPARATOR.split(name);
+                                                    if (names != null && names.length > 0) {
+                                                        for (String n : names) {
+                                                            if (!cachedNames.containsKey(clazz)) {
+                                                                cachedNames.put(clazz, n);
+                                                            }
+                                                            Class<?> c = extensionClasses.get(n);
+                                                            if (c == null) {
+                                                                extensionClasses.put(n, clazz);
+                                                            } else if (c != clazz) {
+                                                                cachedNames.remove(clazz);
+                                                                throw new IllegalStateException(
+                                                                    "Duplicate extension " + type.getName() + " name "
+                                                                                                + n + " on "
+                                                                                                + c.getName() + " and "
+                                                                                                + clazz.getName());
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    } catch (Throwable t) {
+                                        IllegalStateException e = new IllegalStateException(
+                                            "Failed to load extension class(interface: " + type + ", class line: "
+                                                                                            + line + ") in " + url
+                                                                                            + ", cause: "
+                                                                                            + t.getMessage(),
+                                            t);
+                                        exceptions.put(line, e);
+                                    }
+                                }
+                            } // end of while read lines
+                        } finally {
+                            if (reader != null) {
+                                reader.close();
+                            }
+                        }
+                    } catch (Throwable t) {
+                        logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+                                     + ") in " + url,
+                            t);
+                    }
+                } // end of while urls
+            }
+        } catch (Throwable t) {
+            logger.error(
+                "Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
+                t);
+        }
+    }
+
+    private static ClassLoader findClassLoader() {
+        return ExtensionLoader.class.getClassLoader();
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getName() + "[" + type.getName() + "]";
+    }
+
+    private static class Holder<T> {
+
+        private volatile T value;
+
+        private void set(T value) {
+            this.value = value;
+        }
+
+        private T get() {
+            return value;
+        }
+
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java

@@ -0,0 +1,103 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.sql.Types;
+
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 类型转换工具类
+ *
+ * @author machengyuan 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class JdbcTypeUtil {
+
+    private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
+
+    public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
+        if (value == null || value.equals("")) {
+            return null;
+        }
+
+        try {
+            Object res;
+            switch (sqlType) {
+                case Types.INTEGER:
+                    res = Integer.parseInt(value);
+                    break;
+                case Types.SMALLINT:
+                    res = Short.parseShort(value);
+                    break;
+                case Types.BIT:
+                case Types.TINYINT:
+                    res = Byte.parseByte(value);
+                    break;
+                case Types.BIGINT:
+                    if (mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
+                        res = new BigInteger(value);
+                    } else {
+                        res = Long.parseLong(value);
+                    }
+                    break;
+                // case Types.BIT:
+                case Types.BOOLEAN:
+                    res = !"0".equals(value);
+                    break;
+                case Types.DOUBLE:
+                case Types.FLOAT:
+                    res = Double.parseDouble(value);
+                    break;
+                case Types.REAL:
+                    res = Float.parseFloat(value);
+                    break;
+                case Types.DECIMAL:
+                case Types.NUMERIC:
+                    res = new BigDecimal(value);
+                    break;
+                case Types.BINARY:
+                case Types.VARBINARY:
+                case Types.LONGVARBINARY:
+                case Types.BLOB:
+                    res = value.getBytes("ISO-8859-1");
+                    break;
+                case Types.DATE:
+                    if (!value.startsWith("0000-00-00")) {
+                        value = value.trim().replace(" ", "T");
+                        DateTime dt = new DateTime(value);
+                        res = new Date(dt.toDate().getTime());
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.TIME:
+                    value = "T" + value;
+                    DateTime dt = new DateTime(value);
+                    res = new Time(dt.toDate().getTime());
+                    break;
+                case Types.TIMESTAMP:
+                    if (!value.startsWith("0000-00-00")) {
+                        value = value.trim().replace(" ", "T");
+                        dt = new DateTime(value);
+                        res = new Timestamp(dt.toDate().getTime());
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.CLOB:
+                default:
+                    res = value;
+            }
+            return res;
+        } catch (Exception e) {
+            logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
+            return value;
+        }
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java

@@ -0,0 +1,113 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.util.*;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Message对象解析工具类
+ *
+ * @author machengyuan 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class MessageUtil {
+
+    public static void parse4Dml(Message message, Consumer<Dml> consumer) {
+        if (message == null) {
+            return;
+        }
+        List<CanalEntry.Entry> entries = message.getEntries();
+
+        for (CanalEntry.Entry entry : entries) {
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            CanalEntry.RowChange rowChange;
+            try {
+                rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
+                    e);
+            }
+
+            CanalEntry.EventType eventType = rowChange.getEventType();
+
+            final Dml dml = new Dml();
+            dml.setDatabase(entry.getHeader().getSchemaName());
+            dml.setTable(entry.getHeader().getTableName());
+            dml.setType(eventType.toString());
+            dml.setTs(System.currentTimeMillis());
+            dml.setSql(rowChange.getSql());
+            List<Map<String, Object>> data = new ArrayList<>();
+            List<Map<String, Object>> old = new ArrayList<>();
+
+            if (!rowChange.getIsDdl()) {
+                Set<String> updateSet = new HashSet<>();
+                for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                    if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                        && eventType != CanalEntry.EventType.DELETE) {
+                        continue;
+                    }
+
+                    Map<String, Object> row = new LinkedHashMap<>();
+                    List<CanalEntry.Column> columns;
+
+                    if (eventType == CanalEntry.EventType.DELETE) {
+                        columns = rowData.getBeforeColumnsList();
+                    } else {
+                        columns = rowData.getAfterColumnsList();
+                    }
+
+                    for (CanalEntry.Column column : columns) {
+                        row.put(column.getName(),
+                            JdbcTypeUtil.typeConvert(dml.getTable(),
+                                column.getName(),
+                                column.getValue(),
+                                column.getSqlType(),
+                                column.getMysqlType()));
+                        // 获取update为true的字段
+                        if (column.getUpdated()) {
+                            updateSet.add(column.getName());
+                        }
+                    }
+                    if (!row.isEmpty()) {
+                        data.add(row);
+                    }
+
+                    if (eventType == CanalEntry.EventType.UPDATE) {
+                        Map<String, Object> rowOld = new LinkedHashMap<>();
+                        for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                            if (updateSet.contains(column.getName())) {
+                                rowOld.put(column.getName(),
+                                    JdbcTypeUtil.typeConvert(dml.getTable(),
+                                        column.getName(),
+                                        column.getValue(),
+                                        column.getSqlType(),
+                                        column.getMysqlType()));
+                            }
+                        }
+                        // update操作将记录修改前的值
+                        if (!rowOld.isEmpty()) {
+                            old.add(rowOld);
+                        }
+                    }
+                }
+                if (!data.isEmpty()) {
+                    dml.setData(data);
+                }
+                if (!old.isEmpty()) {
+                    dml.setOld(old);
+                }
+                consumer.accept(dml);
+            }
+        }
+    }
+
+    public interface Consumer<T> {
+
+        void accept(T t);
+    }
+}

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java

@@ -0,0 +1,12 @@
+package com.alibaba.otter.canal.client.adapter.support;
+
+import java.lang.annotation.*;
+
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface SPI {
+
+    // Default SPI name
+    String value() default "";
+}

client-adapter/hbase/pom.xml

@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.hbase</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter hbase module for otter ${project.version}</name>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.17</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>1.1.2</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>12.0.1</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <addMavenDescriptor>true</addMavenDescriptor>
+                    </archive>
+                    <excludes>
+                        <exclude>**/hbase-mapping/**</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../client-launcher/target/canal_client/conf/hbase-mapping" overwrite="true" >
+                                    <fileset dir="${project.basedir}/src/main/resources/hbase-mapping" erroronmissingdir="true">
+                                        <include name="*.conf"/>
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java

@@ -0,0 +1,115 @@
+package com.alibaba.otter.canal.client.adapter.hbase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfigLoader;
+import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService;
+import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * HBase外部适配器
+ *
+ * @author machengyuan 2018-8-21 下午8:45:38
+ * @version 1.0.0
+ */
+@SPI("hbase")
+public class HbaseAdapter implements CanalOuterAdapter {
+
+    private static volatile Map<String, MappingConfig> mappingConfigCache = null;
+
+    private Connection                                 conn;
+    private HbaseSyncService                           hbaseSyncService;
+
+    @Override
+    public void init(CanalOuterAdapterConfiguration configuration) {
+        try {
+            if (mappingConfigCache == null) {
+                synchronized (MappingConfig.class) {
+                    if (mappingConfigCache == null) {
+                        Map<String, MappingConfig> hbaseMapping = MappingConfigLoader.load();
+                        mappingConfigCache = new HashMap<>();
+                        for (MappingConfig mappingConfig : hbaseMapping.values()) {
+                            mappingConfigCache.put(mappingConfig.getHbaseOrm().getDatabase() + "-"
+                                                   + mappingConfig.getHbaseOrm().getTable(),
+                                mappingConfig);
+                        }
+                    }
+                }
+            }
+
+            String hosts = configuration.getZkHosts();
+            if (StringUtils.isEmpty(hosts)) {
+                hosts = configuration.getHosts();
+            }
+            if (StringUtils.isEmpty(hosts)) {
+                throw new RuntimeException("Empty zookeeper hosts");
+            }
+            String[] zkHosts = StringUtils.split(hosts, ",");
+            int zkPort = 0;
+            StringBuilder hostsWithoutPort = new StringBuilder();
+            for (String host : zkHosts) {
+                int i = host.indexOf(":");
+                hostsWithoutPort.append(host, 0, i);
+                hostsWithoutPort.append(",");
+                if (zkPort == 0) zkPort = Integer.parseInt(host.substring(i + 1));
+            }
+            hostsWithoutPort.deleteCharAt(hostsWithoutPort.length() - 1);
+
+            String znode = configuration.getProperties().getProperty("znodeParent");
+            if (StringUtils.isEmpty(znode)) {
+                znode = "/hbase";
+            }
+
+            Configuration hbaseConfig = HBaseConfiguration.create();
+            hbaseConfig.set("hbase.zookeeper.quorum", hostsWithoutPort.toString());
+            hbaseConfig.set("hbase.zookeeper.property.clientPort", Integer.toString(zkPort));
+            hbaseConfig.set("zookeeper.znode.parent", znode);
+            conn = ConnectionFactory.createConnection(hbaseConfig);
+            hbaseSyncService = new HbaseSyncService(conn);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void writeOut(Message message) {
+        MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
+
+            @Override
+            public void accept(Dml dml) {
+                if (dml == null) {
+                    return;
+                }
+                String database = dml.getDatabase();
+                String table = dml.getTable();
+                MappingConfig config = mappingConfigCache.get(database + "-" + table);
+                hbaseSyncService.sync(config, dml);
+            }
+        });
+    }
+
+    @Override
+    public void destroy() {
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java

@@ -0,0 +1,337 @@
+package com.alibaba.otter.canal.client.adapter.hbase.config;
+
+import java.util.*;
+
+/**
+ * HBase表映射配置
+ *
+ * @author machengyuan 2018-8-21 下午06:45:49
+ * @version 1.0.0
+ */
+public class MappingConfig {
+
+    private HbaseOrm hbaseOrm;
+
+    public HbaseOrm getHbaseOrm() {
+        return hbaseOrm;
+    }
+
+    public void setHbaseOrm(HbaseOrm hbaseOrm) {
+        this.hbaseOrm = hbaseOrm;
+    }
+
+    public void validate() {
+        if (hbaseOrm.database == null || hbaseOrm.database.isEmpty()) {
+            throw new NullPointerException("hbaseOrm.database");
+        }
+        if (hbaseOrm.table == null || hbaseOrm.table.isEmpty()) {
+            throw new NullPointerException("hbaseOrm.table");
+        }
+        if (hbaseOrm.hbaseTable == null || hbaseOrm.hbaseTable.isEmpty()) {
+            throw new NullPointerException("hbaseOrm.hbaseTable");
+        }
+        if (hbaseOrm.mode == null) {
+            throw new NullPointerException("hbaseOrm.mode");
+        }
+        if (hbaseOrm.rowKey != null && hbaseOrm.rowKeyColumn != null) {
+            throw new RuntimeException("已配置了复合主键作为RowKey，无需再指定RowKey列");
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        MappingConfig config = (MappingConfig) o;
+
+        return hbaseOrm != null ? hbaseOrm.equals(config.hbaseOrm) : config.hbaseOrm == null;
+    }
+
+    @Override
+    public int hashCode() {
+        return hbaseOrm != null ? hbaseOrm.hashCode() : 0;
+    }
+
+    public static class ColumnItem {
+
+        private boolean isRowKey = false;
+        private String  column;
+        private String  family;
+        private String  qualifier;
+        private String  type;
+
+        public boolean isRowKey() {
+            return isRowKey;
+        }
+
+        public void setRowKey(boolean rowKey) {
+            isRowKey = rowKey;
+        }
+
+        public String getColumn() {
+            return column;
+        }
+
+        public void setColumn(String column) {
+            this.column = column;
+        }
+
+        public String getFamily() {
+            return family;
+        }
+
+        public void setFamily(String family) {
+            this.family = family;
+        }
+
+        public String getQualifier() {
+            return qualifier;
+        }
+
+        public void setQualifier(String qualifier) {
+            this.qualifier = qualifier;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public void setType(String type) {
+            this.type = type;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ColumnItem that = (ColumnItem) o;
+            return Objects.equals(column, that.column);
+        }
+
+        @Override
+        public int hashCode() {
+
+            return Objects.hash(column);
+        }
+    }
+
+    public enum Mode {
+                      STRING("STRING"), NATIVE("NATIVE"), PHOENIX("PHOENIX");
+
+        private String type;
+
+        public String getType() {
+            return type;
+        }
+
+        Mode(String type){
+            this.type = type;
+        }
+    }
+
+    public static class HbaseOrm {
+
+        private Mode                    mode               = Mode.STRING;
+        private String                  database;
+        private String                  table;
+        private String                  hbaseTable;
+        private String                  family             = "CF";
+        private boolean                 uppercaseQualifier = true;
+        private boolean                 autoCreateTable    = false;                // 同步时HBase中表不存在的情况下自动建表
+        private String                  rowKey;                                    // 指定复合主键为rowKey
+        private Map<String, String>     columns;
+        private ColumnItem              rowKeyColumn;
+        private String                  etlCondition;
+
+        private Map<String, ColumnItem> columnItems        = new LinkedHashMap<>();
+        private Set<String>             families           = new LinkedHashSet<>();
+        private int                     readBatch          = 5000;
+        private int                     commitBatch        = 5000;
+
+        public Mode getMode() {
+            return mode;
+        }
+
+        public void setMode(Mode mode) {
+            this.mode = mode;
+        }
+
+        public String getDatabase() {
+            return database;
+        }
+
+        public void setDatabase(String database) {
+            this.database = database;
+        }
+
+        public String getTable() {
+            return table;
+        }
+
+        public void setTable(String table) {
+            this.table = table;
+        }
+
+        public String getHbaseTable() {
+            return hbaseTable;
+        }
+
+        public void setHbaseTable(String hbaseTable) {
+            this.hbaseTable = hbaseTable;
+        }
+
+        public Map<String, String> getColumns() {
+            return columns;
+        }
+
+        public boolean isAutoCreateTable() {
+            return autoCreateTable;
+        }
+
+        public void setAutoCreateTable(boolean autoCreateTable) {
+            this.autoCreateTable = autoCreateTable;
+        }
+
+        public int getReadBatch() {
+            return readBatch;
+        }
+
+        public void setReadBatch(int readBatch) {
+            this.readBatch = readBatch;
+        }
+
+        public int getCommitBatch() {
+            return commitBatch;
+        }
+
+        public void setCommitBatch(int commitBatch) {
+            this.commitBatch = commitBatch;
+        }
+
+        public String getRowKey() {
+            return rowKey;
+        }
+
+        public void setRowKey(String rowKey) {
+            this.rowKey = rowKey;
+        }
+
+        public String getEtlCondition() {
+            return etlCondition;
+        }
+
+        public void setEtlCondition(String etlCondition) {
+            this.etlCondition = etlCondition;
+        }
+
+        public void setColumns(Map<String, String> columns) {
+            this.columns = columns;
+
+            if (columns != null) {
+                for (Map.Entry<String, String> columnField : columns.entrySet()) {
+                    String field = columnField.getValue();
+                    String type = null;
+                    if (field != null) {
+                        // 解析类型
+                        int i = field.indexOf("$");
+                        if (i > -1) {
+                            type = field.substring(i + 1);
+                            field = field.substring(0, i);
+                        }
+                    }
+                    ColumnItem columnItem = new ColumnItem();
+                    columnItem.setColumn(columnField.getKey());
+                    columnItem.setType(type);
+                    if ("rowKey".equalsIgnoreCase(field)) {
+                        columnItem.setRowKey(true);
+                        rowKeyColumn = columnItem;
+                    } else {
+                        if (field == null || field.equals("")) {
+                            columnItem.setFamily(family);
+                            columnItem.setQualifier(columnField.getKey());
+                        } else {
+                            int len = field.indexOf(":");
+                            if (len > -1) {
+                                columnItem.setFamily(field.substring(0, len));
+                                columnItem.setQualifier(field.substring(len + 1));
+                            } else {
+                                columnItem.setFamily(family);
+                                columnItem.setQualifier(field);
+                            }
+                        }
+                        if (uppercaseQualifier) {
+                            columnItem.setQualifier(columnItem.getQualifier().toUpperCase());
+                        }
+                        families.add(columnItem.getFamily());
+                    }
+
+                    columnItems.put(columnField.getKey(), columnItem);
+                }
+            } else {
+                this.columns = new LinkedHashMap<>();
+            }
+        }
+
+        public String getFamily() {
+            return family;
+        }
+
+        public void setFamily(String family) {
+            this.family = family;
+            if (family == null) {
+                this.family = "CF";
+            }
+        }
+
+        public boolean isUppercaseQualifier() {
+            return uppercaseQualifier;
+        }
+
+        public void setUppercaseQualifier(boolean uppercaseQualifier) {
+            this.uppercaseQualifier = uppercaseQualifier;
+        }
+
+        public ColumnItem getRowKeyColumn() {
+            return rowKeyColumn;
+        }
+
+        public void setRowKeyColumn(ColumnItem rowKeyColumn) {
+            this.rowKeyColumn = rowKeyColumn;
+        }
+
+        public Map<String, ColumnItem> getColumnItems() {
+            return columnItems;
+        }
+
+        public void setColumnItems(Map<String, ColumnItem> columnItems) {
+            this.columnItems = columnItems;
+        }
+
+        public Set<String> getFamilies() {
+            return families;
+        }
+
+        public void setFamilies(Set<String> families) {
+            this.families = families;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            HbaseOrm hbaseOrm = (HbaseOrm) o;
+
+            if (table != null ? !table.equals(hbaseOrm.table) : hbaseOrm.table != null) return false;
+            return hbaseTable != null ? hbaseTable.equals(hbaseOrm.hbaseTable) : hbaseOrm.hbaseTable == null;
+        }
+
+        @Override
+        public int hashCode() {
+            int result = table != null ? table.hashCode() : 0;
+            result = 31 * result + (hbaseTable != null ? hbaseTable.hashCode() : 0);
+            return result;
+        }
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java

@@ -0,0 +1,149 @@
+package com.alibaba.otter.canal.client.adapter.hbase.config;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * HBase表映射配置加载器
+ * <p>
+ * 配置统一从hbase-mapping/configs.conf文件作为入口, 该文件包含所有表映射配置的名称或者文件名列表。
+ * 每个对应的表配置可以yml配置文件或者以database.table为配置名的简化形式
+ * </p>
+ *
+ * @author machengyuan 2018-8-21 下午06:45:49
+ * @version 1.0.0
+ */
+public class MappingConfigLoader {
+
+    private static Logger       logger    = LoggerFactory.getLogger(MappingConfigLoader.class);
+
+    private static final String BASE_PATH = "hbase-mapping/";
+
+    /**
+     * 加载HBase表映射配置
+     * 
+     * @return 配置名/配置文件名--对象
+     */
+    public static Map<String, MappingConfig> load() {
+        logger.info("## Start loading mapping config ... ");
+        String mappingConfigContent = readConfigContent(BASE_PATH + "configs.conf");
+
+        Map<String, MappingConfig> result = new LinkedHashMap<>();
+
+        String[] configLines = mappingConfigContent.split("\n");
+        for (String c : configLines) {
+            if (c == null) {
+                continue;
+            }
+            c = c.trim();
+            if (c.equals("") || c.startsWith("#")) {
+                continue;
+            }
+
+            MappingConfig config;
+            String configContent = null;
+
+            if (c.endsWith(".yml")) {
+                configContent = readConfigContent(BASE_PATH + "/" + c);
+            }
+
+            // 简单配置database.table@datasourcekey?rowKey=key1,key2
+            if (StringUtils.isEmpty(configContent)) {
+                String[] mapping = c.split("\\?");
+                String params = mapping.length == 2 ? mapping[1] : null;
+                String rowKey = null;
+                String srcMeta = mapping[0];
+                //
+                if (params != null) {
+                    for (String entry : params.split("&")) {
+                        if ("rowKey".equals(entry.split("=")[0])) {
+                            rowKey = entry.split("=")[1];
+                        }
+                    }
+                }
+                String dsKey = srcMeta.split("@").length == 2 ? srcMeta.split("@")[1] : null;
+                String[] dbTable;
+                if (dsKey == null) {
+                    dbTable = srcMeta.split("\\.");
+                } else {
+                    dbTable = srcMeta.split("@")[0].split("\\.");
+                }
+
+                if (dbTable.length == 2) {
+                    config = new MappingConfig();
+
+                    MappingConfig.HbaseOrm hbaseOrm = new MappingConfig.HbaseOrm();
+                    hbaseOrm.setHbaseTable(dbTable[0].toUpperCase() + "." + dbTable[1].toUpperCase());
+                    hbaseOrm.setAutoCreateTable(true);
+                    hbaseOrm.setDatabase(dbTable[0]);
+                    hbaseOrm.setTable(dbTable[1]);
+                    hbaseOrm.setMode(MappingConfig.Mode.STRING);
+                    hbaseOrm.setRowKey(rowKey);
+                    // 有定义rowKey
+                    if (rowKey != null) {
+                        MappingConfig.ColumnItem columnItem = new MappingConfig.ColumnItem();
+                        columnItem.setRowKey(true);
+                        columnItem.setColumn(rowKey);
+                        hbaseOrm.setRowKeyColumn(columnItem);
+                    }
+                    config.setHbaseOrm(hbaseOrm);
+
+                } else {
+                    throw new RuntimeException(String.format("配置项[%s]内容为空, 或格式不符合database.table", c));
+                }
+
+            } else { // 配置文件配置
+                config = new Yaml().loadAs(configContent, MappingConfig.class);
+            }
+
+            try {
+                config.validate();
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e);
+            }
+            result.put(c, config);
+        }
+
+        logger.info("## Mapping config loaded");
+        return result;
+    }
+
+    public static String readConfigContent(String config) {
+        InputStream in = null;
+        try {
+            // 先取本地文件，再取类路径
+            File configFile = new File("config/" + config);
+            if (configFile.exists()) {
+                in = new FileInputStream(configFile);
+            } else {
+                in = MappingConfigLoader.class.getClassLoader().getResourceAsStream(config);
+            }
+            if (in == null) {
+                throw new RuntimeException("Config file not found.");
+            }
+
+            byte[] bytes = new byte[in.available()];
+            in.read(bytes);
+            return new String(bytes, "UTF-8");
+        } catch (IOException e) {
+            throw new RuntimeException("Read ds-config.yml or hbase-mappings.conf error. ", e);
+        } finally {
+            try {
+                if (in != null) {
+                    in.close();
+                }
+            } catch (IOException e) {
+                // ignore
+            }
+        }
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java

@@ -0,0 +1,418 @@
+package com.alibaba.otter.canal.client.adapter.hbase.service;
+
+import java.util.*;
+
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
+import com.alibaba.otter.canal.client.adapter.hbase.support.*;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+
+/**
+ * HBase同步操作业务
+ *
+ * @author machengyuan 2018-8-21 下午06:45:49
+ * @version 1.0.0
+ */
+public class HbaseSyncService {
+
+    private Logger        logger = LoggerFactory.getLogger(this.getClass());
+
+    private HbaseTemplate hbaseTemplate;                                    // HBase操作模板
+
+    public HbaseSyncService(Connection conn){
+        hbaseTemplate = new HbaseTemplate(conn);
+    }
+
+    public void sync(MappingConfig config, Dml dml) {
+        try {
+            if (config != null) {
+                String type = dml.getType();
+                if (type != null && type.equalsIgnoreCase("INSERT")) {
+                    insert(config, dml);
+                } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
+                    update(config, dml);
+                } else if (type != null && type.equalsIgnoreCase("DELETE")) {
+                    delete(config, dml);
+                }
+                if (logger.isDebugEnabled()) {
+                    String res = dml.toString();
+                    logger.debug(res);
+                }
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * 插入操作
+     * 
+     * @param config 配置项
+     * @param dml DML数据
+     */
+    private void insert(MappingConfig config, Dml dml) {
+        List<Map<String, Object>> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm();
+
+        // if (!validHTable(config)) {
+        // logger.error("HBase table '{}' not exists", hbaseOrm.getHbaseTable());
+        // return;
+        // }
+        int i = 1;
+        boolean complete = false;
+        List<HRow> rows = new ArrayList<>();
+        for (Map<String, Object> r : data) {
+            HRow hRow = new HRow();
+
+            // 拼接复合rowKey
+            if (hbaseOrm.getRowKey() != null) {
+                String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(",");
+                String rowKeyVale = getRowKeys(rowKeyColumns, r);
+                // params.put("rowKey", Bytes.toBytes(rowKeyVale));
+                hRow.setRowKey(Bytes.toBytes(rowKeyVale));
+            }
+
+            convertData2Row(hbaseOrm, hRow, r);
+            if (hRow.getRowKey() == null) {
+                throw new RuntimeException("empty rowKey");
+            }
+            rows.add(hRow);
+            complete = false;
+            if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) {
+                hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
+                rows.clear();
+                complete = true;
+            }
+            i++;
+        }
+        if (!complete && !rows.isEmpty()) {
+            hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
+        }
+
+    }
+
+    /**
+     * 将Map数据转换为HRow行数据
+     * 
+     * @param hbaseOrm hbase映射配置
+     * @param hRow 行对象
+     * @param data Map数据
+     */
+    private static void convertData2Row(MappingConfig.HbaseOrm hbaseOrm, HRow hRow, Map<String, Object> data) {
+        Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
+        int i = 0;
+        for (Map.Entry<String, Object> entry : data.entrySet()) {
+            if (entry.getValue() != null) {
+                MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey());
+
+                byte[] bytes = typeConvert(columnItem, hbaseOrm, entry.getValue());
+
+                if (columnItem == null) {
+                    String familyName = hbaseOrm.getFamily();
+                    String qualifier = entry.getKey();
+                    if (hbaseOrm.isUppercaseQualifier()) {
+                        qualifier = qualifier.toUpperCase();
+                    }
+
+                    if (hbaseOrm.getRowKey() == null && i == 0) {
+                        hRow.setRowKey(bytes);
+                    } else {
+                        hRow.addCell(familyName, qualifier, bytes);
+                    }
+                } else {
+                    if (columnItem.isRowKey()) {
+                        // row.put("rowKey", bytes);
+                        hRow.setRowKey(bytes);
+                    } else {
+                        hRow.addCell(columnItem.getFamily(), columnItem.getQualifier(), bytes);
+                    }
+                }
+            }
+            i++;
+        }
+    }
+
+    /**
+     * 更新操作
+     * 
+     * @param config
+     * @param dml
+     */
+    private void update(MappingConfig config, Dml dml) {
+        List<Map<String, Object>> data = dml.getData();
+        List<Map<String, Object>> old = dml.getOld();
+        if (old == null || old.isEmpty() || data == null || data.isEmpty()) {
+            return;
+        }
+
+        MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm();
+
+        // if (!validHTable(config)) {
+        // logger.error("HBase table '{}' not exists", hbaseOrm.getHbaseTable());
+        // return;
+        // }
+
+        MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn();
+        int index = 0;
+        int i = 1;
+        boolean complete = false;
+        List<HRow> rows = new ArrayList<>();
+        out: for (Map<String, Object> r : data) {
+            byte[] rowKeyBytes;
+
+            if (hbaseOrm.getRowKey() != null) {
+                String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(",");
+
+                // 判断是否有复合主键修改
+                for (String updateColumn : old.get(index).keySet()) {
+                    for (String rowKeyColumnName : rowKeyColumns) {
+                        if (rowKeyColumnName.equalsIgnoreCase(updateColumn)) {
+                            // 调用删除插入操作
+                            deleteAndInsert(config, dml);
+                            continue out;
+                        }
+                    }
+                }
+
+                String rowKeyVale = getRowKeys(rowKeyColumns, r);
+                rowKeyBytes = Bytes.toBytes(rowKeyVale);
+            } else if (rowKeyColumn == null) {
+                Map<String, Object> rowKey = data.get(0);
+                rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next());
+            } else {
+                rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, r.get(rowKeyColumn.getColumn()));
+            }
+            if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空");
+
+            Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
+            HRow hRow = new HRow(rowKeyBytes);
+            for (String updateColumn : old.get(index).keySet()) {
+                MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);
+                if (columnItem == null) {
+                    String family = hbaseOrm.getFamily();
+                    String qualifier = updateColumn;
+                    if (hbaseOrm.isUppercaseQualifier()) {
+                        qualifier = qualifier.toUpperCase();
+                    }
+
+                    Object newVal = r.get(updateColumn);
+
+                    if (newVal == null) {
+                        hRow.addCell(family, qualifier, null);
+                    } else {
+                        hRow.addCell(family, qualifier, typeConvert(null, hbaseOrm, newVal));
+                    }
+                } else {
+                    // 排除修改id的情况
+                    if (columnItem.isRowKey()) continue;
+
+                    Object newVal = r.get(updateColumn);
+                    if (newVal == null) {
+                        hRow.addCell(columnItem.getFamily(), columnItem.getQualifier(), null);
+                    } else {
+                        hRow.addCell(columnItem.getFamily(),
+                            columnItem.getQualifier(),
+                            typeConvert(columnItem, hbaseOrm, newVal));
+                    }
+                }
+            }
+            rows.add(hRow);
+            complete = false;
+            if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) {
+                hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
+                rows.clear();
+                complete = true;
+            }
+            i++;
+            index++;
+        }
+        if (!complete && !rows.isEmpty()) {
+            hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
+        }
+    }
+
+    private void delete(MappingConfig config, Dml dml) {
+        List<Map<String, Object>> data = dml.getData();
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm();
+
+        // if (!validHTable(config)) {
+        // logger.error("HBase table '{}' not exists", hbaseOrm.getHbaseTable());
+        // return;
+        // }
+
+        MappingConfig.ColumnItem rowKeyColumn = hbaseOrm.getRowKeyColumn();
+        boolean complete = false;
+        int i = 1;
+        Set<byte[]> rowKeys = new HashSet<>();
+        for (Map<String, Object> r : data) {
+            byte[] rowKeyBytes;
+
+            if (hbaseOrm.getRowKey() != null) {
+                String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(",");
+                String rowKeyVale = getRowKeys(rowKeyColumns, r);
+                rowKeyBytes = Bytes.toBytes(rowKeyVale);
+            } else if (rowKeyColumn == null) {
+                // 如果不需要类型转换
+                Map<String, Object> rowKey = data.get(0);
+                rowKeyBytes = typeConvert(null, hbaseOrm, rowKey.values().iterator().next());
+            } else {
+                Object val = r.get(rowKeyColumn.getColumn());
+                rowKeyBytes = typeConvert(rowKeyColumn, hbaseOrm, val);
+            }
+            if (rowKeyBytes == null) throw new RuntimeException("rowKey值为空");
+            rowKeys.add(rowKeyBytes);
+            complete = false;
+            if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rowKeys.isEmpty()) {
+                hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys);
+                rowKeys.clear();
+                complete = true;
+            }
+            i++;
+        }
+        if (!complete && !rowKeys.isEmpty()) {
+            hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys);
+        }
+    }
+
+    private void deleteAndInsert(MappingConfig config, Dml dml) {
+        List<Map<String, Object>> data = dml.getData();
+        List<Map<String, Object>> old = dml.getOld();
+        if (old == null || old.isEmpty() || data == null || data.isEmpty()) {
+            return;
+        }
+        MappingConfig.HbaseOrm hbaseOrm = config.getHbaseOrm();
+
+        String[] rowKeyColumns = hbaseOrm.getRowKey().trim().split(",");
+
+        int index = 0;
+        int i = 1;
+        boolean complete = false;
+        Set<byte[]> rowKeys = new HashSet<>();
+        List<HRow> rows = new ArrayList<>();
+        for (Map<String, Object> r : data) {
+            // 拼接老的rowKey
+            List<String> updateSubRowKey = new ArrayList<>();
+            for (String rowKeyColumnName : rowKeyColumns) {
+                for (String updateColumn : old.get(index).keySet()) {
+                    if (rowKeyColumnName.equalsIgnoreCase(updateColumn)) {
+                        updateSubRowKey.add(rowKeyColumnName);
+                    }
+                }
+            }
+            if (updateSubRowKey.isEmpty()) {
+                throw new RuntimeException("没有更新复合主键的RowKey");
+            }
+            StringBuilder oldRowKey = new StringBuilder();
+            StringBuilder newRowKey = new StringBuilder();
+            for (String rowKeyColumnName : rowKeyColumns) {
+                newRowKey.append(r.get(rowKeyColumnName).toString()).append("|");
+                if (!updateSubRowKey.contains(rowKeyColumnName)) {
+                    // 从data取
+                    oldRowKey.append(r.get(rowKeyColumnName).toString()).append("|");
+                } else {
+                    // 从old取
+                    oldRowKey.append(old.get(index).get(rowKeyColumnName).toString()).append("|");
+                }
+            }
+            int len = newRowKey.length();
+            newRowKey.delete(len - 1, len);
+            len = oldRowKey.length();
+            oldRowKey.delete(len - 1, len);
+            byte[] newRowKeyBytes = Bytes.toBytes(newRowKey.toString());
+            byte[] oldRowKeyBytes = Bytes.toBytes(oldRowKey.toString());
+
+            rowKeys.add(oldRowKeyBytes);
+            HRow row = new HRow(newRowKeyBytes);
+            convertData2Row(hbaseOrm, row, r);
+            rows.add(row);
+            complete = false;
+            if (i % config.getHbaseOrm().getCommitBatch() == 0 && !rows.isEmpty()) {
+                hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys);
+
+                hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
+                rowKeys.clear();
+                rows.clear();
+                complete = true;
+            }
+            i++;
+            index++;
+        }
+        if (!complete && !rows.isEmpty()) {
+            hbaseTemplate.deletes(hbaseOrm.getHbaseTable(), rowKeys);
+            hbaseTemplate.puts(hbaseOrm.getHbaseTable(), rows);
+        }
+    }
+
+    /**
+     * 根据对应的类型进行转换
+     * 
+     * @param columnItem 列项配置
+     * @param hbaseOrm hbase映射配置
+     * @param value 值
+     * @return 复合字段rowKey
+     */
+    private static byte[] typeConvert(MappingConfig.ColumnItem columnItem, MappingConfig.HbaseOrm hbaseOrm,
+                                      Object value) {
+        if (value == null) {
+            return null;
+        }
+        byte[] bytes = null;
+        if (columnItem == null || columnItem.getType() == null || "".equals(columnItem.getType())) {
+            if (MappingConfig.Mode.STRING == hbaseOrm.getMode()) {
+                bytes = Bytes.toBytes(value.toString());
+            } else if (MappingConfig.Mode.NATIVE == hbaseOrm.getMode()) {
+                bytes = TypeUtil.toBytes(value);
+            } else if (MappingConfig.Mode.PHOENIX == hbaseOrm.getMode()) {
+                PhType phType = PhType.getType(value.getClass());
+                bytes = PhTypeUtil.toBytes(value, phType);
+            }
+        } else {
+            if (hbaseOrm.getMode() == MappingConfig.Mode.STRING) {
+                bytes = Bytes.toBytes(value.toString());
+            } else if (hbaseOrm.getMode() == MappingConfig.Mode.NATIVE) {
+                Type type = Type.getType(columnItem.getType());
+                bytes = TypeUtil.toBytes(value, type);
+            } else if (hbaseOrm.getMode() == MappingConfig.Mode.PHOENIX) {
+                PhType phType = PhType.getType(columnItem.getType());
+                bytes = PhTypeUtil.toBytes(value, phType);
+            }
+        }
+        return bytes;
+    }
+
+    /**
+     * 获取复合字段作为rowKey的拼接
+     *
+     * @param rowKeyColumns 复合rowK对应的字段
+     * @param data 数据
+     * @return
+     */
+    private static String getRowKeys(String[] rowKeyColumns, Map<String, Object> data) {
+        StringBuilder rowKeyValue = new StringBuilder();
+        for (String rowKeyColumnName : rowKeyColumns) {
+            Object obj = data.get(rowKeyColumnName);
+            if (obj != null) {
+                rowKeyValue.append(obj.toString());
+            }
+            rowKeyValue.append("|");
+        }
+        int len = rowKeyValue.length();
+        if (len > 0) {
+            rowKeyValue.delete(len - 1, len);
+        }
+        return rowKeyValue.toString();
+    }
+
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HRow.java

@@ -0,0 +1,84 @@
+package com.alibaba.otter.canal.client.adapter.hbase.support;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * HBase操作对象类
+ *
+ * @author machengyuan 2018-8-21 下午10:12:34
+ * @version 1.0.0
+ */
+public class HRow {
+
+    private byte[]      rowKey;
+    private List<HCell> cells = new ArrayList<>();
+
+    public HRow(){
+    }
+
+    public HRow(byte[] rowKey){
+        this.rowKey = rowKey;
+    }
+
+    public byte[] getRowKey() {
+        return rowKey;
+    }
+
+    public void setRowKey(byte[] rowKey) {
+        this.rowKey = rowKey;
+    }
+
+    public List<HCell> getCells() {
+        return cells;
+    }
+
+    public void setCells(List<HCell> cells) {
+        this.cells = cells;
+    }
+
+    public void addCell(String family, String qualifier, byte[] value) {
+        HCell hCell = new HCell(family, qualifier, value);
+        cells.add(hCell);
+    }
+
+    public static class HCell {
+
+        private String family;
+        private String qualifier;
+        private byte[] value;
+
+        public HCell(){
+        }
+
+        public HCell(String family, String qualifier, byte[] value){
+            this.family = family;
+            this.qualifier = qualifier;
+            this.value = value;
+        }
+
+        public String getFamily() {
+            return family;
+        }
+
+        public void setFamily(String family) {
+            this.family = family;
+        }
+
+        public String getQualifier() {
+            return qualifier;
+        }
+
+        public void setQualifier(String qualifier) {
+            this.qualifier = qualifier;
+        }
+
+        public byte[] getValue() {
+            return value;
+        }
+
+        public void setValue(byte[] value) {
+            this.value = value;
+        }
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/HbaseTemplate.java

@@ -0,0 +1,159 @@
+package com.alibaba.otter.canal.client.adapter.hbase.support;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * HBase操作模板
+ *
+ * @author machengyuan 2018-8-21 下午10:12:34
+ * @version 1.0.0
+ */
+public class HbaseTemplate {
+
+    private Logger     logger = LoggerFactory.getLogger(this.getClass());
+
+    private Connection conn;
+
+    public HbaseTemplate(Connection conn){
+        this.conn = conn;
+    }
+
+    public boolean tableExists(String tableName) {
+        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+
+            return admin.tableExists(TableName.valueOf(tableName));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void createTable(String tableName, String... familyNames) {
+        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+
+            HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+            // 添加列簇
+            if (familyNames != null) {
+                for (String familyName : familyNames) {
+                    HColumnDescriptor hcd = new HColumnDescriptor(familyName);
+                    desc.addFamily(hcd);
+                }
+            }
+            admin.createTable(desc);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void disableTable(String tableName) {
+        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+            admin.disableTable(tableName);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void deleteTable(String tableName) {
+        try (HBaseAdmin admin = (HBaseAdmin) conn.getAdmin()) {
+            if (admin.isTableEnabled(tableName)) {
+                disableTable(tableName);
+            }
+            admin.deleteTable(tableName);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 插入一行数据
+     * 
+     * @param tableName 表名
+     * @param hRow 行数据对象
+     * @return 是否成功
+     */
+    public Boolean put(String tableName, HRow hRow) {
+        boolean flag = false;
+        try {
+            HTable table = (HTable) conn.getTable(TableName.valueOf(tableName));
+            Put put = new Put(hRow.getRowKey());
+            for (HRow.HCell hCell : hRow.getCells()) {
+                put.addColumn(Bytes.toBytes(hCell.getFamily()), Bytes.toBytes(hCell.getQualifier()), hCell.getValue());
+            }
+            table.put(put);
+            flag = true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return flag;
+
+    }
+
+    /**
+     * 批量插入
+     * 
+     * @param tableName 表名
+     * @param rows 行数据对象集合
+     * @return 是否成功
+     */
+    public Boolean puts(String tableName, List<HRow> rows) {
+        boolean flag = false;
+        try {
+            HTable table = (HTable) conn.getTable(TableName.valueOf(tableName));
+            List<Put> puts = new ArrayList<>();
+            for (HRow hRow : rows) {
+                Put put = new Put(hRow.getRowKey());
+                for (HRow.HCell hCell : hRow.getCells()) {
+                    put.addColumn(Bytes.toBytes(hCell.getFamily()),
+                        Bytes.toBytes(hCell.getQualifier()),
+                        hCell.getValue());
+                }
+                puts.add(put);
+            }
+            if (!puts.isEmpty()) {
+                table.put(puts);
+            }
+            flag = true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return flag;
+    }
+
+    /**
+     * 批量删除数据
+     * 
+     * @param tableName 表名
+     * @param rowKeys rowKey集合
+     * @return 是否成功
+     */
+    public Boolean deletes(String tableName, Set<byte[]> rowKeys) {
+        boolean flag = false;
+        try {
+            HTable table = (HTable) conn.getTable(TableName.valueOf(tableName));
+            List<Delete> deletes = new ArrayList<>();
+            for (byte[] rowKey : rowKeys) {
+                Delete delete = new Delete(rowKey);
+                deletes.add(delete);
+            }
+            if (!deletes.isEmpty()) {
+                table.delete(deletes);
+            }
+            flag = true;
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+        return flag;
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhType.java

@@ -0,0 +1,150 @@
+package com.alibaba.otter.canal.client.adapter.hbase.support;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Date;
+
+/**
+ * Phoenix类型
+ *
+ * @author machengyuan 2018-8-21 下午06:12:34
+ * @version 1.0.0
+ */
+public enum PhType {
+    DEFAULT(-1, "VARCHAR"),
+    UNSIGNED_INT(4, "UNSIGNED_INT"),
+    UNSIGNED_LONG(8, "UNSIGNED_LONG"),
+    UNSIGNED_TINYINT(1, "UNSIGNED_TINYINT"),
+    UNSIGNED_SMALLINT(2, "UNSIGNED_SMALLINT"),
+    UNSIGNED_FLOAT(4, "UNSIGNED_FLOAT"),
+    UNSIGNED_DOUBLE(8, "UNSIGNED_DOUBLE"),
+    INTEGER(4, "INTEGER"),
+    BIGINT(8, "BIGINT"),
+    TINYINT(1, "TINYINT"),
+    SMALLINT(2, "SMALLINT"),
+    FLOAT(4, "FLOAT"),
+    DOUBLE(8, "DOUBLE"),
+    DECIMAL(-1, "DECIMAL"),
+    BOOLEAN(1, "BOOLEAN"),
+    UNSIGNED_TIME(8, "UNSIGNED_TIME"),
+    UNSIGNED_DATE(8, "UNSIGNED_DATE"),
+    UNSIGNED_TIMESTAMP(12, "UNSIGNED_TIMESTAMP"),
+    TIME(8, "TIME"),
+    DATE(8, "DATE"),
+    TIMESTAMP(12, "TIMESTAMP"),
+    VARCHAR(-1, "VARCHAR"),
+    VARBINARY(-1, "VARBINARY");
+
+    /**
+     * -1：长度可变
+     */
+    private int len;
+    private String type;
+
+    PhType(int len, String type) {
+        this.len = len;
+        this.type = type;
+    }
+
+    public int getLen() {
+        return len;
+    }
+
+    public String getType() {
+        return this.type;
+    }
+
+    public static PhType getType(Class<?> javaType) {
+        if (javaType == null) return DEFAULT;
+        PhType phType;
+        if (Integer.class.isAssignableFrom(javaType) || int.class.isAssignableFrom(javaType)) {
+            phType = INTEGER;
+        } else if (Long.class.isAssignableFrom(javaType) || long.class.isAssignableFrom(javaType)) {
+            phType = BIGINT;
+        } else if (Byte.class.isAssignableFrom(javaType) || byte.class.isAssignableFrom(javaType)) {
+            phType = TINYINT;
+        } else if (Short.class.isAssignableFrom(javaType) || short.class.isAssignableFrom(javaType)) {
+            phType = SMALLINT;
+        } else if (Float.class.isAssignableFrom(javaType) || float.class.isAssignableFrom(javaType)) {
+            phType = FLOAT;
+        } else if (Double.class.isAssignableFrom(javaType) || double.class.isAssignableFrom(javaType)) {
+            phType = DOUBLE;
+        } else if (Boolean.class.isAssignableFrom(javaType) || boolean.class.isAssignableFrom(javaType)) {
+            phType = BOOLEAN;
+        } else if (java.sql.Date.class.isAssignableFrom(javaType)) {
+            phType = DATE;
+        } else if (Time.class.isAssignableFrom(javaType)) {
+            phType = DATE;
+        } else if (Timestamp.class.isAssignableFrom(javaType)) {
+            phType = TIMESTAMP;
+        } else if (Date.class.isAssignableFrom(javaType)) {
+            phType = DATE;
+        } else if (byte[].class.isAssignableFrom(javaType)) {
+            phType = VARBINARY;
+        } else if (String.class.isAssignableFrom(javaType)) {
+            phType = VARCHAR;
+        } else if (BigDecimal.class.isAssignableFrom(javaType)) {
+            phType = DECIMAL;
+        }  else if (BigInteger.class.isAssignableFrom(javaType)) {
+            phType = UNSIGNED_LONG;
+        } else {
+            phType = DEFAULT;
+        }
+        return phType;
+    }
+
+    public static PhType getType(String type) {
+        if (type == null) return DEFAULT;
+        PhType phType;
+        if (type.equalsIgnoreCase(UNSIGNED_INT.type)) {
+            phType = UNSIGNED_INT;
+        } else if (type.equalsIgnoreCase(UNSIGNED_LONG.type)) {
+            phType = UNSIGNED_LONG;
+        } else if (type.equalsIgnoreCase(UNSIGNED_TINYINT.type)) {
+            phType = UNSIGNED_TINYINT;
+        } else if (type.equalsIgnoreCase(UNSIGNED_SMALLINT.type)) {
+            phType = UNSIGNED_SMALLINT;
+        } else if (type.equalsIgnoreCase(UNSIGNED_FLOAT.type)) {
+            phType = UNSIGNED_FLOAT;
+        } else if (type.equalsIgnoreCase(UNSIGNED_DOUBLE.type)) {
+            phType = UNSIGNED_DOUBLE;
+        } else if (type.equalsIgnoreCase(INTEGER.type)) {
+            phType = INTEGER;
+        } else if (type.equalsIgnoreCase(BIGINT.type)) {
+            phType = BIGINT;
+        } else if (type.equalsIgnoreCase(TINYINT.type)) {
+            phType = TINYINT;
+        } else if (type.equalsIgnoreCase(SMALLINT.type)) {
+            phType = SMALLINT;
+        } else if (type.equalsIgnoreCase(FLOAT.type)) {
+            phType = FLOAT;
+        } else if (type.equalsIgnoreCase(DOUBLE.type)) {
+            phType = DOUBLE;
+        } else if (type.equalsIgnoreCase(BOOLEAN.type)) {
+            phType = BOOLEAN;
+        } else if (type.equalsIgnoreCase(UNSIGNED_TIME.type)) {
+            phType = UNSIGNED_TIME;
+        } else if (type.equalsIgnoreCase(UNSIGNED_DATE.type)) {
+            phType = UNSIGNED_DATE;
+        } else if (type.equalsIgnoreCase(UNSIGNED_TIMESTAMP.type)) {
+            phType = UNSIGNED_TIMESTAMP;
+        } else if (type.equalsIgnoreCase(TIME.type)) {
+            phType = TIME;
+        } else if (type.equalsIgnoreCase(DATE.type)) {
+            phType = DATE;
+        } else if (type.equalsIgnoreCase(TIMESTAMP.type)) {
+            phType = TIMESTAMP;
+        } else if (type.equalsIgnoreCase(VARCHAR.type)) {
+            phType = VARCHAR;
+        } else if (type.equalsIgnoreCase(VARBINARY.type)) {
+            phType = VARBINARY;
+        } else if (type.equalsIgnoreCase(DECIMAL.type)) {
+            phType = DECIMAL;
+        } else {
+            phType = DEFAULT;
+        }
+        return phType;
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java

@@ -0,0 +1,614 @@
+package com.alibaba.otter.canal.client.adapter.hbase.support;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.MathContext;
+import java.math.RoundingMode;
+import java.sql.Timestamp;
+import java.util.Date;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.joda.time.DateTime;
+
+import com.google.common.math.LongMath;
+
+/**
+ * Phoenix类型转换工具类
+ *
+ * @author machengyuan 2018-8-21 下午06:14:26
+ * @version 1.0.0
+ */
+public class PhTypeUtil {
+
+    public static byte[] toBytes(Object v, PhType phType) {
+        if (v == null) return null;
+        byte[] b = null;
+        if (phType == PhType.DEFAULT) {
+            PhType phType1 = PhType.getType(v.getClass());
+            if (phType1 != null && phType1 != PhType.DEFAULT) {
+                toBytes(v, phType1);
+            }
+        } else if (phType == PhType.INTEGER) {
+            b = new byte[Bytes.SIZEOF_INT];
+            encodeInt(((Number) v).intValue(), b, 0);
+        } else if (phType == PhType.UNSIGNED_INT) {
+            b = new byte[Bytes.SIZEOF_INT];
+            encodeUnsignedInt(((Number) v).intValue(), b, 0);
+        } else if (phType == PhType.BIGINT) {
+            b = new byte[Bytes.SIZEOF_LONG];
+            encodeLong(((Number) v).longValue(), b, 0);
+        } else if (phType == PhType.UNSIGNED_LONG) {
+            b = new byte[Bytes.SIZEOF_LONG];
+            encodeUnsignedLong(((Number) v).longValue(), b, 0);
+        } else if (phType == PhType.SMALLINT) {
+            b = new byte[Bytes.SIZEOF_SHORT];
+            encodeShort(((Number) v).shortValue(), b, 0);
+        } else if (phType == PhType.UNSIGNED_SMALLINT) {
+            b = new byte[Bytes.SIZEOF_SHORT];
+            encodeUnsignedShort(((Number) v).shortValue(), b, 0);
+        } else if (phType == PhType.TINYINT) {
+            b = new byte[Bytes.SIZEOF_BYTE];
+            encodeByte(((Number) v).byteValue(), b, 0);
+        } else if (phType == PhType.UNSIGNED_TINYINT) {
+            b = new byte[Bytes.SIZEOF_BYTE];
+            encodeUnsignedByte(((Number) v).byteValue(), b, 0);
+        } else if (phType == PhType.FLOAT) {
+            b = new byte[Bytes.SIZEOF_FLOAT];
+            encodeFloat(((Number) v).floatValue(), b, 0);
+        } else if (phType == PhType.UNSIGNED_FLOAT) {
+            b = new byte[Bytes.SIZEOF_FLOAT];
+            encodeUnsignedFloat(((Number) v).floatValue(), b, 0);
+        } else if (phType == PhType.DOUBLE) {
+            b = new byte[Bytes.SIZEOF_DOUBLE];
+            encodeDouble(((Number) v).doubleValue(), b, 0);
+        } else if (phType == PhType.UNSIGNED_DOUBLE) {
+            b = new byte[Bytes.SIZEOF_DOUBLE];
+            encodeUnsignedDouble(((Number) v).doubleValue(), b, 0);
+        } else if (phType == PhType.BOOLEAN) {
+            if ((Boolean) v) {
+                b = new byte[] { 1 };
+            } else {
+                b = new byte[] { 0 };
+            }
+        } else if (phType == PhType.TIME || phType == PhType.DATE) {
+            b = new byte[Bytes.SIZEOF_LONG];
+            encodeDate(v, b, 0);
+        } else if (phType == PhType.TIMESTAMP) {
+            b = new byte[Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT];
+            encodeTimestamp(v, b, 0);
+        } else if (phType == PhType.UNSIGNED_TIME || phType == PhType.UNSIGNED_DATE) {
+            b = new byte[Bytes.SIZEOF_LONG];
+            encodeUnsignedDate(v, b, 0);
+        } else if (phType == PhType.UNSIGNED_TIMESTAMP) {
+            b = new byte[Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT];
+            encodeUnsignedTimestamp(v, b, 0);
+        } else if (phType == PhType.VARBINARY) {
+            b = (byte[]) v;
+        } else if (phType == PhType.VARCHAR) {
+            b = Bytes.toBytes(v.toString());
+        } else if (phType == PhType.DECIMAL) {
+            if (v instanceof BigDecimal) {
+                b = encodeDecimal(v);
+            } else if (v instanceof Number) {
+                b = encodeDecimal(new BigDecimal(v.toString()));
+            }
+        }
+        return b;
+    }
+
+    public static Object toObject(byte[] b, PhType phType) {
+        if (b == null) return null;
+        Object v = null;
+        if (phType == PhType.INTEGER) {
+            v = decodeInt(b, 0);
+        } else if (phType == PhType.UNSIGNED_INT) {
+            v = decodeUnsignedInt(b, 0);
+        } else if (phType == PhType.BIGINT) {
+            v = decodeLong(b, 0);
+        } else if (phType == PhType.UNSIGNED_LONG) {
+            v = decodeUnsignedLong(b, 0);
+        } else if (phType == PhType.SMALLINT) {
+            v = decodeShort(b, 0);
+        } else if (phType == PhType.UNSIGNED_SMALLINT) {
+            v = decodeUnsignedShort(b, 0);
+        } else if (phType == PhType.TINYINT) {
+            v = decodeByte(b, 0);
+        } else if (phType == PhType.UNSIGNED_TINYINT) {
+            v = decodeUnsignedByte(b, 0);
+        } else if (phType == PhType.FLOAT) {
+            v = decodeFloat(b, 0);
+        } else if (phType == PhType.UNSIGNED_FLOAT) {
+            v = decodeUnsignedFloat(b, 0);
+        } else if (phType == PhType.DOUBLE) {
+            v = decodeDouble(b, 0);
+        } else if (phType == PhType.UNSIGNED_DOUBLE) {
+            v = decodeUnsignedDouble(b, 0);
+        } else if (phType == PhType.BOOLEAN) {
+            checkForSufficientLength(b, 0, Bytes.SIZEOF_BOOLEAN);
+            if (b[0] == 1) {
+                v = true;
+            } else if (b[0] == 0) {
+                v = false;
+            }
+        } else if (phType == PhType.TIME || phType == PhType.DATE) {
+            v = new Date(decodeLong(b, 0));
+        } else if (phType == PhType.TIMESTAMP) {
+            long millisDeserialized = decodeLong(b, 0);
+            Timestamp ts = new Timestamp(millisDeserialized);
+            int nanosDeserialized = decodeUnsignedInt(b, Bytes.SIZEOF_LONG);
+            ts.setNanos(nanosDeserialized < 1000000 ? ts.getNanos() + nanosDeserialized : nanosDeserialized);
+            v = ts;
+        } else if (phType == PhType.UNSIGNED_TIME || phType == PhType.UNSIGNED_DATE) {
+            v = new Date(decodeUnsignedLong(b, 0));
+        } else if (phType == PhType.UNSIGNED_TIMESTAMP) {
+            long millisDeserialized = decodeUnsignedLong(b, 0);
+            Timestamp ts = new Timestamp(millisDeserialized);
+            int nanosDeserialized = decodeUnsignedInt(b, Bytes.SIZEOF_LONG);
+            ts.setNanos(nanosDeserialized < 1000000 ? ts.getNanos() + nanosDeserialized : nanosDeserialized);
+            v = ts;
+        } else if (phType == PhType.VARBINARY) {
+            v = b;
+        } else if (phType == PhType.VARCHAR || phType == PhType.DEFAULT) {
+            v = Bytes.toString(b);
+        } else if (phType == PhType.DECIMAL) {
+            v = decodeDecimal(b, 0, b.length);
+        }
+
+        return v;
+    }
+
+    private static int decodeInt(byte[] bytes, int o) {
+        checkForSufficientLength(bytes, o, Bytes.SIZEOF_INT);
+        int v;
+        v = bytes[o] ^ 0x80; // Flip sign bit back
+        for (int i = 1; i < Bytes.SIZEOF_INT; i++) {
+            v = (v << 8) + (bytes[o + i] & 0xff);
+        }
+        return v;
+    }
+
+    private static int encodeInt(int v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
+        b[o + 0] = (byte) ((v >> 24) ^ 0x80); // Flip sign bit so that INTEGER is binary comparable
+        b[o + 1] = (byte) (v >> 16);
+        b[o + 2] = (byte) (v >> 8);
+        b[o + 3] = (byte) v;
+        return Bytes.SIZEOF_INT;
+    }
+
+    private static int decodeUnsignedInt(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
+
+        int v = Bytes.toInt(b, o);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        return v;
+    }
+
+    private static int encodeUnsignedInt(int v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        Bytes.putInt(b, o, v);
+        return Bytes.SIZEOF_INT;
+    }
+
+    private static long decodeLong(byte[] bytes, int o) {
+        checkForSufficientLength(bytes, o, Bytes.SIZEOF_LONG);
+        long v;
+        byte b = bytes[o];
+        v = b ^ 0x80; // Flip sign bit back
+        for (int i = 1; i < Bytes.SIZEOF_LONG; i++) {
+            b = bytes[o + i];
+            v = (v << 8) + (b & 0xff);
+        }
+        return v;
+    }
+
+    private static int encodeLong(long v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
+        b[o + 0] = (byte) ((v >> 56) ^ 0x80); // Flip sign bit so that INTEGER is binary comparable
+        b[o + 1] = (byte) (v >> 48);
+        b[o + 2] = (byte) (v >> 40);
+        b[o + 3] = (byte) (v >> 32);
+        b[o + 4] = (byte) (v >> 24);
+        b[o + 5] = (byte) (v >> 16);
+        b[o + 6] = (byte) (v >> 8);
+        b[o + 7] = (byte) v;
+        return Bytes.SIZEOF_LONG;
+    }
+
+    private static long decodeUnsignedLong(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
+        long v = 0;
+        for (int i = o; i < o + Bytes.SIZEOF_LONG; i++) {
+            v <<= 8;
+            v ^= b[i] & 0xFF;
+        }
+
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        return v;
+    }
+
+    private static int encodeUnsignedLong(long v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        Bytes.putLong(b, o, v);
+        return Bytes.SIZEOF_LONG;
+    }
+
+    private static short decodeShort(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT);
+        int v;
+        v = b[o] ^ 0x80; // Flip sign bit back
+        for (int i = 1; i < Bytes.SIZEOF_SHORT; i++) {
+            v = (v << 8) + (b[o + i] & 0xff);
+        }
+        return (short) v;
+    }
+
+    private static int encodeShort(short v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT);
+        b[o + 0] = (byte) ((v >> 8) ^ 0x80); // Flip sign bit so that Short is binary comparable
+        b[o + 1] = (byte) v;
+        return Bytes.SIZEOF_SHORT;
+    }
+
+    private static short decodeUnsignedShort(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT);
+        short v = Bytes.toShort(b, o);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        return v;
+    }
+
+    private static int encodeUnsignedShort(short v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_SHORT);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        Bytes.putShort(b, o, v);
+        return Bytes.SIZEOF_SHORT;
+    }
+
+    private static byte decodeByte(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_BYTE);
+        int v;
+        v = b[o] ^ 0x80; // Flip sign bit back
+        return (byte) v;
+    }
+
+    private static int encodeByte(byte v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_BYTE);
+        b[o] = (byte) (v ^ 0x80); // Flip sign bit so that Short is binary comparable
+        return Bytes.SIZEOF_BYTE;
+    }
+
+    private static byte decodeUnsignedByte(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_BYTE);
+        byte v = b[o];
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        return v;
+    }
+
+    private static int encodeUnsignedByte(byte v, byte[] b, int o) {
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        Bytes.putByte(b, o, v);
+        return Bytes.SIZEOF_BYTE;
+    }
+
+    private static float decodeFloat(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
+        int value;
+        value = Bytes.toInt(b, o);
+        value--;
+        value ^= (~value >> Integer.SIZE - 1) | Integer.MIN_VALUE;
+        return Float.intBitsToFloat(value);
+    }
+
+    private static int encodeFloat(float v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_FLOAT);
+        int i = Float.floatToIntBits(v);
+        i = (i ^ ((i >> Integer.SIZE - 1) | Integer.MIN_VALUE)) + 1;
+        Bytes.putInt(b, o, i);
+        return Bytes.SIZEOF_FLOAT;
+    }
+
+    private static float decodeUnsignedFloat(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_FLOAT);
+        float v = Bytes.toFloat(b, o);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        return v;
+    }
+
+    private static int encodeUnsignedFloat(float v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_FLOAT);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        Bytes.putFloat(b, o, v);
+        return Bytes.SIZEOF_FLOAT;
+    }
+
+    private static double decodeDouble(byte[] bytes, int o) {
+        checkForSufficientLength(bytes, o, Bytes.SIZEOF_LONG);
+        long l;
+        l = Bytes.toLong(bytes, o);
+        l--;
+        l ^= (~l >> Long.SIZE - 1) | Long.MIN_VALUE;
+        return Double.longBitsToDouble(l);
+    }
+
+    private static int encodeDouble(double v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
+        long l = Double.doubleToLongBits(v);
+        l = (l ^ ((l >> Long.SIZE - 1) | Long.MIN_VALUE)) + 1;
+        Bytes.putLong(b, o, l);
+        return Bytes.SIZEOF_LONG;
+    }
+
+    private static double decodeUnsignedDouble(byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_DOUBLE);
+        double v = Bytes.toDouble(b, o);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        return v;
+    }
+
+    private static int encodeUnsignedDouble(double v, byte[] b, int o) {
+        checkForSufficientLength(b, o, Bytes.SIZEOF_DOUBLE);
+        if (v < 0) {
+            throw new RuntimeException();
+        }
+        Bytes.putDouble(b, o, v);
+        return Bytes.SIZEOF_DOUBLE;
+    }
+
+    private static int encodeDate(Object v, byte[] b, int o) {
+        if (v instanceof Date) {
+            encodeLong(((Date) v).getTime(), b, 0);
+        } else if (v instanceof String) {
+            String dateStr = (String) v;
+            Date date;
+            try {
+                date = parseDatetime(dateStr);
+                if (date != null) {
+                    encodeLong(date.getTime(), b, 0);
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return Bytes.SIZEOF_LONG;
+    }
+
+    private static int encodeTimestamp(Object v, byte[] b, int o) {
+        if (v instanceof Timestamp) {
+            Timestamp ts = (Timestamp) v;
+            encodeLong(ts.getTime(), b, o);
+            Bytes.putInt(b, Bytes.SIZEOF_LONG, ts.getNanos() % 1000000);
+        } else {
+            encodeDate(v, b, o);
+        }
+        return Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT;
+    }
+
+    private static int encodeUnsignedDate(Object v, byte[] b, int o) {
+        if (v instanceof Date) {
+            encodeUnsignedLong(((Date) v).getTime(), b, 0);
+        } else if (v instanceof String) {
+            String dateStr = (String) v;
+            Date date;
+            try {
+                date = parseDatetime(dateStr);
+                if (date != null) {
+                    encodeUnsignedLong(date.getTime(), b, 0);
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return Bytes.SIZEOF_LONG;
+    }
+
+    private static int encodeUnsignedTimestamp(Object v, byte[] b, int o) {
+        if (v instanceof Timestamp) {
+            Timestamp ts = (Timestamp) v;
+            encodeUnsignedLong(ts.getTime(), b, o);
+            Bytes.putInt(b, Bytes.SIZEOF_LONG, ts.getNanos() % 1000000);
+        } else {
+            encodeUnsignedDate(v, b, o);
+        }
+        return Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT;
+    }
+
+    private static byte[] encodeDecimal(Object object) {
+        if (object == null) {
+            return new byte[0];
+        }
+        BigDecimal v = (BigDecimal) object;
+        v = v.round(DEFAULT_MATH_CONTEXT).stripTrailingZeros();
+        int len = getLength(v);
+        byte[] result = new byte[Math.min(len, 21)];
+        decimalToBytes(v, result, 0, len);
+        return result;
+    }
+
+    private static BigDecimal decodeDecimal(byte[] bytes, int offset, int length) {
+        if (length == 1 && bytes[offset] == ZERO_BYTE) {
+            return BigDecimal.ZERO;
+        }
+        int signum = ((bytes[offset] & 0x80) == 0) ? -1 : 1;
+        int scale;
+        int index;
+        int digitOffset;
+        long multiplier = 100L;
+        int begIndex = offset + 1;
+        if (signum == 1) {
+            scale = (byte) (((bytes[offset] & 0x7F) - 65) * -2);
+            index = offset + length;
+            digitOffset = POS_DIGIT_OFFSET;
+        } else {
+            scale = (byte) ((~bytes[offset] - 65 - 128) * -2);
+            index = offset + length - (bytes[offset + length - 1] == NEG_TERMINAL_BYTE ? 1 : 0);
+            digitOffset = -NEG_DIGIT_OFFSET;
+        }
+        length = index - offset;
+        long l = signum * bytes[--index] - digitOffset;
+        if (l % 10 == 0) { // trailing zero
+            scale--; // drop trailing zero and compensate in the scale
+            l /= 10;
+            multiplier = 10;
+        }
+        // Use long arithmetic for as long as we can
+        while (index > begIndex) {
+            if (l >= MAX_LONG_FOR_DESERIALIZE || multiplier >= Long.MAX_VALUE / 100) {
+                multiplier = LongMath.divide(multiplier, 100L, RoundingMode.UNNECESSARY);
+                break; // Exit loop early so we don't overflow our multiplier
+            }
+            int digit100 = signum * bytes[--index] - digitOffset;
+            l += digit100 * multiplier;
+            multiplier = LongMath.checkedMultiply(multiplier, 100);
+        }
+
+        BigInteger bi;
+        // If still more digits, switch to BigInteger arithmetic
+        if (index > begIndex) {
+            bi = BigInteger.valueOf(l);
+            BigInteger biMultiplier = BigInteger.valueOf(multiplier).multiply(ONE_HUNDRED);
+            do {
+                int digit100 = signum * bytes[--index] - digitOffset;
+                bi = bi.add(biMultiplier.multiply(BigInteger.valueOf(digit100)));
+                biMultiplier = biMultiplier.multiply(ONE_HUNDRED);
+            } while (index > begIndex);
+            if (signum == -1) {
+                bi = bi.negate();
+            }
+        } else {
+            bi = BigInteger.valueOf(l * signum);
+        }
+        // Update the scale based on the precision
+        scale += (length - 2) * 2;
+        BigDecimal v = new BigDecimal(bi, scale);
+        return v;
+    }
+
+    private static int getLength(BigDecimal v) {
+        int signum = v.signum();
+        if (signum == 0) { // Special case for zero
+            return 1;
+        }
+        return (signum < 0 ? 2 : 1) + (v.precision() + 1 + (v.scale() % 2 == 0 ? 0 : 1)) / 2;
+    }
+
+    private static final int         MAX_PRECISION            = 38;
+    private static final MathContext DEFAULT_MATH_CONTEXT     = new MathContext(MAX_PRECISION, RoundingMode.HALF_UP);
+    private static final Integer     MAX_BIG_DECIMAL_BYTES    = 21;
+    private static final byte        ZERO_BYTE                = (byte) 0x80;
+    private static final byte        NEG_TERMINAL_BYTE        = (byte) 102;
+    private static final int         EXP_BYTE_OFFSET          = 65;
+    private static final int         POS_DIGIT_OFFSET         = 1;
+    private static final int         NEG_DIGIT_OFFSET         = 101;
+    private static final BigInteger  MAX_LONG                 = BigInteger.valueOf(Long.MAX_VALUE);
+    private static final BigInteger  MIN_LONG                 = BigInteger.valueOf(Long.MIN_VALUE);
+    private static final BigInteger  ONE_HUNDRED              = BigInteger.valueOf(100);
+    private static final long        MAX_LONG_FOR_DESERIALIZE = Long.MAX_VALUE / 1000;
+
+    private static int decimalToBytes(BigDecimal v, byte[] result, final int offset, int length) {
+        int signum = v.signum();
+        if (signum == 0) {
+            result[offset] = ZERO_BYTE;
+            return 1;
+        }
+        int index = offset + length;
+        int scale = v.scale();
+        int expOffset = scale % 2 * (scale < 0 ? -1 : 1);
+        int multiplyBy;
+        BigInteger divideBy;
+        if (expOffset == 0) {
+            multiplyBy = 1;
+            divideBy = ONE_HUNDRED;
+        } else {
+            multiplyBy = 10;
+            divideBy = BigInteger.TEN;
+        }
+        // Normalize the scale based on what is necessary to end up with a base 100
+        // decimal (i.e. 10.123e3)
+        int digitOffset;
+        BigInteger compareAgainst;
+        if (signum == 1) {
+            digitOffset = POS_DIGIT_OFFSET;
+            compareAgainst = MAX_LONG;
+            scale -= (length - 2) * 2;
+            result[offset] = (byte) ((-(scale + expOffset) / 2 + EXP_BYTE_OFFSET) | 0x80);
+        } else {
+            digitOffset = NEG_DIGIT_OFFSET;
+            compareAgainst = MIN_LONG;
+            // Scale adjustment shouldn't include terminal byte in length
+            scale -= (length - 2 - 1) * 2;
+            result[offset] = (byte) (~(-(scale + expOffset) / 2 + EXP_BYTE_OFFSET + 128) & 0x7F);
+            if (length <= MAX_BIG_DECIMAL_BYTES) {
+                result[--index] = NEG_TERMINAL_BYTE;
+            } else {
+                // Adjust length and offset down because we don't have enough room
+                length = MAX_BIG_DECIMAL_BYTES;
+                index = offset + length;
+            }
+        }
+        BigInteger bi = v.unscaledValue();
+        // Use BigDecimal arithmetic until we can fit into a long
+        while (bi.compareTo(compareAgainst) * signum > 0) {
+            BigInteger[] dandr = bi.divideAndRemainder(divideBy);
+            bi = dandr[0];
+            int digit = dandr[1].intValue();
+            result[--index] = (byte) (digit * multiplyBy + digitOffset);
+            multiplyBy = 1;
+            divideBy = ONE_HUNDRED;
+        }
+        long l = bi.longValue();
+        do {
+            long divBy = 100 / multiplyBy;
+            long digit = l % divBy;
+            l /= divBy;
+            result[--index] = (byte) (digit * multiplyBy + digitOffset);
+            multiplyBy = 1;
+        } while (l != 0);
+
+        return length;
+    }
+
+    private static void checkForSufficientLength(byte[] b, int offset, int requiredLength) {
+        if (b.length < offset + requiredLength) {
+            throw new RuntimeException(
+                "Expected length of at least " + requiredLength + " bytes, but had " + (b.length - offset));
+        }
+    }
+
+    private static Date parseDatetime(String dateStr) {
+        Date date = null;
+        int len = dateStr.length();
+        if (len == 10 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-') {
+            date = new DateTime(dateStr).toDate();
+        } else if (len == 8 && dateStr.charAt(2) == ':' && dateStr.charAt(5) == ':') {
+            date = new DateTime("T" + dateStr).toDate();
+        } else if (len >= 19 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-' && dateStr.charAt(13) == ':'
+                   && dateStr.charAt(16) == ':') {
+            date = new DateTime(dateStr.replace(" ", "T")).toDate();
+        }
+        return date;
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/Type.java

@@ -0,0 +1,104 @@
+package com.alibaba.otter.canal.client.adapter.hbase.support;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/**
+ * Java类型
+ *
+ * @author machengyuan 2018-8-21 下午06:11:36
+ * @version 1.0.0
+ */
+public enum Type {
+    DEFAULT("STRING"),
+    STRING("STRING"),
+    INTEGER("INTEGER"),
+    LONG("LONG"),
+    SHORT("SHORT"),
+    BOOLEAN("BOOLEAN"),
+    FLOAT("FLOAT"),
+    DOUBLE("DOUBLE"),
+    BIGDECIMAL("BIGDECIMAL"),
+    DATE("DATE"),
+    BYTE("BYTE"),
+    BYTES("BYTES");
+
+    private String type;
+
+    Type(String type) {
+        this.type = type;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public static Type getType(String type) {
+        if (type == null) {
+            return DEFAULT;
+        }
+        Type res;
+        if (type.equalsIgnoreCase("STRING")) {
+            res = STRING;
+        } else if (type.equalsIgnoreCase("INTEGER")) {
+            res = INTEGER;
+        } else if (type.equalsIgnoreCase("LONG")) {
+            res = LONG;
+        } else if (type.equalsIgnoreCase("SHORT")) {
+            res = SHORT;
+        } else if (type.equalsIgnoreCase("BOOLEAN")) {
+            res = BOOLEAN;
+        } else if (type.equalsIgnoreCase("FLOAT")) {
+            res = FLOAT;
+        } else if (type.equalsIgnoreCase("DOUBLE")) {
+            res = DOUBLE;
+        } else if (type.equalsIgnoreCase("BIGDECIMAL")) {
+            res = BIGDECIMAL;
+        } else if (type.equalsIgnoreCase("DATE")) {
+            res = DATE;
+        } else if (type.equalsIgnoreCase("BYTE")) {
+            res = BYTE;
+        } else if (type.equalsIgnoreCase("BYTES")) {
+            res = BYTES;
+        } else {
+            res = DEFAULT;
+        }
+        return res;
+    }
+
+    public static Type getType(Class<?> javaType) {
+        if (javaType == null) {
+            return DEFAULT;
+        }
+        Type type;
+        if (Integer.class.isAssignableFrom(javaType) || int.class.isAssignableFrom(javaType)) {
+            type = INTEGER;
+        } else if (Long.class.isAssignableFrom(javaType) || long.class.isAssignableFrom(javaType)) {
+            type = LONG;
+        } else if (Byte.class.isAssignableFrom(javaType) || byte.class.isAssignableFrom(javaType)) {
+            type = BYTE;
+        } else if (Short.class.isAssignableFrom(javaType) || short.class.isAssignableFrom(javaType)) {
+            type = SHORT;
+        } else if (Float.class.isAssignableFrom(javaType) || float.class.isAssignableFrom(javaType)) {
+            type = FLOAT;
+        } else if (Double.class.isAssignableFrom(javaType) || double.class.isAssignableFrom(javaType)) {
+            type = DOUBLE;
+        } else if (Boolean.class.isAssignableFrom(javaType) || boolean.class.isAssignableFrom(javaType)) {
+            type = BOOLEAN;
+        } else if (Date.class.isAssignableFrom(javaType)) {
+            type = DATE;
+        } else if (byte[].class.isAssignableFrom(javaType)) {
+            type = BYTES;
+        } else if (String.class.isAssignableFrom(javaType)) {
+            type = STRING;
+        } else if (BigDecimal.class.isAssignableFrom(javaType)) {
+            type = BIGDECIMAL;
+        } else if (BigInteger.class.isAssignableFrom(javaType)) {
+            type = LONG;
+        } else {
+            type = DEFAULT;
+        }
+        return type;
+    }
+}

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/TypeUtil.java

@@ -0,0 +1,188 @@
+package com.alibaba.otter.canal.client.adapter.hbase.support;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Date;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Java类型转换工具类
+ *
+ * @author machengyuan 2018-8-21 下午06:12:34
+ * @version 1.0.0
+ */
+public class TypeUtil {
+
+    public static byte[] toBytes(Object obj) {
+        if (obj == null) {
+            return null;
+        }
+        byte[] bytes;
+        if (obj instanceof String) {
+            bytes = Bytes.toBytes((String) obj);
+        } else if (obj instanceof Integer) {
+            bytes = Bytes.toBytes((Integer) obj);
+        } else if (obj instanceof Long) {
+            bytes = Bytes.toBytes((Long) obj);
+        } else if (obj instanceof Short) {
+            bytes = Bytes.toBytes((Short) obj);
+        } else if (obj instanceof Boolean) {
+            bytes = Bytes.toBytes((Boolean) obj);
+        } else if (obj instanceof Float) {
+            bytes = Bytes.toBytes((Float) obj);
+        } else if (obj instanceof Double) {
+            bytes = Bytes.toBytes((Double) obj);
+        } else if (obj instanceof BigDecimal) {
+            bytes = Bytes.toBytes((BigDecimal) obj);
+        } else if (obj instanceof BigInteger) {
+            bytes = Bytes.toBytes(((BigInteger) obj).longValue());
+        } else if (obj instanceof Date) {
+            bytes = Bytes.toBytes(((Date) obj).getTime());
+        } else if (obj instanceof Byte) {
+            bytes = new byte[] { (byte) obj };
+        } else if (obj instanceof byte[]) {
+            bytes = (byte[]) obj;
+        } else {
+            // 其余类型统一转换为string
+            bytes = Bytes.toBytes(obj.toString());
+        }
+        return bytes;
+    }
+
+    public static byte[] toBytes(Object v, Type type) {
+        if (v == null) {
+            return null;
+        }
+        byte[] b = null;
+        if (type == Type.DEFAULT) {
+            Type type1 = Type.getType(v.getClass());
+            if (type1 != null && type1 != Type.DEFAULT) {
+                b = toBytes(v, type1);
+            }
+        } else if (type == Type.STRING) {
+            b = Bytes.toBytes(v.toString());
+        } else if (type == Type.INTEGER) {
+            b = Bytes.toBytes(((Number) v).intValue());
+        } else if (type == Type.LONG) {
+            b = Bytes.toBytes(((Number) v).longValue());
+        } else if (type == Type.SHORT) {
+            b = Bytes.toBytes(((Number) v).shortValue());
+        } else if (type == Type.BYTE) {
+            b = Bytes.toBytes(((Number) v).byteValue());
+        } else if (type == Type.FLOAT) {
+            b = Bytes.toBytes(((Number) v).floatValue());
+        } else if (type == Type.DOUBLE) {
+            b = Bytes.toBytes(((Number) v).doubleValue());
+        } else if (type == Type.BOOLEAN) {
+            b = Bytes.toBytes(((Boolean) v));
+        } else if (type == Type.DATE) {
+            b = Bytes.toBytes(((Date) v).getTime());
+        } else if (type == Type.BYTES) {
+            b = (byte[]) v;
+        } else if (type == Type.BIGDECIMAL) {
+            if (v instanceof BigDecimal) {
+                b = Bytes.toBytes((BigDecimal) v);
+            } else {
+                b = Bytes.toBytes(new BigDecimal(v.toString()));
+            }
+        }
+        return b;
+    }
+
+    public static <T> T toObject(byte[] bytes, Class<T> clazz) {
+        if (bytes == null) {
+            return null;
+        }
+        Object res;
+        if (String.class.isAssignableFrom(clazz)) {
+            res = Bytes.toString(bytes);
+        } else if (Integer.class.isAssignableFrom(clazz)) {
+            res = Bytes.toInt(bytes);
+        } else if (Long.class.isAssignableFrom(clazz)) {
+            res = Bytes.toLong(bytes);
+        } else if (Short.class.isAssignableFrom(clazz)) {
+            res = Bytes.toShort(bytes);
+        } else if (Boolean.class.isAssignableFrom(clazz)) {
+            res = Bytes.toBoolean(bytes);
+        } else if (Float.class.isAssignableFrom(clazz)) {
+            res = Bytes.toFloat(bytes);
+        } else if (Double.class.isAssignableFrom(clazz)) {
+            res = Bytes.toDouble(bytes);
+        } else if (BigDecimal.class.isAssignableFrom(clazz)) {
+            res = Bytes.toBigDecimal(bytes);
+        } else if (BigInteger.class.isAssignableFrom(clazz)) {
+            res = Bytes.toLong(bytes);
+        } else if (java.sql.Date.class.isAssignableFrom(clazz)) {
+            long ts = Bytes.toLong(bytes);
+            res = new java.sql.Date(ts);
+        } else if (Time.class.isAssignableFrom(clazz)) {
+            long ts = Bytes.toLong(bytes);
+            res = new Time(ts);
+        } else if (Timestamp.class.isAssignableFrom(clazz)) {
+            long ts = Bytes.toLong(bytes);
+            res = new Timestamp(ts);
+        } else if (Date.class.isAssignableFrom(clazz)) {
+            long ts = Bytes.toLong(bytes);
+            res = new Date(ts);
+        } else if (Byte.class.isAssignableFrom(clazz)) {
+            res = bytes[0];
+        } else {
+            throw new IllegalArgumentException("mismatch class type");
+        }
+        // noinspection unchecked
+        return (T) res;
+    }
+
+    public static <T> T toObject(byte[] bytes, Type type) {
+        if (bytes == null) {
+            return null;
+        }
+        Object res = null;
+        if (type == Type.STRING || type == Type.DEFAULT) {
+            res = Bytes.toString(bytes);
+        } else if (type == Type.INTEGER) {
+            if (bytes.length == Bytes.SIZEOF_INT) {
+                res = Bytes.toInt(bytes);
+            }
+        } else if (type == Type.LONG) {
+            if (bytes.length == Bytes.SIZEOF_LONG) {
+                res = Bytes.toLong(bytes);
+            }
+        } else if (type == Type.SHORT) {
+            if (bytes.length == Bytes.SIZEOF_SHORT) {
+                res = Bytes.toShort(bytes);
+            }
+        } else if (type == Type.BYTE) {
+            if (bytes.length == Bytes.SIZEOF_BYTE) {
+                res = bytes[0];
+            }
+        } else if (type == Type.FLOAT) {
+            if (bytes.length == Bytes.SIZEOF_FLOAT) {
+                res = Bytes.toFloat(bytes);
+            }
+        } else if (type == Type.DOUBLE) {
+            if (bytes.length == Bytes.SIZEOF_DOUBLE) {
+                res = Bytes.toDouble(bytes);
+            }
+        } else if (type == Type.BOOLEAN) {
+            if (bytes.length == Bytes.SIZEOF_BOOLEAN) {
+                res = Bytes.toBoolean(bytes);
+            }
+        } else if (type == Type.DATE) {
+            if (bytes.length == Bytes.SIZEOF_LONG) {
+                res = new Date(Bytes.toLong(bytes));
+            }
+        } else if (type == Type.BYTES) {
+            res = bytes;
+        } else if (type == Type.BIGDECIMAL) {
+            res = Bytes.toBigDecimal(bytes);
+        } else {
+            throw new IllegalArgumentException("mismatch class type");
+        }
+        // noinspection unchecked
+        return (T) res;
+    }
+}

client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter

@@ -0,0 +1 @@
+hbase=com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter

client-adapter/hbase/src/main/resources/hbase-mapping/configs.conf

@@ -0,0 +1,5 @@
+# 详细映射配置
+mytest_person2.yml
+
+# 简易配置, 只用指定数据库名.表名, 详细配置全部使用默认
+mytest.person

client-adapter/hbase/src/main/resources/hbase-mapping/mytest_person2.yml

@@ -0,0 +1,56 @@
+hbaseOrm:
+  mode: PHOENIX  #NATIVE   #STRING
+  database: mytest  # 数据库名
+  table: person2     # 数据库表名
+  hbaseTable: MYTEST.PERSON2   # HBase表名
+  family: CF  # 默认统一Family名称
+  uppercaseQualifier: true  # 字段名转大写, 默认为true
+  commitBatch: 3000 # 批量提交的大小
+  #rowKey: id,type  # 复合字段rowKey不能和columns中的rowKey重复
+  columns:
+    # 数据库字段:HBase对应字段
+    id: ROWKEY$UNSIGNED_LONG
+    name: NAME
+    email: EMAIL
+    type: $DECIMAL
+    c_time: C_TIME$UNSIGNED_TIMESTAMP
+    birthday: BIRTHDAY$DATE
+
+# -- NATIVE类型
+# $DEFAULT
+# $STRING
+# $INTEGER
+# $LONG
+# $SHORT
+# $BOOLEAN
+# $FLOAT
+# $DOUBLE
+# $BIGDECIMAL
+# $DATE
+# $BYTE
+# $BYTES
+
+# -- PHOENIX类型
+# $DEFAULT                  对应PHOENIX里的VARCHAR
+# $UNSIGNED_INT             对应PHOENIX里的UNSIGNED_INT           4字节
+# $UNSIGNED_LONG            对应PHOENIX里的UNSIGNED_LONG          8字节
+# $UNSIGNED_TINYINT         对应PHOENIX里的UNSIGNED_TINYINT       1字节
+# $UNSIGNED_SMALLINT        对应PHOENIX里的UNSIGNED_SMALLINT      2字节
+# $UNSIGNED_FLOAT           对应PHOENIX里的UNSIGNED_FLOAT         4字节
+# $UNSIGNED_DOUBLE          对应PHOENIX里的UNSIGNED_DOUBLE        8字节
+# $INTEGER                  对应PHOENIX里的INTEGER                4字节
+# $BIGINT                   对应PHOENIX里的BIGINT                 8字节
+# $TINYINT                  对应PHOENIX里的TINYINT                1字节
+# $SMALLINT                 对应PHOENIX里的SMALLINT               2字节
+# $FLOAT                    对应PHOENIX里的FLOAT                  4字节
+# DOUBLE                    对应PHOENIX里的DOUBLE                 8字节
+# $BOOLEAN                  对应PHOENIX里的BOOLEAN                1字节
+# $TIME                     对应PHOENIX里的TIME                   8字节
+# $DATE                     对应PHOENIX里的DATE                   8字节
+# $TIMESTAMP                对应PHOENIX里的TIMESTAMP              12字节
+# $UNSIGNED_TIME            对应PHOENIX里的UNSIGNED_TIME          8字节
+# $UNSIGNED_DATE            对应PHOENIX里的UNSIGNED_DATE          8字节
+# $UNSIGNED_TIMESTAMP       对应PHOENIX里的UNSIGNED_TIMESTAMP     12字节
+# $VARCHAR                  对应PHOENIX里的VARCHAR                动态长度
+# $VARBINARY                对应PHOENIX里的VARBINARY              动态长度
+# $DECIMAL                  对应PHOENIX里的DECIMAL                动态长度

client-adapter/logger/pom.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.client-adapter</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>client-adapter.logger</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client adapter logger example module for otter ${project.version}</name>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java

@@ -0,0 +1,45 @@
+package com.alibaba.otter.canal.client.adapter.logger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
+import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * 外部适配器示例
+ *
+ * @author machengyuan 2018-8-19 下午11:45:38
+ * @version 1.0.0
+ */
+@SPI("logger") // logger参数对应CanalOuterAdapterConfiguration配置中的name
+public class LoggerAdapterExample implements CanalOuterAdapter {
+
+    private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+    @Override
+    public void writeOut(Message message) {
+        // 直接输出日志信息
+        MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
+
+            @Override
+            public void accept(Dml dml) {
+                logger.info(dml.toString());
+            }
+        });
+    }
+
+    @Override
+    public void init(CanalOuterAdapterConfiguration configuration) {
+
+    }
+
+    @Override
+    public void destroy() {
+
+    }
+}

client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.CanalOuterAdapter

@@ -0,0 +1 @@
+logger=com.alibaba.otter.canal.client.adapter.logger.LoggerAdapterExample

client-adapter/pom.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>canal.client-adapter</artifactId>
+    <packaging>pom</packaging>
+    <name>canal client adapter module for otter ${project.version}</name>
+    <modules>
+        <module>common</module>
+        <module>logger</module>
+        <module>hbase</module>
+    </modules>
+
+</project>

client-launcher/pom.xml

@@ -0,0 +1,149 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>canal.client-launcher</artifactId>
+    <packaging>jar</packaging>
+    <name>canal client launcher module for otter ${project.version}</name>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.client</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <!-- 单独引入kafka依赖 -->
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>1.17</version>
+        </dependency>
+
+        <!-- outer adapter -->
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.logger</artifactId>
+            <version>${project.version}</version>
+            <classifier>jar-with-dependencies</classifier>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>client-adapter.hbase</artifactId>
+            <version>${project.version}</version>
+            <classifier>jar-with-dependencies</classifier>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 -->
+            <plugin>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <addMavenDescriptor>true</addMavenDescriptor>
+                    </archive>
+                    <excludes>
+                        <exclude>**/logback.xml</exclude>
+                        <exclude>**/canal-client.yml</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <!-- 这是最新版本，推荐使用这个版本 -->
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>assemble</id>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+                <configuration>
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <attach>false</attach>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <profiles>
+        <profile>
+            <id>dev</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>env</name>
+                    <value>!release</value>
+                </property>
+            </activation>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <configuration>
+                            <!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
+                            <descriptors>
+                                <descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
+                            </descriptors>
+                            <finalName>canal_client</finalName>
+                            <outputDirectory>${project.build.directory}</outputDirectory>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+
+        </profile>
+
+        <profile>
+            <id>release</id>
+            <activation>
+                <property>
+                    <name>env</name>
+                    <value>release</value>
+                </property>
+            </activation>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <configuration>
+                            <!-- 发布模式使用的maven assembly插件描述文件 -->
+                            <descriptors>
+                                <descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
+                            </descriptors>
+                            <!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
+                            <finalName>${project.artifactId}-${project.version}</finalName>
+                            <!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
+                            <outputDirectory>${project.parent.build.directory}</outputDirectory>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+</project>

client-launcher/src/main/assembly/dev.xml

@@ -0,0 +1,57 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	<id>dist</id>
+	<formats>
+		<format>dir</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>.</directory>
+			<outputDirectory>/</outputDirectory>
+			<includes>
+				<include>README*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/bin</directory>
+			<outputDirectory>bin</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<fileMode>0755</fileMode>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/conf</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/resources</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<excludes>
+				<exclude>META-INF/**</exclude>
+			</excludes>
+		</fileSet>
+		<fileSet>
+			<directory>target</directory>
+			<outputDirectory>logs</outputDirectory>
+			<excludes>
+				<exclude>**/*</exclude>
+			</excludes>
+		</fileSet>
+	</fileSets>
+	<dependencySets>
+		<dependencySet>
+			<outputDirectory>lib</outputDirectory>
+			<excludes>
+				<exclude>junit:junit</exclude>
+			</excludes>
+		</dependencySet>
+	</dependencySets>
+</assembly>

client-launcher/src/main/assembly/release.xml

@@ -0,0 +1,57 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	<id>dist</id>
+	<formats>
+		<format>tar.gz</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>.</directory>
+			<outputDirectory>/</outputDirectory>
+			<includes>
+				<include>README*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/bin</directory>
+			<outputDirectory>bin</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<fileMode>0755</fileMode>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/conf</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/resources</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<excludes>
+				<exclude>META-INF/**</exclude>
+			</excludes>
+		</fileSet>
+		<fileSet>
+			<directory>target</directory>
+			<outputDirectory>logs</outputDirectory>
+			<excludes>
+				<exclude>**/*</exclude>
+			</excludes>
+		</fileSet>
+	</fileSets>
+	<dependencySets>
+		<dependencySet>
+			<outputDirectory>lib</outputDirectory>
+			<excludes>
+				<exclude>junit:junit</exclude>
+			</excludes>
+		</dependencySet>
+	</dependencySets>
+</assembly>

client-launcher/src/main/bin/startup.bat

@@ -0,0 +1,25 @@
+@echo off
+@if not "%ECHO%" == ""  echo %ECHO%
+@if "%OS%" == "Windows_NT"  setlocal
+
+set ENV_PATH=.\
+if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
+
+set conf_dir=%ENV_PATH%\..\conf
+set canal_conf=%conf_dir%\canal-client.yml
+set logback_configurationFile=%conf_dir%\logback.xml
+
+set CLASSPATH=%conf_dir%
+set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
+
+set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m
+set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
+set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n
+set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%"
+
+set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
+
+set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.client.ClientLauncher
+echo start cmd : %CMD_STR%
+
+java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.client.ClientLauncher

client-launcher/src/main/bin/startup.sh

@@ -0,0 +1,100 @@
+#!/bin/bash 
+
+current_path=`pwd`
+case "`uname`" in
+    Linux)
+		bin_abs_path=$(readlink -f $(dirname $0))
+		;;
+	*)
+		bin_abs_path=`cd $(dirname $0); pwd`
+		;;
+esac
+base=${bin_abs_path}/..
+canal_conf=$base/conf/canal-client.yml
+logback_configurationFile=$base/conf/logback.xml
+export LANG=en_US.UTF-8
+export BASE=$base
+
+if [ -f $base/bin/canal_client.pid ] ; then
+	echo "found canal_client.pid , Please run stop.sh first ,then startup.sh" 2>&2
+    exit 1
+fi
+
+## set java path
+if [ -z "$JAVA" ] ; then
+  JAVA=$(which java)
+fi
+
+ALIBABA_JAVA="/usr/alibaba/java/bin/java"
+TAOBAO_JAVA="/opt/taobao/java/bin/java"
+if [ -z "$JAVA" ]; then
+  if [ -f $ALIBABA_JAVA ] ; then
+  	JAVA=$ALIBABA_JAVA
+  elif [ -f $TAOBAO_JAVA ] ; then
+  	JAVA=$TAOBAO_JAVA
+  else
+  	echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
+    exit 1
+  fi
+fi
+
+case "$#" 
+in
+0 ) 
+	;;
+1 )	
+	var=$*
+	if [ -f $var ] ; then 
+		canal_conf=$var
+	else
+		echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
+        exit
+	fi;;
+2 )	
+	var=$1
+	if [ -f $var ] ; then
+		canal_conf=$var
+	else 
+		if [ "$1" = "debug" ]; then
+			DEBUG_PORT=$2
+			DEBUG_SUSPEND="n"
+			JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
+		fi
+     fi;;
+* )
+	echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
+	exit;;
+esac
+
+str=`file -L $JAVA | grep 64-bit`
+if [ -n "$str" ]; then
+	JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError"
+else
+	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m "
+fi
+
+JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
+CANAL_OPTS="-DappName=otter-canal-client -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
+
+if [ -e $canal_conf -a -e $logback_configurationFile ]
+then 
+	
+	for i in $base/lib/*;
+		do CLASSPATH=$i:"$CLASSPATH";
+	done
+ 	CLASSPATH="$base/conf:$CLASSPATH";
+ 	
+ 	echo "cd to $bin_abs_path for workaround relative path"
+  	cd $bin_abs_path
+ 	
+	echo LOG CONFIGURATION : $logback_configurationFile
+	echo canal conf : $canal_conf 
+	echo CLASSPATH :$CLASSPATH
+	$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.client.ClientLauncher 1>>$base/logs/canal_client.log 2>&1 &
+	echo $! > $base/bin/canal_client.pid
+	
+	echo "cd to $current_path for continue"
+  	cd $current_path
+else 
+	echo "canal client conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!"
+fi

client-launcher/src/main/bin/stop.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+cygwin=false;
+linux=false;
+case "`uname`" in
+    CYGWIN*)
+        cygwin=true
+        ;;
+    Linux*)
+    	linux=true
+    	;;
+esac
+
+get_pid() {	
+	STR=$1
+	PID=$2
+    if $cygwin; then
+        JAVA_CMD="$JAVA_HOME\bin\java"
+        JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
+        JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
+    else
+    	if $linux; then
+	        if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    else
+	    	if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    fi
+    fi
+    echo $JAVA_PID;
+}
+
+base=`dirname $0`/..
+pidfile=$base/bin/canal_client.pid
+if [ ! -f "$pidfile" ];then
+	echo "canal client is not running. exists"
+	exit
+fi
+
+pid=`cat $pidfile`
+if [ "$pid" == "" ] ; then
+	pid=`get_pid "appName=otter-canal-client"`
+fi
+
+echo -e "`hostname`: stopping canal $pid ... "
+kill $pid
+
+LOOPS=0
+while (true); 
+do 
+	gpid=`get_pid "appName=otter-canal-client" "$pid"`
+    if [ "$gpid" == "" ] ; then
+    	echo "Oook! cost:$LOOPS"
+    	`rm $pidfile`
+    	break;
+    fi
+    let LOOPS=LOOPS+1
+    sleep 1
+done

client-launcher/src/main/java/com/alibaba/otter/canal/client/ClientLauncher.java

@@ -0,0 +1,65 @@
+package com.alibaba.otter.canal.client;
+
+import com.alibaba.otter.canal.client.adapter.loader.CanalAdapterLoader;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.FileInputStream;
+
+public class ClientLauncher {
+
+    private static final String CLASSPATH_URL_PREFIX = "classpath:";
+    private static final Logger logger               = LoggerFactory.getLogger(ClientLauncher.class);
+
+    public static void main(String[] args) {
+        try {
+            logger.info("## set default uncaught exception handler");
+            setGlobalUncaughtExceptionHandler();
+
+            logger.info("## load canal client configurations");
+            String conf = System.getProperty("client.conf", "classpath:canal-client.yml");
+            CanalClientConfig canalClientConfig;
+            if (conf.startsWith(CLASSPATH_URL_PREFIX)) {
+                conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);
+                canalClientConfig = new Yaml().loadAs(ClientLauncher.class.getClassLoader().getResourceAsStream(conf),
+                    CanalClientConfig.class);
+            } else {
+                canalClientConfig = new Yaml().loadAs(new FileInputStream(conf), CanalClientConfig.class);
+            }
+            logger.info("## start the canal client adapters.");
+            final CanalAdapterLoader adapterLoader = new CanalAdapterLoader(canalClientConfig);
+            adapterLoader.init();
+            logger.info("## the canal client adapters are running now ......");
+            Runtime.getRuntime().addShutdownHook(new Thread() {
+
+                public void run() {
+                    try {
+                        logger.info("## stop the canal client adapters");
+                        adapterLoader.destroy();
+                    } catch (Throwable e) {
+                        logger.warn("## something goes wrong when stopping canal client adapters:", e);
+                    } finally {
+                        logger.info("## canal client adapters are down.");
+                    }
+                }
+
+            });
+        } catch (Throwable e) {
+            logger.error("## something goes wrong when starting up the canal client adapters:", e);
+            System.exit(0);
+        }
+    }
+
+    private static void setGlobalUncaughtExceptionHandler() {
+        Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+
+            @Override
+            public void uncaughtException(Thread t, Throwable e) {
+                logger.error("UnCaughtException", e);
+            }
+        });
+    }
+}

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java

@@ -0,0 +1,83 @@
+package com.alibaba.otter.canal.client.adapter.loader;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * 适配器工作线程抽象类
+ *
+ * @author machengyuan 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public abstract class AbstractCanalAdapterWorker {
+
+    protected final Logger                    logger  = LoggerFactory.getLogger(this.getClass());
+
+    protected String                          canalDestination;                                                 // canal实例
+    protected List<List<CanalOuterAdapter>>   canalOuterAdapters;                                               // 外部适配器
+    protected ExecutorService                 groupInnerExecutorService;                                        // 组内工作线程池
+    protected volatile boolean                running = false;                                                  // 是否运行中
+    protected Thread                          thread  = null;
+    protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
+
+                                                          @Override
+                                                          public void uncaughtException(Thread t, Throwable e) {
+                                                              logger.error("parse events has an error", e);
+                                                          }
+                                                      };
+
+    protected void writeOut(final Message message) {
+        List<Future<Boolean>> futures = new ArrayList<>();
+        // 组间适配器并行运行
+        for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
+            final List<CanalOuterAdapter> adapters = outerAdapters;
+            futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
+
+                @Override
+                public Boolean call() {
+                    try {
+                        // 组内适配器穿行运行，尽量不要配置组内适配器
+                        for (CanalOuterAdapter c : adapters) {
+                            long begin = System.currentTimeMillis();
+                            c.writeOut(message);
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("{} elapsed time: {}",
+                                    c.getClass().getName(),
+                                    (System.currentTimeMillis() - begin));
+                            }
+                        }
+                        return true;
+                    } catch (Exception e) {
+                        return false;
+                    }
+                }
+            }));
+
+            // 等待所有适配器写入完成
+            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
+            for (Future<Boolean> f : futures) {
+                try {
+                    if (!f.get()) {
+                        logger.error("Outer adapter write failed");
+                    }
+                } catch (InterruptedException | ExecutionException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
+    }
+
+    public abstract void start();
+
+    public abstract void stop();
+}

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java

@@ -0,0 +1,179 @@
+package com.alibaba.otter.canal.client.adapter.loader;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.common.errors.WakeupException;
+
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * kafka对应的client适配器工作线程
+ *
+ * @author machengyuan 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
+
+    private KafkaCanalConnector connector;
+
+    private String              topic;
+
+    public CanalAdapterKafkaWorker(String zkServers, String bootstrapServers, String topic, String groupId,
+                                   List<List<CanalOuterAdapter>> canalOuterAdapters){
+        this.canalOuterAdapters = canalOuterAdapters;
+        this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
+        this.topic = topic;
+        this.canalDestination = topic;
+        connector = KafkaCanalConnectors.newKafkaConnector(zkServers, bootstrapServers, topic, null, groupId);
+        // connector.setSessionTimeout(5L, TimeUnit.MINUTES);
+
+        // super.initSwitcher(topic);
+    }
+
+    @Override
+    public void start() {
+        if (!running) {
+            thread = new Thread(new Runnable() {
+
+                @Override
+                public void run() {
+                    process();
+                }
+            });
+            thread.setUncaughtExceptionHandler(handler);
+            running = true;
+            thread.start();
+        }
+    }
+
+    @Override
+    public void stop() {
+        try {
+            if (!running) {
+                return;
+            }
+
+            connector.stopRunning();
+            running = false;
+
+            // if (switcher != null && !switcher.state()) {
+            // switcher.set(true);
+            // }
+
+            if (thread != null) {
+                try {
+                    thread.join();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            groupInnerExecutorService.shutdown();
+            logger.info("topic {} connectors' worker thread dead!", this.topic);
+            for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
+                for (CanalOuterAdapter adapter : outerAdapters) {
+                    adapter.destroy();
+                }
+            }
+            logger.info("topic {} all connectors destroyed!", this.topic);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private void process() {
+        while (!running)
+            ;
+        ExecutorService executor = Executors.newSingleThreadExecutor();
+        final AtomicBoolean executing = new AtomicBoolean(true);
+        while (running) {
+            try {
+                logger.info("=============> Start to connect topic: {} <=============", this.topic);
+                connector.connect();
+                logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
+                connector.subscribe();
+                logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
+                while (running) {
+                    try {
+                        // switcher.get(); //等待开关开启
+
+                        final Message message = connector.getWithoutAck();
+
+                        executing.set(true);
+                        if (message != null) {
+                            executor.submit(new Runnable() {
+
+                                @Override
+                                public void run() {
+                                    try {
+                                        if (logger.isDebugEnabled()) {
+                                            logger.debug("topic: {} batchId: {} batchSize: {} ",
+                                                topic,
+                                                message.getId(),
+                                                message.getEntries().size());
+                                        }
+                                        long begin = System.currentTimeMillis();
+                                        writeOut(message);
+                                        long now = System.currentTimeMillis();
+                                        if ((System.currentTimeMillis() - begin) > 5 * 60 * 1000) {
+                                            logger.error("topic: {} batchId {} elapsed time: {} ms",
+                                                topic,
+                                                message.getId(),
+                                                now - begin);
+                                        }
+                                        if (logger.isDebugEnabled()) {
+                                            logger.debug("topic: {} batchId {} elapsed time: {} ms",
+                                                topic,
+                                                message.getId(),
+                                                now - begin);
+                                        }
+                                    } catch (Exception e) {
+                                        logger.error(e.getMessage(), e);
+                                    } finally {
+                                        executing.compareAndSet(true, false);
+                                    }
+                                }
+                            });
+
+                            // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
+                            long currentTS = System.currentTimeMillis();
+                            while (executing.get()) {
+                                // 大于1分钟未消费完ack一次keep alive
+                                if (System.currentTimeMillis() - currentTS > 60000) {
+                                    connector.ack();
+                                    currentTS = System.currentTimeMillis();
+                                }
+                            }
+                        } else {
+                            connector.ack();
+                        }
+                    } catch (CommitFailedException e) {
+                        logger.warn(e.getMessage());
+                    } catch (Exception e) {
+                        logger.error(e.getMessage(), e);
+                        TimeUnit.SECONDS.sleep(1L);
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        executor.shutdown();
+
+        try {
+            connector.unsubscribe();
+        } catch (WakeupException e) {
+            // No-op. Continue process
+        }
+        connector.disconnnect();
+        logger.info("=============> Disconnect topic: {} <=============", this.topic);
+    }
+}

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java

@@ -0,0 +1,170 @@
+package com.alibaba.otter.canal.client.adapter.loader;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
+import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
+
+/**
+ * 外部适配器的加载器
+ *
+ * @author machengyuan 2018-8-19 下午11:45:49
+ * @version 1.0.0
+ */
+public class CanalAdapterLoader {
+
+    private static final Logger                  logger            = LoggerFactory.getLogger(CanalAdapterLoader.class);
+
+    private CanalClientConfig                    canalClientConfig;
+
+    private Map<String, CanalAdapterWorker>      canalWorkers      = new HashMap<>();
+
+    private Map<String, CanalAdapterKafkaWorker> canalKafkaWorkers = new HashMap<>();
+
+    private ExtensionLoader<CanalOuterAdapter>   loader;
+
+    public CanalAdapterLoader(CanalClientConfig canalClientConfig){
+        this.canalClientConfig = canalClientConfig;
+    }
+
+    /**
+     * 初始化canal-client、 canal-client-kafka的适配器
+     */
+    public void init() {
+        // canal instances 和 kafka topics 配置不能同时为空
+        if (canalClientConfig.getCanalInstances() == null && canalClientConfig.getKafkaTopics() == null) {
+            throw new RuntimeException("Blank config property: canalInstances or canalKafkaTopics");
+        }
+
+        loader = ExtensionLoader.getExtensionLoader(CanalOuterAdapter.class,
+            "" /* TODO canalClientConfig.getClassloaderPolicy() */);
+
+        String canalServerHost = this.canalClientConfig.getCanalServerHost();
+        SocketAddress sa = null;
+        if (canalServerHost != null) {
+            String[] ipPort = canalServerHost.split(":");
+            sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
+        }
+        String zkHosts = this.canalClientConfig.getZookeeperHosts();
+
+        // if (zkHosts == null && sa == null) {
+        // throw new RuntimeException("Blank config property: canalServerHost or
+        // zookeeperHosts");
+        // }
+
+        // 初始化canal-client的适配器
+        if (canalClientConfig.getCanalInstances() != null) {
+            for (CanalClientConfig.CanalInstance instance : canalClientConfig.getCanalInstances()) {
+                List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
+
+                for (CanalClientConfig.AdapterGroup connectorGroup : instance.getAdapterGroups()) {
+                    List<CanalOuterAdapter> canalOutConnectors = new ArrayList<>();
+                    for (CanalOuterAdapterConfiguration c : connectorGroup.getOutAdapters()) {
+                        loadConnector(c, canalOutConnectors);
+                    }
+                    canalOuterAdapterGroups.add(canalOutConnectors);
+                }
+                CanalAdapterWorker worker;
+                if (zkHosts != null) {
+                    worker = new CanalAdapterWorker(instance.getInstance(), zkHosts, canalOuterAdapterGroups);
+                } else {
+                    worker = new CanalAdapterWorker(instance.getInstance(), sa, canalOuterAdapterGroups);
+                }
+                canalWorkers.put(instance.getInstance(), worker);
+                worker.start();
+                logger.info("Start adapter for canal instance: {} succeed", instance.getInstance());
+            }
+        }
+
+        // 初始化canal-client-kafka的适配器
+        if (canalClientConfig.getKafkaTopics() != null) {
+            for (CanalClientConfig.KafkaTopic kafkaTopic : canalClientConfig.getKafkaTopics()) {
+                for (CanalClientConfig.Group group : kafkaTopic.getGroups()) {
+                    List<List<CanalOuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
+
+                    List<CanalOuterAdapter> canalOuterAdapters = new ArrayList<>();
+
+                    for (CanalOuterAdapterConfiguration config : group.getOutAdapters()) {
+                        // for (CanalOuterAdapterConfiguration config : adaptor.getOutAdapters()) {
+                        loadConnector(config, canalOuterAdapters);
+                        // }
+                    }
+                    canalOuterAdapterGroups.add(canalOuterAdapters);
+
+                    // String zkServers = canalClientConfig.getZookeeperHosts();
+                    CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(zkHosts,
+                        canalClientConfig.getBootstrapServers(),
+                        kafkaTopic.getTopic(),
+                        group.getGroupId(),
+                        canalOuterAdapterGroups);
+                    canalKafkaWorkers.put(kafkaTopic.getTopic() + "-" + group.getGroupId(), canalKafkaWorker);
+                    canalKafkaWorker.start();
+                    logger.info("Start adapter for canal-client kafka topic: {} succeed",
+                        kafkaTopic.getTopic() + "-" + group.getGroupId());
+                }
+            }
+        }
+    }
+
+    private void loadConnector(CanalOuterAdapterConfiguration config, List<CanalOuterAdapter> canalOutConnectors) {
+        try {
+            CanalOuterAdapter adapter = loader.getExtension(config.getName());
+            ClassLoader cl = Thread.currentThread().getContextClassLoader();
+            // 替换ClassLoader
+            Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
+            adapter.init(config);
+            Thread.currentThread().setContextClassLoader(cl);
+            canalOutConnectors.add(adapter);
+            logger.info("Load canal adapter: {} succeed", config.getName());
+        } catch (Exception e) {
+            logger.error("Load canal adapter: {} failed", config.getName(), e);
+        }
+    }
+
+    /**
+     * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
+     */
+    public void destroy() {
+        if (canalWorkers.size() > 0) {
+            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
+            for (CanalAdapterWorker v : canalWorkers.values()) {
+                final CanalAdapterWorker caw = v;
+                stopExecutorService.submit(new Runnable() {
+
+                    @Override
+                    public void run() {
+                        caw.stop();
+                    }
+                });
+            }
+            stopExecutorService.shutdown();
+        }
+        if (canalKafkaWorkers.size() > 0) {
+            ExecutorService stopKafkaExecutorService = Executors.newFixedThreadPool(canalKafkaWorkers.size());
+            for (CanalAdapterKafkaWorker v : canalKafkaWorkers.values()) {
+                final CanalAdapterKafkaWorker cakw = v;
+                stopKafkaExecutorService.submit(new Runnable() {
+
+                    @Override
+                    public void run() {
+                        cakw.stop();
+                    }
+                });
+            }
+            stopKafkaExecutorService.shutdown();
+        }
+        logger.info("All canal adapters destroyed");
+    }
+}

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java

@@ -0,0 +1,186 @@
+package com.alibaba.otter.canal.client.adapter.loader;
+
+import java.net.SocketAddress;
+import java.util.List;
+import java.util.concurrent.Executors;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalConnectors;
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * 原生canal-server对应的client适配器工作线程
+ *
+ * @author machengyuan 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
+
+    private static final int BATCH_SIZE = 50;
+    private static final int SO_TIMEOUT = 0;
+
+    private CanalConnector   connector;
+
+    /**
+     * 单台client适配器worker的构造方法
+     *
+     * @param canalDestination canal实例名
+     * @param address canal-server地址
+     * @param canalOuterAdapters 外部适配器组
+     */
+    public CanalAdapterWorker(String canalDestination, SocketAddress address,
+                              List<List<CanalOuterAdapter>> canalOuterAdapters){
+        this.canalOuterAdapters = canalOuterAdapters;
+        this.canalDestination = canalDestination;
+        groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
+        connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
+    }
+
+    /**
+     * HA模式下client适配器worker的构造方法
+     *
+     * @param canalDestination canal实例名
+     * @param zookeeperHosts zookeeper地址
+     * @param canalOuterAdapters 外部适配器组
+     */
+    public CanalAdapterWorker(String canalDestination, String zookeeperHosts,
+                              List<List<CanalOuterAdapter>> canalOuterAdapters){
+        this.canalOuterAdapters = canalOuterAdapters;
+        this.canalDestination = canalDestination;
+        groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
+        connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
+        ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
+
+        // super.initSwitcher(canalDestination);
+    }
+
+    @Override
+    public void start() {
+        if (!running) {
+            thread = new Thread(new Runnable() {
+
+                @Override
+                public void run() {
+                    process();
+                }
+            });
+            thread.setUncaughtExceptionHandler(handler);
+            thread.start();
+            running = true;
+        }
+    }
+
+    @Override
+    public void stop() {
+        try {
+            if (!running) {
+                return;
+            }
+
+            // if (switcher != null && !switcher.state()) {
+            // switcher.set(true);
+            // }
+
+            connector.stopRunning();
+            running = false;
+
+            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
+            if (thread != null) {
+                try {
+                    thread.join();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            groupInnerExecutorService.shutdown();
+            logger.info("destination {} adapters' worker thread dead!", canalDestination);
+            for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
+                for (CanalOuterAdapter adapter : outerAdapters) {
+                    adapter.destroy();
+                }
+            }
+            logger.info("destination {} all adapters destroyed!", canalDestination);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    private void process() {
+        while (!running)
+            ; // waiting until running == true
+        while (running) {
+            try {
+                // if (switcher != null) {
+                // switcher.get();
+                // }
+                logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
+                connector.connect();
+                logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination);
+                connector.subscribe();
+                logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
+                while (running) {
+                    // try {
+                    // if (switcher != null) {
+                    // switcher.get();
+                    // }
+                    // } catch (TimeoutException e) {
+                    // break;
+                    // }
+
+                    // server配置canal.instance.network.soTimeout(默认: 30s)
+                    // 范围内未与server交互，server将关闭本次socket连接
+                    Message message = connector.getWithoutAck(BATCH_SIZE); // 获取指定数量的数据
+                    long batchId = message.getId();
+                    try {
+                        int size = message.getEntries().size();
+
+                        if (batchId == -1 || size == 0) {
+                            try {
+                                Thread.sleep(1000);
+                            } catch (InterruptedException e) {
+                                // ignore
+                            }
+                        } else {
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("destination: {} batchId: {} batchSize: {} ",
+                                    this.canalDestination,
+                                    batchId,
+                                    size);
+                            }
+                            long begin = System.currentTimeMillis();
+                            writeOut(message);
+                            long now = System.currentTimeMillis();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("destination: {} batchId: {} elapsed time: {} ms",
+                                    this.canalDestination,
+                                    batchId,
+                                    now - begin);
+                            }
+                        }
+                        connector.ack(batchId); // 提交确认
+                    } catch (Exception e) {
+                        connector.rollback(batchId); // 处理失败, 回滚数据
+                        throw e;
+                    }
+                }
+
+            } catch (Exception e) {
+                logger.error("process error!", e);
+            } finally {
+                connector.disconnect();
+                logger.info("=============> Disconnect destination: {} <=============", this.canalDestination);
+            }
+
+            if (running) { // is reconnect
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+
+        }
+    }
+}

client-launcher/src/main/resources/canal-client.yml

@@ -0,0 +1,21 @@
+canalServerHost: 127.0.0.1:11111
+#zookeeperHosts: slave1:2181
+#bootstrapServers: slave1:6667,slave2:6667
+
+canalInstances:
+- instance: example
+  adapterGroups:
+  - outAdapters:
+    - name: logger
+    - name: hbase
+      hosts: slave1:2181
+      properties: {znodeParent: "/hbase-unsecure"}
+#kafkaTopics:
+#- topic: example
+#  groups:
+#  - groupId: example_g1
+#    outAdapters:
+#    - name: logger
+#    - name: hbase
+#      hosts: slave1:2181
+#      properties: {znodeParent: "/hbase-unsecure"}

client-launcher/src/main/resources/logback.xml

@@ -0,0 +1,47 @@
+<configuration scan="true" scanPeriod=" 5 seconds">
+	<jmxConfigurator />
+	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+			</pattern>
+		</encoder>
+	</appender>
+	
+	<appender name="CANAL-ROOT" class="ch.qos.logback.classic.sift.SiftingAppender">
+		<discriminator>
+			<Key>destination</Key>
+			<DefaultValue>canal_client</DefaultValue>
+		</discriminator>
+		<sift>
+			<appender name="FILE-${destination}" class="ch.qos.logback.core.rolling.RollingFileAppender">
+				<File>../logs/${destination}.log</File>
+				<rollingPolicy
+					class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+					<!-- rollover daily -->
+					<fileNamePattern>../logs/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
+					<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+						<!-- or whenever the file size reaches 100MB -->
+						<maxFileSize>512MB</maxFileSize>
+					</timeBasedFileNamingAndTriggeringPolicy>
+					<maxHistory>60</maxHistory>
+				</rollingPolicy>
+				<encoder>
+					<pattern>
+						%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+					</pattern>
+				</encoder>
+			</appender>
+		</sift>
+	</appender>
+
+	<logger name="com.alibaba.otter.canal.client" additivity="false">
+		<level value="INFO" />
+		<!--<appender-ref ref="STDOUT"/>-->
+		<appender-ref ref="CANAL-ROOT" />
+	</logger>
+    
+	<root level="WARN">
+		<!--<appender-ref ref="STDOUT"/>-->
+		<appender-ref ref="CANAL-ROOT" />
+	</root>
+</configuration>

client/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.1.1-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>
@@ -16,6 +16,13 @@
 			<artifactId>canal.protocol</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		<!-- 客户端要使用请单独引入kafka依赖 -->
+		<dependency>
+			<groupId>org.apache.kafka</groupId>
+			<artifactId>kafka-clients</artifactId>
+			<version>1.1.1</version>
+			<scope>provided</scope>
+		</dependency>
 
 		<!-- junit -->
 		<dependency>
@@ -69,7 +76,7 @@
 							<links>
 								<link>https://github.com/alibaba/canal</link>
 							</links>
-							<outputDirectory>${project.build.directory}/apidocs/apidocs/${pom.version}</outputDirectory>
+							<outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}</outputDirectory>
 						</configuration>
 					</plugin>
 					<plugin>

client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java

@@ -19,14 +19,14 @@ public interface CanalConnector {
      * 
      * @throws CanalClientException
      */
-    public void connect() throws CanalClientException;
+    void connect() throws CanalClientException;
 
     /**
      * 释放链接
      * 
      * @throws CanalClientException
      */
-    public void disconnect() throws CanalClientException;
+    void disconnect() throws CanalClientException;
 
     /**
      * 检查下链接是否合法
@@ -43,7 +43,7 @@ public interface CanalConnector {
      * 
      * @throws CanalClientException
      */
-    public boolean checkValid() throws CanalClientException;
+    boolean checkValid() throws CanalClientException;
 
     /**
      * 客户端订阅，重复订阅时会更新对应的filter信息
@@ -56,7 +56,6 @@ public interface CanalConnector {
      * TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
      * </pre>
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void subscribe(String filter) throws CanalClientException;
@@ -64,7 +63,6 @@ public interface CanalConnector {
     /**
      * 客户端订阅，不提交客户端filter，以服务端的filter为准
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void subscribe() throws CanalClientException;
@@ -72,7 +70,6 @@ public interface CanalConnector {
     /**
      * 取消订阅
      * 
-     * @param clientIdentity
      * @throws CanalClientException
      */
     void unsubscribe() throws CanalClientException;
@@ -140,17 +137,23 @@ public interface CanalConnector {
     void ack(long batchId) throws CanalClientException;
 
     /**
-     * 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId
+     * 回滚到未进行 {@link #ack} 的地方，指定回滚具体的batchId
      * 
      * @throws CanalClientException
      */
     void rollback(long batchId) throws CanalClientException;
 
     /**
-     * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
+     * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
      * 
      * @throws CanalClientException
      */
     void rollback() throws CanalClientException;
 
+    /**
+     * 中断的阻塞，用于优雅停止client
+     * 
+     * @throws CanalClientException
+     */
+    void stopRunning() throws CanalClientException;
 }

client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java

@@ -19,8 +19,9 @@ public class CanalConnectors {
 
     /**
      * 创建单链接的客户端链接
-     * 
+     *
      * @param address
+     * @param destination
      * @param username
      * @param password
      * @return
@@ -28,43 +29,48 @@ public class CanalConnectors {
     public static CanalConnector newSingleConnector(SocketAddress address, String destination, String username,
                                                     String password) {
         SimpleCanalConnector canalConnector = new SimpleCanalConnector(address, username, password, destination);
-        canalConnector.setSoTimeout(30 * 1000);
+        canalConnector.setSoTimeout(60 * 1000);
+        canalConnector.setIdleTimeout(60 * 60 * 1000);
         return canalConnector;
     }
 
     /**
      * 创建带cluster模式的客户端链接，自动完成failover切换
-     * 
+     *
      * @param addresses
+     * @param destination
      * @param username
      * @param password
      * @return
      */
     public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
                                                      String username, String password) {
-        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
-                                                                         new SimpleNodeAccessStrategy(addresses));
+            password,
-        canalConnector.setSoTimeout(30 * 1000);
+            destination,
+            new SimpleNodeAccessStrategy(addresses));
+        canalConnector.setSoTimeout(60 * 1000);
+        canalConnector.setIdleTimeout(60 * 60 * 1000);
         return canalConnector;
     }
 
     /**
      * 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描
-     * 
+     *
+     * @param zkServers
+     * @param destination
      * @param username
      * @param password
      * @return
      */
     public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
                                                      String password) {
-        ClusterCanalConnector canalConnector = new ClusterCanalConnector(
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
-                                                                         username,
+            password,
-                                                                         password,
+            destination,
-                                                                         destination,
+            new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkServers)));
-                                                                         new ClusterNodeAccessStrategy(
+        canalConnector.setSoTimeout(60 * 1000);
-                                                                                                       destination,
+        canalConnector.setIdleTimeout(60 * 60 * 1000);
-                                                                                                       ZkClientx.getZkClient(zkServers)));
-        canalConnector.setSoTimeout(30 * 1000);
         return canalConnector;
     }
 }

client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java

@@ -10,5 +10,7 @@ import java.net.SocketAddress;
  */
 public interface CanalNodeAccessStrategy {
 
+    SocketAddress currentNode();
+
     SocketAddress nextNode();
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.client.impl;
 import java.net.SocketAddress;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -23,8 +22,9 @@ public class ClusterCanalConnector implements CanalConnector {
     private final Logger            logger        = LoggerFactory.getLogger(this.getClass());
     private String                  username;
     private String                  password;
-    private int                     soTimeout     = 10000;
+    private int                     soTimeout     = 60000;
-    private int                     retryTimes    = 3;
+    private int                     idleTimeout   = 60 * 60 * 1000;
+    private int                     retryTimes    = 3;                                       // 设置-1时可以subscribe阻塞等待时优雅停机
     private int                     retryInterval = 5000;                                    // 重试的时间间隔，默认5秒
     private CanalNodeAccessStrategy accessStrategy;
     private SimpleCanalConnector    currentConnector;
@@ -41,12 +41,19 @@ public class ClusterCanalConnector implements CanalConnector {
 
     public void connect() throws CanalClientException {
         while (currentConnector == null) {
-            SocketAddress nextAddress = this.accessStrategy.nextNode();
             int times = 0;
             while (true) {
                 try {
-                    currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination);
+                    currentConnector = new SimpleCanalConnector(null, username, password, destination) {
+
+                        @Override
+                        public SocketAddress getNextAddress() {
+                            return accessStrategy.nextNode();
+                        }
+
+                    };
                     currentConnector.setSoTimeout(soTimeout);
+                    currentConnector.setIdleTimeout(idleTimeout);
                     if (filter != null) {
                         currentConnector.setFilter(filter);
                     }
@@ -57,7 +64,7 @@ public class ClusterCanalConnector implements CanalConnector {
                     currentConnector.connect();
                     break;
                 } catch (Exception e) {
-                    logger.warn("failed to connect to:{} after retry {} times", nextAddress, times);
+                    logger.warn("failed to connect to:{} after retry {} times", accessStrategy.currentNode(), times);
                     currentConnector.disconnect();
                     currentConnector = null;
                     // retry for #retryTimes for each node when trying to
@@ -101,12 +108,17 @@ public class ClusterCanalConnector implements CanalConnector {
                 this.filter = filter;
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when subscribing from server:{}\n{}",
+                if (retryTimes == -1 && t.getCause() instanceof InterruptedException) {
-                    currentConnector.getAddress(),
+                    logger.info("block waiting interrupted by other thread.");
-                    ExceptionUtils.getFullStackTrace(t));
+                    return;
-                times++;
+                } else {
-                restart();
+                    logger.warn(String.format("something goes wrong when subscribing from server: %s",
-                logger.info("restart the connector for next round retry.");
+                        currentConnector != null ? currentConnector.getAddress() : "null"), t);
+                    times++;
+                    restart();
+                    logger.info("restart the connector for next round retry.");
+                }
+
             }
         }
 
@@ -120,9 +132,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.unsubscribe();
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when unsubscribing from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -138,9 +149,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.get(batchSize);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getting data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getting data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -156,9 +166,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.get(batchSize, timeout, unit);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getting data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getting data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -174,9 +183,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.getWithoutAck(batchSize);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -192,9 +200,8 @@ public class ClusterCanalConnector implements CanalConnector {
                 Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
                 return msg;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when getWithoutAck data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"), t);
-                    ExceptionUtils.getFullStackTrace(t));
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -210,9 +217,9 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.rollback(batchId);
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"),
-                    ExceptionUtils.getFullStackTrace(t));
+                    t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -228,9 +235,9 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.rollback();
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when rollbacking data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"),
-                    ExceptionUtils.getFullStackTrace(t));
+                    t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -247,9 +254,9 @@ public class ClusterCanalConnector implements CanalConnector {
                 currentConnector.ack(batchId);
                 return;
             } catch (Throwable t) {
-                logger.warn("something goes wrong when acking data from server:{}\n{}",
+                logger.warn(String.format("something goes wrong when acking data from server:%s",
-                    currentConnector.getAddress(),
+                    currentConnector != null ? currentConnector.getAddress() : "null"),
-                    ExceptionUtils.getFullStackTrace(t));
+                    t);
                 times++;
                 restart();
                 logger.info("restart the connector for next round retry.");
@@ -296,6 +303,14 @@ public class ClusterCanalConnector implements CanalConnector {
         this.soTimeout = soTimeout;
     }
 
+    public int getIdleTimeout() {
+        return idleTimeout;
+    }
+
+    public void setIdleTimeout(int idleTimeout) {
+        this.idleTimeout = idleTimeout;
+    }
+
     public int getRetryTimes() {
         return retryTimes;
     }
@@ -324,4 +339,8 @@ public class ClusterCanalConnector implements CanalConnector {
         return currentConnector;
     }
 
+    public void stopRunning() {
+        currentConnector.stopRunning();
+    }
+
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java

@@ -62,6 +62,11 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
         initRunning(this.zkClient.readData(runningPath, true));
     }
 
+    @Override
+    public SocketAddress currentNode() {
+        return nextNode();
+    }
+
     public SocketAddress nextNode() {
         if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点
             return runningAddress;

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java

@@ -5,7 +5,11 @@ import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.SocketChannel;
+import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -37,11 +41,10 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 
 /**
  * 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作
- * 
+ *
  * @author jianghang 2012-10-24 下午05:37:20
  * @version 1.0.0
  */
@@ -52,11 +55,14 @@ public class SimpleCanalConnector implements CanalConnector {
     private String               username;
     private String               password;
     private int                  soTimeout             = 60000;                                              // milliseconds
+    private int                  idleTimeout           = 60 * 60 * 1000;                                     // client和server之间的空闲链接超时的时间,默认为1小时
     private String               filter;                                                                     // 记录上一次的filter提交值,便于自动重试时提交
 
     private final ByteBuffer     readHeader            = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
     private final ByteBuffer     writeHeader           = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
     private SocketChannel        channel;
+    private ReadableByteChannel  readableChannel;
+    private WritableByteChannel  writableChannel;
     private List<Compression>    supportedCompressions = new ArrayList<Compression>();
     private ClientIdentity       clientIdentity;
     private ClientRunningMonitor runningMonitor;                                                             // 运行控制
@@ -65,21 +71,29 @@ public class SimpleCanalConnector implements CanalConnector {
     private volatile boolean     connected             = false;                                              // 代表connected是否已正常执行，因为有HA，不代表在工作中
     private boolean              rollbackOnConnect     = true;                                               // 是否在connect链接成功后，自动执行rollback操作
     private boolean              rollbackOnDisConnect  = false;                                              // 是否在connect链接成功后，自动执行rollback操作
-
+    private boolean              lazyParseEntry        = false;                                              // 是否自动化解析Entry对象,如果考虑最大化性能可以延后解析
     // 读写数据分别使用不同的锁进行控制，减小锁粒度,读也需要排他锁，并发度容易造成数据包混乱，反序列化失败
     private Object               readDataLock          = new Object();
     private Object               writeDataLock         = new Object();
 
+    private volatile boolean     running               = false;
+
     public SimpleCanalConnector(SocketAddress address, String username, String password, String destination){
-        this(address, username, password, destination, 60000);
+        this(address, username, password, destination, 60000, 60 * 60 * 1000);
     }
 
     public SimpleCanalConnector(SocketAddress address, String username, String password, String destination,
                                 int soTimeout){
+        this(address, username, password, destination, soTimeout, 60 * 60 * 1000);
+    }
+
+    public SimpleCanalConnector(SocketAddress address, String username, String password, String destination,
+                                int soTimeout, int idleTimeout){
         this.address = address;
         this.username = username;
         this.password = password;
         this.soTimeout = soTimeout;
+        this.idleTimeout = idleTimeout;
         this.clientIdentity = new ClientIdentity(destination, (short) 1001);
     }
 
@@ -94,6 +108,9 @@ public class SimpleCanalConnector implements CanalConnector {
             }
         } else {
             waitClientRunning();
+            if (!running) {
+                return;
+            }
             doConnect();
             if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
                 subscribe(filter);
@@ -102,7 +119,7 @@ public class SimpleCanalConnector implements CanalConnector {
                 rollback();
             }
         }
-        
+
         connected = true;
     }
 
@@ -125,8 +142,14 @@ public class SimpleCanalConnector implements CanalConnector {
         try {
             channel = SocketChannel.open();
             channel.socket().setSoTimeout(soTimeout);
+            SocketAddress address = getAddress();
+            if (address == null) {
+                address = getNextAddress();
+            }
             channel.connect(address);
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            readableChannel = Channels.newChannel(channel.socket().getInputStream());
+            writableChannel = Channels.newChannel(channel.socket().getOutputStream());
+            Packet p = Packet.parseFrom(readNextPacket());
             if (p.getVersion() != 1) {
                 throw new CanalClientException("unsupported version at this client.");
             }
@@ -140,17 +163,17 @@ public class SimpleCanalConnector implements CanalConnector {
             //
             ClientAuth ca = ClientAuth.newBuilder()
                 .setUsername(username != null ? username : "")
-                .setNetReadTimeout(soTimeout)
+                .setPassword(ByteString.copyFromUtf8(password != null ? password : ""))
-                .setNetWriteTimeout(soTimeout)
+                .setNetReadTimeout(idleTimeout)
+                .setNetWriteTimeout(idleTimeout)
                 .build();
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.CLIENTAUTHENTICATION)
-                    .setType(PacketType.CLIENTAUTHENTICATION)
+                .setBody(ca.toByteString())
-                    .setBody(ca.toByteString())
+                .build()
-                    .build()
+                .toByteArray());
-                    .toByteArray());
             //
-            Packet ack = Packet.parseFrom(readNextPacket(channel));
+            Packet ack = Packet.parseFrom(readNextPacket());
             if (ack.getType() != PacketType.ACK) {
                 throw new CanalClientException("unexpected packet type when ack is expected");
             }
@@ -169,36 +192,50 @@ public class SimpleCanalConnector implements CanalConnector {
     }
 
     private void doDisconnnect() throws CanalClientException {
+        if (readableChannel != null) {
+            quietlyClose(readableChannel);
+            readableChannel = null;
+        }
+        if (writableChannel != null) {
+            quietlyClose(writableChannel);
+            writableChannel = null;
+        }
         if (channel != null) {
-            try {
+            quietlyClose(channel);
-                channel.close();
-            } catch (IOException e) {
-                logger.warn("exception on closing channel:{} \n {}", channel, e);
-            }
             channel = null;
         }
     }
 
+    private void quietlyClose(Channel channel) {
+        try {
+            channel.close();
+        } catch (IOException e) {
+            logger.warn("exception on closing channel:{} \n {}", channel, e);
+        }
+    }
+
     public void subscribe() throws CanalClientException {
         subscribe(""); // 传递空字符即可
     }
 
     public void subscribe(String filter) throws CanalClientException {
         waitClientRunning();
+        if (!running) {
+            return;
+        }
         try {
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.SUBSCRIPTION)
-                    .setType(PacketType.SUBSCRIPTION)
+                .setBody(Sub.newBuilder()
-                    .setBody(Sub.newBuilder()
+                    .setDestination(clientIdentity.getDestination())
-                        .setDestination(clientIdentity.getDestination())
+                    .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                    .setFilter(filter != null ? filter : "")
-                        .setFilter(filter != null ? filter : "")
-                        .build()
-                        .toByteString())
                     .build()
-                    .toByteArray());
+                    .toByteString())
+                .build()
+                .toByteArray());
             //
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Packet p = Packet.parseFrom(readNextPacket());
             Ack ack = Ack.parseFrom(p.getBody());
             if (ack.getErrorCode() > 0) {
                 throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
@@ -212,19 +249,21 @@ public class SimpleCanalConnector implements CanalConnector {
 
     public void unsubscribe() throws CanalClientException {
         waitClientRunning();
+        if (!running) {
+            return;
+        }
         try {
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.UNSUBSCRIPTION)
-                    .setType(PacketType.UNSUBSCRIPTION)
+                .setBody(Unsub.newBuilder()
-                    .setBody(Unsub.newBuilder()
+                    .setDestination(clientIdentity.getDestination())
-                        .setDestination(clientIdentity.getDestination())
+                    .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .build()
-                        .toByteString())
                     .build()
-                    .toByteArray());
+                    .toByteString())
+                .build()
+                .toByteArray());
             //
-            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Packet p = Packet.parseFrom(readNextPacket());
             Ack ack = Ack.parseFrom(p.getBody());
             if (ack.getErrorCode() > 0) {
                 throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
@@ -250,6 +289,9 @@ public class SimpleCanalConnector implements CanalConnector {
 
     public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
         waitClientRunning();
+        if (!running) {
+            return null;
+        }
         try {
             int size = (batchSize <= 0) ? 1000 : batchSize;
             long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制
@@ -257,29 +299,28 @@ public class SimpleCanalConnector implements CanalConnector {
                 unit = TimeUnit.MILLISECONDS;
             }
 
-            writeWithHeader(channel,
+            writeWithHeader(Packet.newBuilder()
-                Packet.newBuilder()
+                .setType(PacketType.GET)
-                    .setType(PacketType.GET)
+                .setBody(Get.newBuilder()
-                    .setBody(Get.newBuilder()
+                    .setAutoAck(false)
-                        .setAutoAck(false)
+                    .setDestination(clientIdentity.getDestination())
-                        .setDestination(clientIdentity.getDestination())
+                    .setClientId(String.valueOf(clientIdentity.getClientId()))
-                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                    .setFetchSize(size)
-                        .setFetchSize(size)
+                    .setTimeout(time)
-                        .setTimeout(time)
+                    .setUnit(unit.ordinal())
-                        .setUnit(unit.ordinal())
-                        .build()
-                        .toByteString())
                     .build()
-                    .toByteArray());
+                    .toByteString())
-
+                .build()
+                .toByteArray());
             return receiveMessages();
         } catch (IOException e) {
             throw new CanalClientException(e);
         }
     }
 
-    private Message receiveMessages() throws InvalidProtocolBufferException, IOException {
+    private Message receiveMessages() throws IOException {
-        Packet p = Packet.parseFrom(readNextPacket(channel));
+        byte[] data = readNextPacket();
+        Packet p = Packet.parseFrom(data);
         switch (p.getType()) {
             case MESSAGES: {
                 if (!p.getCompression().equals(Compression.NONE)) {
@@ -288,8 +329,13 @@ public class SimpleCanalConnector implements CanalConnector {
 
                 Messages messages = Messages.parseFrom(p.getBody());
                 Message result = new Message(messages.getBatchId());
-                for (ByteString byteString : messages.getMessagesList()) {
+                if (lazyParseEntry) {
-                    result.addEntry(Entry.parseFrom(byteString));
+                    // byteString
+                    result.setRawEntries(messages.getMessagesList());
+                } else {
+                    for (ByteString byteString : messages.getMessagesList()) {
+                        result.addEntry(Entry.parseFrom(byteString));
+                    }
                 }
                 return result;
             }
@@ -305,13 +351,16 @@ public class SimpleCanalConnector implements CanalConnector {
 
     public void ack(long batchId) throws CanalClientException {
         waitClientRunning();
+        if (!running) {
+            return;
+        }
         ClientAck ca = ClientAck.newBuilder()
             .setDestination(clientIdentity.getDestination())
             .setClientId(String.valueOf(clientIdentity.getClientId()))
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(channel, Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTACK)
                 .setBody(ca.toByteString())
                 .build()
@@ -329,7 +378,7 @@ public class SimpleCanalConnector implements CanalConnector {
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(channel, Packet.newBuilder()
+            writeWithHeader(Packet.newBuilder()
                 .setType(PacketType.CLIENTROLLBACK)
                 .setBody(ca.toByteString())
                 .build()
@@ -346,7 +395,15 @@ public class SimpleCanalConnector implements CanalConnector {
 
     // ==================== helper method ====================
 
-    private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
+    private void writeWithHeader(byte[] body) throws IOException {
+        writeWithHeader(writableChannel, body);
+    }
+
+    private byte[] readNextPacket() throws IOException {
+        return readNextPacket(readableChannel);
+    }
+
+    private void writeWithHeader(WritableByteChannel channel, byte[] body) throws IOException {
         synchronized (writeDataLock) {
             writeHeader.clear();
             writeHeader.putInt(body.length);
@@ -356,7 +413,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private byte[] readNextPacket(SocketChannel channel) throws IOException {
+    private byte[] readNextPacket(ReadableByteChannel channel) throws IOException {
         synchronized (readDataLock) {
             readHeader.clear();
             read(channel, readHeader);
@@ -367,7 +424,7 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-    private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
+    private void read(ReadableByteChannel channel, ByteBuffer buffer) throws IOException {
         while (buffer.hasRemaining()) {
             int r = channel.read(buffer);
             if (r == -1) {
@@ -411,7 +468,6 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
-
     private void waitClientRunning() {
         try {
             if (zkClientx != null) {
@@ -419,7 +475,11 @@ public class SimpleCanalConnector implements CanalConnector {
                     throw new CanalClientException("should connect first");
                 }
 
+                running = true;
                 mutex.get();// 阻塞等待
+            } else {
+                // 单机模式直接设置为running
+                running = true;
             }
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
@@ -435,6 +495,10 @@ public class SimpleCanalConnector implements CanalConnector {
         }
     }
 
+    public SocketAddress getNextAddress() {
+        return null;
+    }
+
     public SocketAddress getAddress() {
         return address;
     }
@@ -455,6 +519,14 @@ public class SimpleCanalConnector implements CanalConnector {
         this.soTimeout = soTimeout;
     }
 
+    public int getIdleTimeout() {
+        return idleTimeout;
+    }
+
+    public void setIdleTimeout(int idleTimeout) {
+        this.idleTimeout = idleTimeout;
+    }
+
     public void setZkClientx(ZkClientx zkClientx) {
         this.zkClientx = zkClientx;
         initClientRunningMonitor(this.clientIdentity);
@@ -472,4 +544,21 @@ public class SimpleCanalConnector implements CanalConnector {
         this.filter = filter;
     }
 
+    public boolean isLazyParseEntry() {
+        return lazyParseEntry;
+    }
+
+    public void setLazyParseEntry(boolean lazyParseEntry) {
+        this.lazyParseEntry = lazyParseEntry;
+    }
+
+    public void stopRunning() {
+        if (running) {
+            running = false; // 设置为非running状态
+            if (!mutex.state()) {
+                mutex.set(true); // 中断阻塞
+            }
+        }
+    }
+
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java

@@ -32,4 +32,9 @@ public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
         }
     }
 
+    @Override
+    public SocketAddress currentNode() {
+        return nodes.get(index);
+    }
+
 }

client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.client.impl.running;
 
 import java.net.InetSocketAddress;
+import java.text.MessageFormat;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -20,6 +21,7 @@ import com.alibaba.otter.canal.common.utils.BooleanMutex;
 import com.alibaba.otter.canal.common.utils.JsonUtils;
 import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 
 /**
  * clinet running控制
@@ -62,6 +64,8 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
             public void handleDataDeleted(String dataPath) throws Exception {
                 MDC.put("destination", destination);
                 mutex.set(false);
+                // 触发一下退出,可能是人为干预的释放操作或者网络闪断引起的session expired timeout
+                processActiveExit();
                 if (!release && activeData != null && isMine(activeData.getAddress())) {
                     // 如果上一次active的状态就是本机，则即时触发一下active抢占
                     initRunning();
@@ -94,9 +98,17 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
         String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
         zkClient.unsubscribeDataChanges(path, dataListener);
         releaseRunning(); // 尝试一下release
+        //Fix issue #697
+        if (delayExector != null) {
+            delayExector.shutdown();
+        }
     }
 
-    public void initRunning() {
+    // 改动记录：
+    // 1,在方法上加synchronized关键字,保证同步顺序执行;
+    // 2,判断Zk上已经存在的activeData是否是本机，是的话把mutex重置为true，否则会导致死锁
+    // 3,增加异常处理，保证出现异常时，running节点能被删除,否则会导致死锁
+    public synchronized void initRunning() {
         if (!isStart()) {
             return;
         }
@@ -116,11 +128,22 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
                 initRunning();
             } else {
                 activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+                // 如果发现已经存在,判断一下是否自己,避免活锁
+                if (activeData.getAddress().contains(":") && isMine(activeData.getAddress())) {
+                    mutex.set(true);
+                }
             }
         } catch (ZkNoNodeException e) {
             zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()),
                 true); // 尝试创建父节点
             initRunning();
+        } catch (Throwable t) {
+            logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].",
+                destination),
+                t);
+            // 出现任何异常尝试release
+            releaseRunning();
+            throw new CanalClientException("something goes wrong in initRunning method. ", t);
         }
     }
 

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -0,0 +1,241 @@
+package com.alibaba.otter.canal.client.kafka;
+
+import java.util.Collections;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.StringDeserializer;
+
+import com.alibaba.otter.canal.client.kafka.running.ClientRunningData;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
+/**
+ * canal kafka 数据操作客户端
+ *
+ * @author machengyuan @ 2018-6-12
+ * @version 1.0.0
+ */
+public class KafkaCanalConnector {
+
+    private KafkaConsumer<String, Message> kafkaConsumer;
+    private String                         topic;
+    private Integer                        partition;
+    private Properties                     properties;
+    // private ClientRunningMonitor runningMonitor; // 运行控制
+    // private BooleanMutex mutex = new BooleanMutex(false);
+    private ZkClientx                      zkClientx;
+    private volatile boolean               connected = false;
+    private volatile boolean               running   = false;
+
+    public KafkaCanalConnector(String zkServers, String servers, String topic, Integer partition, String groupId){
+        this.topic = topic;
+        this.partition = partition;
+
+        properties = new Properties();
+        properties.put("bootstrap.servers", servers);
+        properties.put("group.id", groupId);
+        properties.put("enable.auto.commit", false);
+        properties.put("auto.commit.interval.ms", "1000");
+        properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
+        properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
+        properties.put("session.timeout.ms", "30000"); // 默认为30秒
+        properties.put("max.poll.records", "1"); // 一次只取一条message数据
+        properties.put("key.deserializer", StringDeserializer.class.getName());
+        properties.put("value.deserializer", MessageDeserializer.class.getName());
+
+        if (zkServers != null) {
+            zkClientx = new ZkClientx(zkServers);
+
+            ClientRunningData clientData = new ClientRunningData();
+            clientData.setGroupId(groupId);
+            clientData.setAddress(AddressUtils.getHostIp());
+
+            // runningMonitor = new ClientRunningMonitor();
+            // runningMonitor.setTopic(topic);
+            // runningMonitor.setZkClient(zkClientx);
+            // runningMonitor.setClientData(clientData);
+            // runningMonitor.setListener(new ClientRunningListener() {
+            // public void processActiveEnter() {
+            // mutex.set(true);
+            // }
+            //
+            // public void processActiveExit() {
+            // mutex.set(false);
+            // }
+            // });
+        }
+
+    }
+
+    /**
+     * 重新设置sessionTime
+     *
+     * @param timeout
+     * @param unit
+     */
+    public void setSessionTimeout(Long timeout, TimeUnit unit) {
+        long t = unit.toMillis(timeout);
+        properties.put("request.timeout.ms", String.valueOf(t + 60000));
+        properties.put("session.timeout.ms", String.valueOf(t));
+    }
+
+    /**
+     * 打开连接
+     */
+    public void connect() {
+        if (connected) {
+            return;
+        }
+
+        // if (runningMonitor != null) {
+        // if (!runningMonitor.isStart()) {
+        // runningMonitor.start();
+        // }
+        // }
+
+        connected = true;
+
+        if (kafkaConsumer == null) {
+            kafkaConsumer = new KafkaConsumer<String, Message>(properties);
+        }
+    }
+
+    /**
+     * 关闭链接
+     */
+    public void disconnnect() {
+        kafkaConsumer.close();
+
+        connected = false;
+        // if (runningMonitor.isStart()) {
+        // runningMonitor.stop();
+        // }
+    }
+
+    private void waitClientRunning() {
+        try {
+            if (zkClientx != null) {
+                if (!connected) {// 未调用connect
+                    throw new CanalClientException("should connect first");
+                }
+
+                running = true;
+                // mutex.get();// 阻塞等待
+            } else {
+                // 单机模式直接设置为running
+                running = true;
+            }
+        } catch (Exception e) {
+            Thread.currentThread().interrupt();
+            throw new CanalClientException(e);
+        }
+    }
+
+    public boolean checkValid() {
+        if (zkClientx != null) {
+            // return mutex.state();
+            return true;
+        } else {
+            return true;// 默认都放过
+        }
+    }
+
+    /**
+     * 订阅topic
+     */
+    public void subscribe() {
+        waitClientRunning();
+        if (!running) {
+            return;
+        }
+
+        if (partition == null) {
+            kafkaConsumer.subscribe(Collections.singletonList(topic));
+        } else {
+            TopicPartition topicPartition = new TopicPartition(topic, partition);
+            kafkaConsumer.assign(Collections.singletonList(topicPartition));
+        }
+    }
+
+    /**
+     * 取消订阅
+     */
+    public void unsubscribe() {
+        waitClientRunning();
+        if (!running) {
+            return;
+        }
+
+        kafkaConsumer.unsubscribe();
+    }
+
+    /**
+     * 获取数据，自动进行确认
+     *
+     * @return
+     */
+    public Message get() {
+        return get(100L, TimeUnit.MILLISECONDS);
+    }
+
+    public Message get(Long timeout, TimeUnit unit) {
+        waitClientRunning();
+        if (!running) {
+            return null;
+        }
+
+        Message message = getWithoutAck(timeout, unit);
+        this.ack();
+        return message;
+    }
+
+    public Message getWithoutAck() {
+        return getWithoutAck(100L, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * 获取数据，不进行确认，等待处理完成手工确认
+     *
+     * @return
+     */
+    public Message getWithoutAck(Long timeout, TimeUnit unit) {
+        waitClientRunning();
+        if (!running) {
+            return null;
+        }
+
+        ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据
+
+        if (!records.isEmpty()) {
+            return records.iterator().next().value();
+        }
+        return null;
+    }
+
+    /**
+     * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败
+     */
+    public void ack() {
+        waitClientRunning();
+        if (!running) {
+            return;
+        }
+
+        kafkaConsumer.commitSync();
+    }
+
+    public void stopRunning() {
+        if (running) {
+            running = false; // 设置为非running状态
+            // if (!mutex.state()) {
+            // mutex.set(true); // 中断阻塞
+            // }
+        }
+    }
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java

@@ -0,0 +1,62 @@
+package com.alibaba.otter.canal.client.kafka;
+
+/**
+ * canal kafka connectors创建工具类
+ *
+ * @author machengyuan @ 2018-6-12
+ * @version 1.0.0
+ */
+public class KafkaCanalConnectors {
+
+    /**
+     * 创建kafka客户端链接，独立运行不注册zk信息
+     *
+     * @param servers
+     * @param topic
+     * @param partition
+     * @param groupId
+     * @return
+     */
+    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) {
+        return new KafkaCanalConnector(null, servers, topic, partition, groupId);
+    }
+
+    /**
+     * 创建kafka客户端链接，独立运行不注册zk信息
+     *
+     * @param servers
+     * @param topic
+     * @param groupId
+     * @return
+     */
+    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
+        return new KafkaCanalConnector(null, servers, topic, null, groupId);
+    }
+
+    /**
+     * 创建kafka客户端链接
+     *
+     * @param zkServers
+     * @param servers
+     * @param topic
+     * @param partition
+     * @param groupId
+     * @return
+     */
+    public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, Integer partition, String groupId) {
+        return new KafkaCanalConnector(zkServers, servers, topic, partition, groupId);
+    }
+
+    /**
+     * 创建kafka客户端链接
+     *
+     * @param zkServers
+     * @param servers
+     * @param topic
+     * @param groupId
+     * @return
+     */
+    public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, String groupId) {
+        return new KafkaCanalConnector(zkServers, servers, topic, null, groupId);
+    }
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java

@@ -0,0 +1,64 @@
+package com.alibaba.otter.canal.client.kafka;
+
+import java.util.Map;
+
+import org.apache.kafka.common.serialization.Deserializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.protobuf.ByteString;
+
+/**
+ * Kafka Message类的反序列化
+ *
+ * @author machengyuan @ 2018-6-12
+ * @version 1.0.0
+ */
+public class MessageDeserializer implements Deserializer<Message> {
+
+    private static Logger logger = LoggerFactory.getLogger(MessageDeserializer.class);
+
+    @Override
+    public void configure(Map<String, ?> configs, boolean isKey) {
+    }
+
+    @Override
+    public Message deserialize(String topic, byte[] data) {
+        try {
+            if (data == null) {
+                return null;
+            }
+            else {
+                CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
+                switch (p.getType()) {
+                    case MESSAGES: {
+                        if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
+                            throw new CanalClientException("compression is not supported in this connector");
+                        }
+
+                        CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
+                        Message result = new Message(messages.getBatchId());
+                        for (ByteString byteString : messages.getMessagesList()) {
+                            result.addEntry(CanalEntry.Entry.parseFrom(byteString));
+                        }
+                        return result;
+                    }
+                    default:
+                        break;
+                }
+            }
+        } catch (Exception e) {
+            logger.error("Error when deserializing byte[] to message ");
+        }
+        return null;
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningData.java

@@ -0,0 +1,39 @@
+package com.alibaba.otter.canal.client.kafka.running;
+
+/**
+ * client running状态信息
+ *
+ * @author machengyuan 2018-06-20 下午04:10:12
+ * @version 1.0.0
+ */
+public class ClientRunningData {
+
+    private String  groupId;
+    private String  address;
+    private boolean active = true;
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public void setAddress(String address) {
+        this.address = address;
+    }
+
+    public boolean isActive() {
+        return active;
+    }
+
+    public void setActive(boolean active) {
+        this.active = active;
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningListener.java

@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.client.kafka.running;
+
+/**
+ * client running状态信息
+ *
+ * @author machengyuan 2018-06-20 下午04:10:12
+ * @version 1.0.0
+ */
+public interface ClientRunningListener {
+
+    /**
+     * 触发现在轮到自己做为active
+     */
+    public void processActiveEnter();
+
+    /**
+     * 触发一下当前active模式失败
+     */
+    public void processActiveExit();
+
+}

client/src/main/java/com/alibaba/otter/canal/client/kafka/running/ClientRunningMonitor.java

@@ -0,0 +1,281 @@
+package com.alibaba.otter.canal.client.kafka.running;
+
+import java.text.MessageFormat;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.I0Itec.zkclient.IZkDataListener;
+import org.I0Itec.zkclient.exception.ZkException;
+import org.I0Itec.zkclient.exception.ZkInterruptedException;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.utils.BooleanMutex;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
+
+/**
+ * kafka client running状态信息
+ *
+ * @author machengyuan 2018-06-20 下午04:10:12
+ * @version 1.0.0
+ */
+public class ClientRunningMonitor extends AbstractCanalLifeCycle {
+
+    private static final String TOPIC_ROOT_NODE             = ZookeeperPathUtils.CANAL_ROOT_NODE
+                                                              + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + "topics";
+
+    private static final String TOPIC_NODE                  = TOPIC_ROOT_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
+                                                              + "{0}";
+
+    private static final String TOPIC_CLIENTID_NODE         = TOPIC_NODE + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
+                                                              + "{1}";
+
+    private static final String TOPIC_CLIENTID_RUNNING_NODE = TOPIC_CLIENTID_NODE
+                                                              + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR
+                                                              + ZookeeperPathUtils.RUNNING_NODE;
+
+    private static String getTopicClientRunning(String topic, String groupId) {
+        return MessageFormat.format(TOPIC_CLIENTID_RUNNING_NODE, topic, groupId);
+    }
+
+    private static String getClientIdNodePath(String topic, String groupId) {
+        return MessageFormat.format(TOPIC_CLIENTID_NODE, topic, groupId);
+    }
+
+    private static final Logger        logger       = LoggerFactory.getLogger(ClientRunningMonitor.class);
+    private ZkClientx zkClient;
+    private String                     topic;
+    private ClientRunningData clientData;
+    private IZkDataListener            dataListener;
+    private BooleanMutex mutex        = new BooleanMutex(false);
+    private volatile boolean           release      = false;
+    private volatile ClientRunningData activeData;
+    private ScheduledExecutorService   delayExector = Executors.newScheduledThreadPool(1);
+    private ClientRunningListener listener;
+    private int                        delayTime    = 5;
+
+    private static Integer             virtualPort;
+
+    public ClientRunningMonitor(){
+        if (virtualPort == null) {
+            Random rand = new Random();
+            virtualPort = rand.nextInt(9000) + 1000;
+        }
+
+        dataListener = new IZkDataListener() {
+
+            public void handleDataChange(String dataPath, Object data) throws Exception {
+                MDC.put("kafkaTopic", topic);
+                ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class);
+                if (!isMine(runningData.getAddress())) {
+                    mutex.set(false);
+                }
+
+                if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
+                    release = true;
+                    releaseRunning();// 彻底释放mainstem
+                }
+
+                activeData = (ClientRunningData) runningData;
+            }
+
+            public void handleDataDeleted(String dataPath) throws Exception {
+                MDC.put("kafkaTopic", topic);
+
+                mutex.set(false);
+                // 触发一下退出,可能是人为干预的释放操作或者网络闪断引起的session expired timeout
+                processActiveExit();
+                if (!release && activeData != null && isMine(activeData.getAddress())) {
+                    // 如果上一次active的状态就是本机，则即时触发一下active抢占
+                    initRunning();
+                } else {
+                    // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
+                    delayExector.schedule(new Runnable() {
+
+                        public void run() {
+                            initRunning();
+                        }
+                    }, delayTime, TimeUnit.SECONDS);
+                }
+            }
+
+        };
+
+    }
+
+    public void start() {
+        super.start();
+
+        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
+
+        zkClient.subscribeDataChanges(path, dataListener);
+        initRunning();
+    }
+
+    public void stop() {
+        super.stop();
+        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
+        zkClient.unsubscribeDataChanges(path, dataListener);
+        releaseRunning(); // 尝试一下release
+        // Fix issue #697
+        if (delayExector != null) {
+            delayExector.shutdown();
+        }
+    }
+
+    // 改动记录：
+    // 1,在方法上加synchronized关键字,保证同步顺序执行;
+    // 2,判断Zk上已经存在的activeData是否是本机，是的话把mutex重置为true，否则会导致死锁
+    // 3,增加异常处理，保证出现异常时，running节点能被删除,否则会导致死锁
+    public synchronized void initRunning() {
+        if (!isStart()) {
+            return;
+        }
+
+        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
+        // 序列化
+        byte[] bytes = JsonUtils.marshalToByte(clientData);
+        try {
+            mutex.set(false);
+            zkClient.create(path, bytes, CreateMode.EPHEMERAL);
+            processActiveEnter();// 触发一下事件
+            activeData = clientData;
+            mutex.set(true);
+        } catch (ZkNodeExistsException e) {
+            bytes = zkClient.readData(path, true);
+            if (bytes == null) {// 如果不存在节点，立即尝试一次
+                initRunning();
+            } else {
+                activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+                // 如果发现已经存在,判断一下是否自己,避免活锁
+                if (activeData.getAddress().contains(":") && isMine(activeData.getAddress())) {
+                    mutex.set(true);
+                }
+            }
+        } catch (ZkNoNodeException e) {
+            zkClient.createPersistent(getClientIdNodePath(this.topic, clientData.getGroupId()), true); // 尝试创建父节点
+            initRunning();
+        } catch (Throwable t) {
+            logger.error(MessageFormat.format("There is an error when execute initRunning method, with destination [{0}].",
+                topic),
+                t);
+            // 出现任何异常尝试release
+            releaseRunning();
+            throw new CanalClientException("something goes wrong in initRunning method. ", t);
+        }
+    }
+
+    /**
+     * 阻塞等待自己成为active，如果自己成为active，立马返回
+     *
+     * @throws InterruptedException
+     */
+    public void waitForActive() throws InterruptedException {
+        initRunning();
+        mutex.get();
+    }
+
+    /**
+     * 检查当前的状态
+     */
+    public boolean check() {
+        String path = getTopicClientRunning(this.topic, clientData.getGroupId());
+        // ZookeeperPathUtils.getDestinationClientRunning(this.destination,
+        // clientData.getClientId());
+        try {
+            byte[] bytes = zkClient.readData(path);
+            ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+            activeData = eventData;// 更新下为最新值
+            // 检查下nid是否为自己
+            boolean result = isMine(activeData.getAddress());
+            if (!result) {
+                logger.warn("canal is running in [{}] , but not in [{}]",
+                    activeData.getAddress(),
+                    clientData.getAddress());
+            }
+            return result;
+        } catch (ZkNoNodeException e) {
+            logger.warn("canal is not run any in node");
+            return false;
+        } catch (ZkInterruptedException e) {
+            logger.warn("canal check is interrupt");
+            Thread.interrupted();// 清除interrupt标记
+            return check();
+        } catch (ZkException e) {
+            logger.warn("canal check is failed");
+            return false;
+        }
+    }
+
+    public boolean releaseRunning() {
+        if (check()) {
+            String path = getTopicClientRunning(this.topic, clientData.getGroupId());
+            zkClient.delete(path);
+            mutex.set(false);
+            processActiveExit();
+            return true;
+        }
+
+        return false;
+    }
+
+    // ====================== helper method ======================
+
+    private boolean isMine(String address) {
+        return address.equals(clientData.getAddress());
+    }
+
+    private void processActiveEnter() {
+        if (listener != null) {
+            // 触发回调
+            listener.processActiveEnter();
+            this.clientData.setAddress(/* address */AddressUtils.getHostIp() + ":" + virtualPort);
+
+            String path = getTopicClientRunning(this.topic, clientData.getGroupId());
+            // 序列化
+            byte[] bytes = JsonUtils.marshalToByte(clientData);
+            zkClient.writeData(path, bytes);
+        }
+    }
+
+    private void processActiveExit() {
+        if (listener != null) {
+            listener.processActiveExit();
+        }
+    }
+
+    public void setListener(ClientRunningListener listener) {
+        this.listener = listener;
+    }
+
+    // ===================== setter / getter =======================
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public void setClientData(ClientRunningData clientData) {
+        this.clientData = clientData;
+    }
+
+    public void setDelayTime(int delayTime) {
+        this.delayTime = delayTime;
+    }
+
+    public void setZkClient(ZkClientx zkClient) {
+        this.zkClient = zkClient;
+    }
+
+}

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/AbstractKafkaTest.java

@@ -0,0 +1,26 @@
+package com.alibaba.otter.canal.client.running.kafka;
+
+import org.junit.Assert;
+
+/**
+ * Kafka 测试基类
+ *
+ * @author machengyuan @ 2018-6-12
+ * @version 1.0.0
+ */
+public abstract class AbstractKafkaTest {
+
+    public static String  topic     = "example";
+    public static Integer partition = null;
+    public static String  groupId   = "g4";
+    public static String  servers   = "slave1:6667,slave2:6667,slave3:6667";
+    public static String  zkServers = "slave1:2181,slave2:2181,slave3:2181";
+
+    public void sleep(long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+}

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java

@@ -0,0 +1,144 @@
+package com.alibaba.otter.canal.client.running.kafka;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.common.errors.WakeupException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Kafka client example
+ *
+ * @author machengyuan @ 2018-6-12
+ * @version 1.0.0
+ */
+public class CanalKafkaClientExample {
+
+    protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
+
+    private KafkaCanalConnector connector;
+
+    private static volatile boolean running = false;
+
+    private Thread thread = null;
+
+    private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
+
+        public void uncaughtException(Thread t, Throwable e) {
+            logger.error("parse events has an error", e);
+        }
+    };
+
+    public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId) {
+        connector = KafkaCanalConnectors.newKafkaConnector(zkServers, servers, topic, partition, groupId);
+    }
+
+    public static void main(String[] args) {
+        try {
+            final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
+                    AbstractKafkaTest.servers,
+                    AbstractKafkaTest.topic,
+                    AbstractKafkaTest.partition,
+                    AbstractKafkaTest.groupId);
+            logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
+            kafkaCanalClientExample.start();
+            logger.info("## the canal kafka consumer is running now ......");
+            Runtime.getRuntime().addShutdownHook(new Thread() {
+
+                public void run() {
+                    try {
+                        logger.info("## stop the kafka consumer");
+                        kafkaCanalClientExample.stop();
+                    } catch (Throwable e) {
+                        logger.warn("##something goes wrong when stopping kafka consumer:", e);
+                    } finally {
+                        logger.info("## kafka consumer is down.");
+                    }
+                }
+
+            });
+            while (running)
+                ;
+        } catch (Throwable e) {
+            logger.error("## Something goes wrong when starting up the kafka consumer:", e);
+            System.exit(0);
+        }
+    }
+
+    public void start() {
+        Assert.notNull(connector, "connector is null");
+        thread = new Thread(new Runnable() {
+
+            public void run() {
+                process();
+            }
+        });
+        thread.setUncaughtExceptionHandler(handler);
+        thread.start();
+        running = true;
+    }
+
+    public void stop() {
+        if (!running) {
+            return;
+        }
+        connector.stopRunning();
+        running = false;
+        if (thread != null) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+    }
+
+    private void process() {
+        while (!running)
+            ;
+        while (running) {
+            try {
+                connector.connect();
+                connector.subscribe();
+                while (running) {
+                    try {
+                        Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
+                        if (message == null) {
+                            continue;
+                        }
+                        long batchId = message.getId();
+                        int size = message.getEntries().size();
+                        if (batchId == -1 || size == 0) {
+                            // try {
+                            // Thread.sleep(1000);
+                            // } catch (InterruptedException e) {
+                            // }
+                        } else {
+                            // printSummary(message, batchId, size);
+                            // printEntry(message.getEntries());
+                            logger.info(message.toString());
+                        }
+
+                        connector.ack(); // 提交确认
+                    } catch (Exception e) {
+                        logger.error(e.getMessage(), e);
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+
+        try {
+            connector.unsubscribe();
+        } catch (WakeupException e) {
+            // No-op. Continue process
+        }
+        connector.disconnnect();
+    }
+}

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java

@@ -0,0 +1,62 @@
+package com.alibaba.otter.canal.client.running.kafka;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.common.errors.WakeupException;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
+import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Kafka consumer获取Message的测试例子
+ *
+ * @author machengyuan @ 2018-6-12
+ * @version 1.0.0
+ */
+public class KafkaClientRunningTest extends AbstractKafkaTest {
+
+    private Logger  logger  = LoggerFactory.getLogger(KafkaClientRunningTest.class);
+
+    private boolean running = true;
+
+    @Test
+    public void testKafkaConsumer() {
+        final ExecutorService executor = Executors.newFixedThreadPool(1);
+
+        final KafkaCanalConnector connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId);
+
+        executor.submit(new Runnable() {
+
+            @Override
+            public void run() {
+                connector.connect();
+                connector.subscribe();
+                while (running) {
+                    try {
+                        Message message = connector.getWithoutAck(3L, TimeUnit.SECONDS);
+                        if (message != null) {
+                            System.out.println(message);
+                        }
+                        connector.ack();
+                    } catch (WakeupException e) {
+                        // ignore
+                    }
+                }
+                connector.unsubscribe();
+                connector.disconnnect();
+            }
+        });
+
+        sleep(60000);
+        running = false;
+        executor.shutdown();
+        logger.info("shutdown completed");
+    }
+
+}

client/src/test/java/logback.xml

@@ -8,7 +8,7 @@
 		</encoder>
 	</appender>
 	
-	<root level="WARN">
+	<root level="INFO">
 		<appender-ref ref="STDOUT"/>
 	</root>
 </configuration>

codeformat.xml

@@ -0,0 +1,291 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<profiles version="12">
+<profile kind="CodeFormatterProfile" name="canal-format" version="12">
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
+<setting id="org.eclipse.jdt.core.compiler.source" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="120"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="84"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="18"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="true"/>
+</profile>
+</profiles>

codetemplates.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><templates><template autoinsert="true" context="gettercomment_context" deleted="false" description="Comment for getter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name="gettercomment">/**
+ * @return the ${bare_field_name}
+ */</template><template autoinsert="true" context="settercomment_context" deleted="false" description="Comment for setter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.settercomment" name="settercomment">/**
+ * @param ${param} the ${bare_field_name} to set
+ */</template><template autoinsert="true" context="constructorcomment_context" deleted="false" description="Comment for created constructors" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name="constructorcomment">/**
+ * ${tags}
+ */</template><template autoinsert="false" context="filecomment_context" deleted="false" description="Comment for created Java files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.filecomment" name="filecomment"/><template autoinsert="false" context="typecomment_context" deleted="false" description="Comment for created types" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.typecomment" name="typecomment">/**
+ * @author ${user} ${date} ${time}
+ * @since 1.0.0
+ */</template><template autoinsert="true" context="fieldcomment_context" deleted="false" description="Comment for fields" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name="fieldcomment">/**
+ * 
+ */</template><template autoinsert="true" context="methodcomment_context" deleted="false" description="Comment for non-overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name="methodcomment">/**
+ * ${tags}
+ */</template><template autoinsert="false" context="overridecomment_context" deleted="false" description="Comment for overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name="overridecomment"/><template autoinsert="true" context="delegatecomment_context" deleted="false" description="Comment for delegate methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name="delegatecomment">/**
+ * ${tags}
+ * ${see_to_target}
+ */</template><template autoinsert="false" context="newtype_context" deleted="false" description="Newly created files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.newtype" name="newtype">${filecomment}
+${package_declaration}
+${typecomment}
+${type_declaration}</template><template autoinsert="true" context="classbody_context" deleted="false" description="Code in new class type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.classbody" name="classbody">
+</template><template autoinsert="true" context="interfacebody_context" deleted="false" description="Code in new interface type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name="interfacebody">
+</template><template autoinsert="true" context="enumbody_context" deleted="false" description="Code in new enum type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.enumbody" name="enumbody">
+</template><template autoinsert="true" context="annotationbody_context" deleted="false" description="Code in new annotation type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name="annotationbody">
+</template><template autoinsert="true" context="catchblock_context" deleted="false" description="Code in new catch blocks" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.catchblock" name="catchblock">// ${todo} Auto-generated catch block
+${exception_var}.printStackTrace();</template><template autoinsert="true" context="methodbody_context" deleted="false" description="Code in created method stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodbody" name="methodbody">// ${todo} Auto-generated method stub
+${body_statement}</template><template autoinsert="true" context="constructorbody_context" deleted="false" description="Code in created constructor stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name="constructorbody">${body_statement}
+// ${todo} Auto-generated constructor stub</template><template autoinsert="true" context="getterbody_context" deleted="false" description="Code in created getters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.getterbody" name="getterbody">return ${field};</template><template autoinsert="true" context="setterbody_context" deleted="false" description="Code in created setters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.setterbody" name="setterbody">${field} = ${param};</template></templates>

common/pom.xml

@@ -3,22 +3,26 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.1.1-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
-	<groupId>com.alibaba.otter</groupId>
 	<artifactId>canal.common</artifactId>
 	<packaging>jar</packaging>
 	<name>canal common module for otter ${project.version}</name>
 	<url>http://b2b-doc.alibaba-inc.com/display/opentech/Otter</url>
 	<dependencies>
+		<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
+		<dependency>
+			<groupId>io.netty</groupId>
+			<artifactId>netty-all</artifactId>
+		</dependency>
 		<!-- zk -->
 		<dependency>
 			<groupId>org.apache.zookeeper</groupId>
 			<artifactId>zookeeper</artifactId>
 		</dependency>
 		<dependency>
-			<groupId>com.github.sgroschupf</groupId>
+			<groupId>com.101tec</groupId>
 			<artifactId>zkclient</artifactId>
 		</dependency>
 		<!-- external -->
@@ -29,11 +33,10 @@
 		<dependency>
 			<groupId>commons-lang</groupId>
 			<artifactId>commons-lang</artifactId>
-			<version>2.6</version>
 		</dependency>
 		<dependency>
-			<groupId>org.springframework</groupId>
+			<groupId>commons-codec</groupId>
-			<artifactId>spring</artifactId>
+			<artifactId>commons-codec</artifactId>
 		</dependency>
 		<dependency>
 			<groupId>com.alibaba</groupId>
@@ -60,10 +63,37 @@
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-aop</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-context</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-jdbc</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-orm</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-test</artifactId>
+			<version>${spring_version}</version>
+			<scope>test</scope>
+		</dependency>
 		<!-- junit -->
 		<dependency>
-		  <groupId>junit</groupId>
+			<groupId>junit</groupId>
-		  <artifactId>junit</artifactId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
 		</dependency>
-  </dependencies>
+	</dependencies>
 </project>

common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java

@@ -6,9 +6,9 @@ package com.alibaba.otter.canal.common;
  */
 public interface CanalLifeCycle {
 
-    public void start();
+    void start();
 
-    public void stop();
+    void stop();
 
-    public boolean isStart();
+    boolean isStart();
 }

common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java

@@ -14,8 +14,7 @@ public interface CanalAlarmHandler extends CanalLifeCycle {
      * 发送对应destination的报警
      * 
      * @param destination
-     * @param title
      * @param msg
      */
-    public void sendAlarm(String destination, String msg);
+    void sendAlarm(String destination, String msg);
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java

@@ -38,7 +38,8 @@ public class AddressUtils {
     private static boolean isValidHostAddress(InetAddress address) {
         if (address == null || address.isLoopbackAddress()) return false;
         String name = address.getHostAddress();
-        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches());
+        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name)
+            .matches());
     }
 
     public static String getHostIp() {

common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java

@@ -1,12 +1,20 @@
 package com.alibaba.otter.canal.common.utils;
 
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.net.Inet4Address;
+import java.net.Inet6Address;
+import java.net.InetAddress;
 import java.util.Arrays;
 import java.util.List;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson.parser.ParserConfig;
 import com.alibaba.fastjson.serializer.JSONSerializer;
+import com.alibaba.fastjson.serializer.ObjectSerializer;
 import com.alibaba.fastjson.serializer.PropertyFilter;
+import com.alibaba.fastjson.serializer.SerializeConfig;
 import com.alibaba.fastjson.serializer.SerializeWriter;
 import com.alibaba.fastjson.serializer.SerializerFeature;
 
@@ -17,6 +25,13 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
  */
 public class JsonUtils {
 
+    static {
+        SerializeConfig.getGlobalInstance().put(InetAddress.class, InetAddressSerializer.instance);
+        SerializeConfig.getGlobalInstance().put(Inet4Address.class, InetAddressSerializer.instance);
+        SerializeConfig.getGlobalInstance().put(Inet6Address.class, InetAddressSerializer.instance);
+        ParserConfig.getGlobalInstance().setAutoTypeSupport(true);
+    }
+
     public static <T> T unmarshalFromByte(byte[] bytes, Class<T> targetClass) {
         return (T) JSON.parseObject(bytes, targetClass);// 默认为UTF-8
     }
@@ -70,4 +85,22 @@ public class JsonUtils {
             out.close();
         }
     }
+
+    public static class InetAddressSerializer implements ObjectSerializer {
+
+        public static InetAddressSerializer instance = new InetAddressSerializer();
+
+        @Override
+        public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features)
+                                                                                                                   throws IOException {
+            if (object == null) {
+                serializer.writeNull();
+                return;
+            }
+
+            InetAddress address = (InetAddress) object;
+            // 优先使用name
+            serializer.write(address.getHostName());
+        }
+    }
 }

common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java

@@ -23,6 +23,11 @@ public class NamedThreadFactory implements ThreadFactory {
 
                                                                        public void uncaughtException(Thread t,
                                                                                                      Throwable e) {
+                                                                           if (e instanceof InterruptedException
+                                                                               || (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
+                                                                               return;
+                                                                           }
+
                                                                            logger.error("from " + t.getName(), e);
                                                                        }
                                                                    };

common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java

@@ -54,6 +54,7 @@ public final class UriUtils {
         }
         String query = uri.getRawQuery();
         HashMap<String, String> params = new HashMap<String, String>();
+        @SuppressWarnings("resource")
         Scanner scan = new Scanner(query);
         scan.useDelimiter(SPLIT);
         while (scan.hasNext()) {

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java

@@ -12,7 +12,7 @@ import org.I0Itec.zkclient.serialize.ZkSerializer;
 import org.apache.zookeeper.CreateMode;
 
 import com.google.common.base.Function;
-import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
 
 /**
  * 使用自定义的ZooKeeperx for zk connection
@@ -23,7 +23,7 @@ import com.google.common.collect.MapMaker;
 public class ZkClientx extends ZkClient {
 
     // 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
-    private static Map<String, ZkClientx> clients = new MapMaker().makeComputingMap(new Function<String, ZkClientx>() {
+    private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>() {
 
                                                       public ZkClientx apply(String servers) {
                                                           return new ZkClientx(servers);
@@ -62,10 +62,12 @@ public class ZkClientx extends ZkClient {
      * Create a persistent Sequential node.
      * 
      * @param path
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * @param createParents if true all parent dirs are created as well and no
-     * in case the path already exists
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws ZkInterruptedException if operation was interrupted, or a
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */
@@ -89,10 +91,12 @@ public class ZkClientx extends ZkClient {
      * 
      * @param path
      * @param data
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * @param createParents if true all parent dirs are created as well and no
-     * in case the path already exists
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws ZkInterruptedException if operation was interrupted, or a
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */
@@ -118,10 +122,12 @@ public class ZkClientx extends ZkClient {
      * 
      * @param path
      * @param data
-     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * @param createParents if true all parent dirs are created as well and no
-     * in case the path already exists
+     * {@link ZkNodeExistsException} is thrown in case the path already exists
-     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws ZkInterruptedException if operation was interrupted, or a
-     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the
+     * ZooKeeper event thread
      * @throws ZkException if any ZooKeeper exception occurred
      * @throws RuntimeException if any other exception occurs
      */

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java

@@ -8,20 +8,15 @@ import java.util.List;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.I0Itec.zkclient.IZkConnection;
+import org.I0Itec.zkclient.ZkConnection;
 import org.I0Itec.zkclient.exception.ZkException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zookeeper.ClientCnxn;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.client.ConnectStringParser;
 import org.apache.zookeeper.client.HostProvider;
 import org.apache.zookeeper.client.StaticHostProvider;
-import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.ReflectionUtils;
@@ -32,20 +27,20 @@ import org.springframework.util.ReflectionUtils;
  * @author jianghang 2012-7-10 下午02:31:42
  * @version 1.0.0
  */
-public class ZooKeeperx implements IZkConnection {
+public class ZooKeeperx extends ZkConnection {
 
     private static final String SERVER_COMMA            = ";";
     private static final Logger logger                  = LoggerFactory.getLogger(ZooKeeperx.class);
     private static final Field  clientCnxnField         = ReflectionUtils.findField(ZooKeeper.class, "cnxn");
     private static final Field  hostProviderField       = ReflectionUtils.findField(ClientCnxn.class, "hostProvider");
     private static final Field  serverAddressesField    = ReflectionUtils.findField(StaticHostProvider.class,
-                                                                                    "serverAddresses");
+                                                            "serverAddresses");
+    private static final Field  zookeeperLockField      = ReflectionUtils.findField(ZkConnection.class,
+                                                            "_zookeeperLock");
+    private static final Field  zookeeperFiled          = ReflectionUtils.findField(ZkConnection.class, "_zk");
     private static final int    DEFAULT_SESSION_TIMEOUT = 90000;
 
-    private ZooKeeper           _zk                     = null;
+    private final List<String>  _serversList;
-    private Lock                _zookeeperLock          = new ReentrantLock();
-
-    private final List<String>  _servers;
     private final int           _sessionTimeOut;
 
     public ZooKeeperx(String zkServers){
@@ -53,115 +48,60 @@ public class ZooKeeperx implements IZkConnection {
     }
 
     public ZooKeeperx(String zkServers, int sessionTimeOut){
-        _servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA));
+        super(zkServers, sessionTimeOut);
+        _serversList = Arrays.asList(StringUtils.split(this.getServers(), SERVER_COMMA));
         _sessionTimeOut = sessionTimeOut;
     }
 
     @Override
     public void connect(Watcher watcher) {
+        ReflectionUtils.makeAccessible(zookeeperLockField);
+        ReflectionUtils.makeAccessible(zookeeperFiled);
+        Lock _zookeeperLock = (ReentrantLock) ReflectionUtils.getField(zookeeperLockField, this);
+        ZooKeeper _zk = (ZooKeeper) ReflectionUtils.getField(zookeeperFiled, this);
+
         _zookeeperLock.lock();
         try {
             if (_zk != null) {
                 throw new IllegalStateException("zk client has already been started");
             }
+            String zkServers = _serversList.get(0);
 
             try {
-                logger.debug("Creating new ZookKeeper instance to connect to " + _servers + ".");
+                logger.debug("Creating new ZookKeeper instance to connect to " + zkServers + ".");
-                _zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher);
+                _zk = new ZooKeeper(zkServers, _sessionTimeOut, watcher);
                 configMutliCluster(_zk);
+                ReflectionUtils.setField(zookeeperFiled, this, _zk);
             } catch (IOException e) {
-                throw new ZkException("Unable to connect to " + _servers, e);
+                throw new ZkException("Unable to connect to " + zkServers, e);
             }
         } finally {
             _zookeeperLock.unlock();
         }
     }
 
-    public void close() throws InterruptedException {
-        _zookeeperLock.lock();
-        try {
-            if (_zk != null) {
-                logger.debug("Closing ZooKeeper connected to " + _servers);
-                _zk.close();
-                _zk = null;
-            }
-        } finally {
-            _zookeeperLock.unlock();
-        }
-    }
-
-    public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException {
-        return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode);
-    }
-
-    public void delete(String path) throws InterruptedException, KeeperException {
-        _zk.delete(path, -1);
-    }
-
-    public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException {
-        return _zk.exists(path, watch) != null;
-    }
-
-    public List<String> getChildren(final String path, final boolean watch) throws KeeperException,
-                                                                           InterruptedException {
-        return _zk.getChildren(path, watch);
-    }
-
-    public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException {
-        return _zk.getData(path, watch, stat);
-    }
-
-    public void writeData(String path, byte[] data) throws KeeperException, InterruptedException {
-        writeData(path, data, -1);
-    }
-
-    public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException {
-        _zk.setData(path, data, version);
-    }
-
-    public States getZookeeperState() {
-        return _zk != null ? _zk.getState() : null;
-    }
-
-    public ZooKeeper getZookeeper() {
-        return _zk;
-    }
-
-    public long getCreateTime(String path) throws KeeperException, InterruptedException {
-        Stat stat = _zk.exists(path, false);
-        if (stat != null) {
-            return stat.getCtime();
-        }
-        return -1;
-    }
-
-    public String getServers() {
-        return StringUtils.join(_servers, SERVER_COMMA);
-    }
-
     // ===============================
 
     public void configMutliCluster(ZooKeeper zk) {
-        if (_servers.size() == 1) {
+        if (_serversList.size() == 1) {
             return;
         }
-        String cluster1 = _servers.get(0);
+        String cluster1 = _serversList.get(0);
         try {
-            if (_servers.size() > 1) {
+            if (_serversList.size() > 1) {
                 // 强制的声明accessible
                 ReflectionUtils.makeAccessible(clientCnxnField);
                 ReflectionUtils.makeAccessible(hostProviderField);
                 ReflectionUtils.makeAccessible(serverAddressesField);
 
                 // 添加第二组集群列表
-                for (int i = 1; i < _servers.size(); i++) {
+                for (int i = 1; i < _serversList.size(); i++) {
-                    String cluster = _servers.get(i);
+                    String cluster = _serversList.get(i);
                     // 强制获取zk中的地址信息
                     ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
                     HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
-                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(
+                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(serverAddressesField,
-                                                                                                             serverAddressesField,
+                        hostProvider);
-                                                                                                             hostProvider);
                     // 添加第二组集群列表
                     serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());
                 }

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java

@@ -120,8 +120,10 @@ public class ZookeeperPathUtils {
     }
 
     public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) {
-        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName,
+        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH,
-                                    String.valueOf(clientId), getBatchMarkNode(batchId));
+            destinationName,
+            String.valueOf(clientId),
+            getBatchMarkNode(batchId));
     }
 
     public static String getCursorPath(String destination, short clientId) {

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java

@@ -87,18 +87,29 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
 
     }
 
-    public void start() {
+    public void init() {
-        super.start();
         processStart();
-        if (zkClient != null) {
+    }
-            // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
-            String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
-            zkClient.subscribeDataChanges(path, dataListener);
 
-            initRunning();
+    public synchronized void start() {
-        } else {
+        super.start();
-            processActiveEnter();// 没有zk，直接启动
+        try {
+            processStart();
+            if (zkClient != null) {
+                // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
+                String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
+                zkClient.subscribeDataChanges(path, dataListener);
+
+                initRunning();
+            } else {
+                processActiveEnter();// 没有zk，直接启动
+            }
+        } catch (Exception e) {
+            logger.error("start failed", e);
+            // 没有正常启动，重置一下状态，避免干扰下一次start
+            stop();
         }
+
     }
 
     public void release() {
@@ -109,7 +120,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
         }
     }
 
-    public void stop() {
+    public synchronized void stop() {
         super.stop();
 
         if (zkClient != null) {
@@ -230,11 +241,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
 
     private void processActiveEnter() {
         if (listener != null) {
-            try {
+            listener.processActiveEnter();
-                listener.processActiveEnter();
-            } catch (Exception e) {
-                logger.error("processActiveEnter failed", e);
-            }
         }
     }
 

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java

@@ -11,7 +11,8 @@ import java.util.Map;
 public class ServerRunningMonitors {
 
     private static ServerRunningData serverData;
-    private static Map               runningMonitors; // <String, ServerRunningMonitor>
+    private static Map               runningMonitors; // <String,
+                                                      // ServerRunningMonitor>
 
     public static ServerRunningData getServerData() {
         return serverData;

common/src/main/java/com/google/common/collect/MigrateMap.java

@@ -0,0 +1,19 @@
+package com.google.common.collect;
+
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.base.Function;
+
+public class MigrateMap {
+
+    @SuppressWarnings("deprecation")
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker,
+                                                              Function<? super K, ? extends V> computingFunction) {
+        return maker.makeComputingMap(computingFunction);
+    }
+
+    @SuppressWarnings("deprecation")
+    public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
+        return new MapMaker().makeComputingMap(computingFunction);
+    }
+}

dbsync/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>com.alibaba.otter</groupId>
 		<artifactId>canal</artifactId>
-		<version>1.0.19</version>
+		<version>1.1.1-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>com.alibaba.otter</groupId>
@@ -11,6 +11,11 @@
 	<packaging>jar</packaging>
 	<name>canal dbsync module for otter ${project.version}</name>
 	<dependencies>
+		<dependency>
+			<groupId>com.alibaba.otter</groupId>
+			<artifactId>canal.parse.driver</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 		<!-- log -->
 		<dependency>
 			<groupId>ch.qos.logback</groupId>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java

@@ -6,24 +6,24 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * An utility class implements MySQL/Java charsets conversion.  you can see com.mysql.jdbc.CharsetMapping.
+ * An utility class implements MySQL/Java charsets conversion. you can see
+ * com.mysql.jdbc.CharsetMapping.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  */
-public final class CharsetConversion
+public final class CharsetConversion {
-{
+
     static final Log logger = LogFactory.getLog(CharsetConversion.class);
 
-    static final class Entry
+    static final class Entry {
-    {
+
         protected final int    charsetId;
         protected final String mysqlCharset;
         protected final String mysqlCollation;
         protected final String javaCharset;
 
         Entry(final int id, String mysqlCharset, // NL
-                String mysqlCollation, String javaCharset)
+              String mysqlCollation, String javaCharset){
-        {
             this.charsetId = id;
             this.mysqlCharset = mysqlCharset;
             this.mysqlCollation = mysqlCollation;
@@ -34,38 +34,31 @@ public final class CharsetConversion
     // Character set data used in lookups. The array will be sparse.
     static final Entry[] entries = new Entry[0xff];
 
-    static Entry getEntry(final int id)
+    static Entry getEntry(final int id) {
-    {
+        if (id >= 0 && id < entries.length) {
-        if (id >= 0 && id < entries.length)
-        {
             return entries[id];
-        }
+        } else {
-        else
-        {
             throw new IllegalArgumentException("Invalid charset id: " + id);
         }
     }
 
     // Loads character set information.
-    static void putEntry(final int charsetId, String mysqlCharset,
+    static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation, String javaCharset) {
-            String mysqlCollation, String javaCharset)
-    {
         entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
-                mysqlCollation, javaCharset);
+            mysqlCollation,
+            javaCharset);
     }
 
     // Loads character set information.
     @Deprecated
-    static void putEntry(final int charsetId, String mysqlCharset,
+    static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation) {
-            String mysqlCollation)
-    {
         entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
-                mysqlCollation, /* Unknown java charset */null);
+            mysqlCollation, /* Unknown java charset */
+            null);
     }
 
     // Load character set data statically.
-    static
+    static {
-    {
         putEntry(1, "big5", "big5_chinese_ci", "Big5");
         putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
         putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
@@ -110,8 +103,8 @@ public final class CharsetConversion
         putEntry(42, "latin7", "latin7_general_cs", "ISO8859_7");
         putEntry(43, "macce", "macce_bin", "MacCentralEurope");
         putEntry(44, "cp1250", "cp1250_croatian_ci", "Cp1250");
-        putEntry(45, "utf8mb4", "utf8mb4_general_ci", "MacCentralEurope");
+        putEntry(45, "utf8mb4", "utf8mb4_general_ci", "UTF-8");
-        putEntry(46, "utf8mb4", "utf8mb4_bin", "MacCentralEurope");
+        putEntry(46, "utf8mb4", "utf8mb4_bin", "UTF-8");
         putEntry(47, "latin1", "latin1_bin", "ISO8859_1");
         putEntry(48, "latin1", "latin1_general_ci", "ISO8859_1");
         putEntry(49, "latin1", "latin1_general_cs", "ISO8859_1");
@@ -135,7 +128,7 @@ public final class CharsetConversion
         putEntry(69, "dec8", "dec8_bin", "US-ASCII");
         putEntry(70, "greek", "greek_bin", "ISO8859_7");
         putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
-        putEntry(72, "hp8", "hp8_bin" , "US-ASCII");
+        putEntry(72, "hp8", "hp8_bin", "US-ASCII");
         putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
         putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
         putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
@@ -274,16 +267,12 @@ public final class CharsetConversion
     /**
      * Return defined charset name for mysql.
      */
-    public static String getCharset(final int id)
+    public static String getCharset(final int id) {
-    {
         Entry entry = getEntry(id);
 
-        if (entry != null)
+        if (entry != null) {
-        {
             return entry.mysqlCharset;
-        }
+        } else {
-        else
-        {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
@@ -292,16 +281,12 @@ public final class CharsetConversion
     /**
      * Return defined collaction name for mysql.
      */
-    public static String getCollation(final int id)
+    public static String getCollation(final int id) {
-    {
         Entry entry = getEntry(id);
 
-        if (entry != null)
+        if (entry != null) {
-        {
             return entry.mysqlCollation;
-        }
+        } else {
-        else
-        {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
@@ -310,55 +295,41 @@ public final class CharsetConversion
     /**
      * Return converted charset name for java.
      */
-    public static String getJavaCharset(final int id)
+    public static String getJavaCharset(final int id) {
-    {
         Entry entry = getEntry(id);
 
-        if (entry != null)
+        if (entry != null) {
-        {
+            if (entry.javaCharset != null) {
-            if (entry.javaCharset != null)
-            {
                 return entry.javaCharset;
-            }
+            } else {
-            else
+                logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
-            {
+                            + entry.mysqlCollation);
-                logger.warn("Unknown java charset for: id = " + id
-                        + ", name = " + entry.mysqlCharset + ", coll = "
-                        + entry.mysqlCollation);
                 return null;
             }
-        }
+        } else {
-        else
-        {
             logger.warn("Unexpect mysql charset: " + id);
             return null;
         }
     }
 
-    public static void main(String[] args)
+    public static void main(String[] args) {
-    {
+        for (int i = 0; i < entries.length; i++) {
-        for (int i = 0; i < entries.length; i++)
-        {
             Entry entry = entries[i];
 
             System.out.print(i);
             System.out.print(',');
             System.out.print(' ');
-            if (entry != null)
+            if (entry != null) {
-            {
                 System.out.print(entry.mysqlCharset);
                 System.out.print(',');
                 System.out.print(' ');
                 System.out.print(entry.javaCharset);
-                if (entry.javaCharset != null)
+                if (entry.javaCharset != null) {
-                {
                     System.out.print(',');
                     System.out.print(' ');
                     System.out.print(Charset.forName(entry.javaCharset).name());
                 }
-            }
+            } else {
-            else
-            {
                 System.out.print("null");
             }
             System.out.println();

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java

@@ -21,16 +21,13 @@ import org.apache.commons.logging.LogFactory;
  * DirectLogFetcher fetcher = new DirectLogFetcher();
  * fetcher.open(conn, file, 0, 13);
  * 
- * while (fetcher.fetch())
+ * while (fetcher.fetch()) {
- * {
  *     LogEvent event;
- *     do
+ *     do {
- *     {
  *         event = decoder.decode(fetcher, context);
  * 
  *         // process log event.
- *     }
+ *     } while (event != null);
- *     while (event != null);
  * }
  * // connection closed.
  * </pre>
@@ -38,97 +35,79 @@ import org.apache.commons.logging.LogFactory;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DirectLogFetcher extends LogFetcher
+public final class DirectLogFetcher extends LogFetcher {
-{
+
-    protected static final Log logger                = LogFactory.getLog(DirectLogFetcher.class);
+    protected static final Log logger                          = LogFactory.getLog(DirectLogFetcher.class);
 
     /** Command to dump binlog */
-    public static final byte   COM_BINLOG_DUMP       = 18;
+    public static final byte   COM_BINLOG_DUMP                 = 18;
 
     /** Packet header sizes */
-    public static final int    NET_HEADER_SIZE       = 4;
+    public static final int    NET_HEADER_SIZE                 = 4;
-    public static final int    SQLSTATE_LENGTH       = 5;
+    public static final int    SQLSTATE_LENGTH                 = 5;
 
     /** Packet offsets */
-    public static final int    PACKET_LEN_OFFSET     = 0;
+    public static final int    PACKET_LEN_OFFSET               = 0;
-    public static final int    PACKET_SEQ_OFFSET     = 3;
+    public static final int    PACKET_SEQ_OFFSET               = 3;
 
     /** Maximum packet length */
-    public static final int    MAX_PACKET_LENGTH     = (256 * 256 * 256 - 1);
+    public static final int    MAX_PACKET_LENGTH               = (256 * 256 * 256 - 1);
 
     /** BINLOG_DUMP options */
-    public static final int    BINLOG_DUMP_NON_BLOCK = 1;
+    public static final int    BINLOG_DUMP_NON_BLOCK           = 1;
     public static final int    BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
 
     private Connection         conn;
     private OutputStream       mysqlOutput;
     private InputStream        mysqlInput;
 
-    public DirectLogFetcher()
+    public DirectLogFetcher(){
-    {
         super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
     }
 
-    public DirectLogFetcher(final int initialCapacity)
+    public DirectLogFetcher(final int initialCapacity){
-    {
         super(initialCapacity, DEFAULT_GROWTH_FACTOR);
     }
 
-    public DirectLogFetcher(final int initialCapacity, final float growthFactor)
+    public DirectLogFetcher(final int initialCapacity, final float growthFactor){
-    {
         super(initialCapacity, growthFactor);
     }
 
-    private static final Object unwrapConnection(Object conn, Class<?> connClazz)
+    private static final Object unwrapConnection(Object conn, Class<?> connClazz) throws IOException {
-            throws IOException
+        while (!connClazz.isInstance(conn)) {
-    {
+            try {
-        while (!connClazz.isInstance(conn))
-        {
-            try
-            {
                 Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
-                if (connProxy.isInstance(conn))
+                if (connProxy.isInstance(conn)) {
-                {
                     conn = invokeMethod(conn, connProxy, "getTargetConnection");
                     continue;
                 }
-            }
+            } catch (ClassNotFoundException e) {
-            catch (ClassNotFoundException e)
+                // org.springframework.jdbc.datasource.ConnectionProxy not
-            {
+                // found.
-                // org.springframework.jdbc.datasource.ConnectionProxy not found.
             }
 
-            try
+            try {
-            {
                 Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
-                if (connProxy.isInstance(conn))
+                if (connProxy.isInstance(conn)) {
-                {
                     conn = getDeclaredField(conn, connProxy, "_conn");
                     continue;
                 }
-            }
+            } catch (ClassNotFoundException e) {
-            catch (ClassNotFoundException e)
-            {
                 // org.apache.commons.dbcp.DelegatingConnection not found.
             }
 
-            try
+            try {
-            {
+                if (conn instanceof java.sql.Wrapper) {
-                if (conn instanceof java.sql.Wrapper)
-                {
                     Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
                     conn = ((java.sql.Wrapper) conn).unwrap(connIface);
                     continue;
                 }
-            }
+            } catch (ClassNotFoundException e) {
-            catch (ClassNotFoundException e)
-            {
                 // com.mysql.jdbc.Connection not found.
-            }
+            } catch (SQLException e) {
-            catch (SQLException e)
+                logger.warn("Unwrap " + conn.getClass().getName() + " to " + connClazz.getName() + " failed: "
-            {
+                            + e.getMessage(),
-                logger.warn("Unwrap " + conn.getClass().getName() + " to "
+                    e);
-                        + connClazz.getName() + " failed: " + e.getMessage(), e);
             }
 
             return null;
@@ -136,127 +115,85 @@ public final class DirectLogFetcher extends LogFetcher
         return conn;
     }
 
-    private static final Object invokeMethod(Object obj, Class<?> objClazz,
+    private static final Object invokeMethod(Object obj, Class<?> objClazz, String name) {
-            String name)
+        try {
-    {
-        try
-        {
             Method method = objClazz.getMethod(name, (Class<?>[]) null);
             return method.invoke(obj, (Object[]) null);
-        }
+        } catch (NoSuchMethodException e) {
-        catch (NoSuchMethodException e)
+            throw new IllegalArgumentException("No such method: \'" + name + "\' @ " + objClazz.getName(), e);
-        {
+        } catch (IllegalAccessException e) {
-            throw new IllegalArgumentException("No such method: \'" + name
+            throw new IllegalArgumentException("Cannot invoke method: \'" + name + "\' @ " + objClazz.getName(), e);
-                    + "\' @ " + objClazz.getName(), e);
+        } catch (InvocationTargetException e) {
-        }
+            throw new IllegalArgumentException("Invoke method failed: \'" + name + "\' @ " + objClazz.getName(),
-        catch (IllegalAccessException e)
+                e.getTargetException());
-        {
-            throw new IllegalArgumentException("Cannot invoke method: \'"
-                    + name + "\' @ " + objClazz.getName(), e);
-        }
-        catch (InvocationTargetException e)
-        {
-            throw new IllegalArgumentException("Invoke method failed: \'"
-                    + name + "\' @ " + objClazz.getName(),
-                    e.getTargetException());
         }
     }
 
-    private static final Object getDeclaredField(Object obj, Class<?> objClazz,
+    private static final Object getDeclaredField(Object obj, Class<?> objClazz, String name) {
-            String name)
+        try {
-    {
-        try
-        {
             Field field = objClazz.getDeclaredField(name);
             field.setAccessible(true);
             return field.get(obj);
-        }
+        } catch (NoSuchFieldException e) {
-        catch (NoSuchFieldException e)
+            throw new IllegalArgumentException("No such field: \'" + name + "\' @ " + objClazz.getName(), e);
-        {
+        } catch (IllegalAccessException e) {
-            throw new IllegalArgumentException("No such field: \'" + name
+            throw new IllegalArgumentException("Cannot get field: \'" + name + "\' @ " + objClazz.getName(), e);
-                    + "\' @ " + objClazz.getName(), e);
-        }
-        catch (IllegalAccessException e)
-        {
-            throw new IllegalArgumentException("Cannot get field: \'" + name
-                    + "\' @ " + objClazz.getName(), e);
         }
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final int serverId)
+    public void open(Connection conn, String fileName, final int serverId) throws IOException {
-            throws IOException
-    {
         open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final int serverId,
+    public void open(Connection conn, String fileName, final int serverId, boolean nonBlocking) throws IOException {
-            boolean nonBlocking) throws IOException
-    {
         open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, final long filePosition,
+    public void open(Connection conn, String fileName, final long filePosition, final int serverId) throws IOException {
-            final int serverId) throws IOException
-    {
         open(conn, fileName, filePosition, serverId, false);
     }
 
     /**
      * Connect MySQL master to fetch binlog.
      */
-    public void open(Connection conn, String fileName, long filePosition,
+    public void open(Connection conn, String fileName, long filePosition, final int serverId, boolean nonBlocking)
-            final int serverId, boolean nonBlocking) throws IOException
+                                                                                                                  throws IOException {
-    {
+        try {
-        try
-        {
             this.conn = conn;
             Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
             Object unwrapConn = unwrapConnection(conn, connClazz);
-            if (unwrapConn == null)
+            if (unwrapConn == null) {
-            {
+                throw new IOException("Unable to unwrap " + conn.getClass().getName()
-                throw new IOException("Unable to unwrap "
+                                      + " to com.mysql.jdbc.ConnectionImpl");
-                        + conn.getClass().getName()
-                        + " to com.mysql.jdbc.ConnectionImpl");
             }
 
             // Get underlying IO streams for network communications.
             Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
-            if (connIo == null)
+            if (connIo == null) {
-            {
+                throw new IOException("Get null field:" + conn.getClass().getName() + "#io");
-                throw new IOException("Get null field:"
-                        + conn.getClass().getName() + "#io");
             }
-            mysqlOutput = (OutputStream) getDeclaredField(connIo,
+            mysqlOutput = (OutputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlOutput");
-                    connIo.getClass(), "mysqlOutput");
+            mysqlInput = (InputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlInput");
-            mysqlInput = (InputStream) getDeclaredField(connIo,
-                    connIo.getClass(), "mysqlInput");
 
-            if (filePosition == 0)
+            if (filePosition == 0) filePosition = BIN_LOG_HEADER_SIZE;
-                filePosition = BIN_LOG_HEADER_SIZE;
             sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
             position = 0;
-        }
+        } catch (IOException e) {
-        catch (IOException e)
-        {
             close(); /* Do cleanup */
-            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
+            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName + ", position = " + filePosition);
-                    + ", position = " + filePosition);
             throw e;
-        }
+        } catch (ClassNotFoundException e) {
-        catch (ClassNotFoundException e)
-        {
             close(); /* Do cleanup */
-            throw new IOException(
+            throw new IOException("Unable to load com.mysql.jdbc.ConnectionImpl", e);
-                    "Unable to load com.mysql.jdbc.ConnectionImpl", e);
         }
     }
 
@@ -265,8 +202,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param b the byte to put in the buffer
      */
-    protected final void putByte(byte b)
+    protected final void putByte(byte b) {
-    {
         ensureCapacity(position + 1);
 
         buffer[position++] = b;
@@ -277,8 +213,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param i16 the integer to put in the buffer
      */
-    protected final void putInt16(int i16)
+    protected final void putInt16(int i16) {
-    {
         ensureCapacity(position + 2);
 
         byte[] buf = buffer;
@@ -291,8 +226,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param i32 the integer to put in the buffer
      */
-    protected final void putInt32(long i32)
+    protected final void putInt32(long i32) {
-    {
         ensureCapacity(position + 4);
 
         byte[] buf = buffer;
@@ -307,8 +241,7 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @param s the value to put in the buffer
      */
-    protected final void putString(String s)
+    protected final void putString(String s) {
-    {
         ensureCapacity(position + (s.length() * 2) + 1);
 
         System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
@@ -316,10 +249,8 @@ public final class DirectLogFetcher extends LogFetcher
         buffer[position++] = 0;
     }
 
-    protected final void sendBinlogDump(String fileName,
+    protected final void sendBinlogDump(String fileName, final long filePosition, final int serverId,
-            final long filePosition, final int serverId, boolean nonBlocking)
+                                        boolean nonBlocking) throws IOException {
-            throws IOException
-    {
         position = NET_HEADER_SIZE;
 
         putByte(COM_BINLOG_DUMP);
@@ -327,7 +258,7 @@ public final class DirectLogFetcher extends LogFetcher
         int binlog_flags = nonBlocking ? BINLOG_DUMP_NON_BLOCK : 0;
         binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
         putInt16(binlog_flags); // binlog_flags
-        putInt32(serverId); // slave's server-id 
+        putInt32(serverId); // slave's server-id
         putString(fileName);
 
         final byte[] buf = buffer;
@@ -345,13 +276,10 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
      */
-    public boolean fetch() throws IOException
+    public boolean fetch() throws IOException {
-    {
+        try {
-        try
-        {
             // Fetching packet header from input.
-            if (!fetch0(0, NET_HEADER_SIZE))
+            if (!fetch0(0, NET_HEADER_SIZE)) {
-            {
                 logger.warn("Reached end of input stream while fetching header");
                 return false;
             }
@@ -359,61 +287,48 @@ public final class DirectLogFetcher extends LogFetcher
             // Fetching the first packet(may a multi-packet).
             int netlen = getUint24(PACKET_LEN_OFFSET);
             int netnum = getUint8(PACKET_SEQ_OFFSET);
-            if (!fetch0(NET_HEADER_SIZE, netlen))
+            if (!fetch0(NET_HEADER_SIZE, netlen)) {
-            {
+                logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
-                logger.warn("Reached end of input stream: packet #" + netnum
-                        + ", len = " + netlen);
                 return false;
             }
 
             // Detecting error code.
             final int mark = getUint8(NET_HEADER_SIZE);
-            if (mark != 0)
+            if (mark != 0) {
-            {
                 if (mark == 255) // error from master
                 {
-                    // Indicates an error, for example trying to fetch from wrong
+                    // Indicates an error, for example trying to fetch from
+                    // wrong
                     // binlog position.
                     position = NET_HEADER_SIZE + 1;
                     final int errno = getInt16();
                     String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
                     String errmsg = getFixString(limit - position);
-                    throw new IOException("Received error packet:"
+                    throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
-                            + " errno = " + errno + ", sqlstate = " + sqlstate
+                                          + " errmsg = " + errmsg);
-                            + " errmsg = " + errmsg);
+                } else if (mark == 254) {
-                }
-                else if (mark == 254)
-                {
                     // Indicates end of stream. It's not clear when this would
                     // be sent.
-                    logger.warn("Received EOF packet from server, apparent"
+                    logger.warn("Received EOF packet from server, apparent" + " master disconnected.");
-                            + " master disconnected.");
                     return false;
-                }
+                } else {
-                else
-                {
                     // Should not happen.
-                    throw new IOException("Unexpected response " + mark
+                    throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
-                            + " while fetching binlog: packet #" + netnum
+                                          + ", len = " + netlen);
-                            + ", len = " + netlen);
                 }
             }
 
             // The first packet is a multi-packet, concatenate the packets.
-            while (netlen == MAX_PACKET_LENGTH)
+            while (netlen == MAX_PACKET_LENGTH) {
-            {
+                if (!fetch0(0, NET_HEADER_SIZE)) {
-                if (!fetch0(0, NET_HEADER_SIZE))
-                {
                     logger.warn("Reached end of input stream while fetching header");
                     return false;
                 }
 
                 netlen = getUint24(PACKET_LEN_OFFSET);
                 netnum = getUint8(PACKET_SEQ_OFFSET);
-                if (!fetch0(limit, netlen))
+                if (!fetch0(limit, netlen)) {
-                {
+                    logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
-                    logger.warn("Reached end of input stream: packet #"
-                            + netnum + ", len = " + netlen);
                     return false;
                 }
             }
@@ -423,43 +338,32 @@ public final class DirectLogFetcher extends LogFetcher
             position = origin;
             limit -= origin;
             return true;
-        }
+        } catch (SocketTimeoutException e) {
-        catch (SocketTimeoutException e)
-        {
             close(); /* Do cleanup */
             logger.error("Socket timeout expired, closing connection", e);
             throw e;
-        }
+        } catch (InterruptedIOException e) {
-        catch (InterruptedIOException e)
-        {
             close(); /* Do cleanup */
             logger.warn("I/O interrupted while reading from client socket", e);
             throw e;
-        }
+        } catch (IOException e) {
-        catch (IOException e)
-        {
             close(); /* Do cleanup */
             logger.error("I/O error while reading from client socket", e);
             throw e;
         }
     }
 
-    private final boolean fetch0(final int off, final int len)
+    private final boolean fetch0(final int off, final int len) throws IOException {
-            throws IOException
-    {
         ensureCapacity(off + len);
 
-        for (int count, n = 0; n < len; n += count)
+        for (int count, n = 0; n < len; n += count) {
-        {
+            if (0 > (count = mysqlInput.read(buffer, off + n, len - n))) {
-            if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
-            {
                 // Reached end of input stream
                 return false;
             }
         }
 
-        if (limit < off + len)
+        if (limit < off + len) limit = off + len;
-            limit = off + len;
         return true;
     }
 
@@ -468,19 +372,14 @@ public final class DirectLogFetcher extends LogFetcher
      * 
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
      */
-    public void close() throws IOException
+    public void close() throws IOException {
-    {
+        try {
-        try
+            if (conn != null) conn.close();
-        {
-            if (conn != null)
-                conn.close();
 
             conn = null;
             mysqlInput = null;
             mysqlOutput = null;
-        }
+        } catch (SQLException e) {
-        catch (SQLException e)
-        {
             logger.warn("Unable to close connection", e);
         }
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java

@@ -75,7 +75,9 @@ public final class FileLogFetcher extends LogFetcher {
         fin = new FileInputStream(file);
 
         ensureCapacity(BIN_LOG_HEADER_SIZE);
-        if (BIN_LOG_HEADER_SIZE != fin.read(buffer, 0, BIN_LOG_HEADER_SIZE)) throw new IOException("No binlog file header");
+        if (BIN_LOG_HEADER_SIZE != fin.read(buffer, 0, BIN_LOG_HEADER_SIZE)) {
+            throw new IOException("No binlog file header");
+        }
 
         if (buffer[0] != BINLOG_MAGIC[0] || buffer[1] != BINLOG_MAGIC[1] || buffer[2] != BINLOG_MAGIC[2]
             || buffer[3] != BINLOG_MAGIC[3]) {
@@ -128,12 +130,25 @@ public final class FileLogFetcher extends LogFetcher {
                 return true;
             }
         } else if (limit > 0) {
+            if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
+                int lenPosition = position + 4 + 1 + 4;
+                long eventLen = ((long) (0xff & buffer[lenPosition++])) | ((long) (0xff & buffer[lenPosition++]) << 8)
+                                | ((long) (0xff & buffer[lenPosition++]) << 16)
+                                | ((long) (0xff & buffer[lenPosition++]) << 24);
+
+                if (limit >= eventLen) {
+                    return true;
+                } else {
+                    ensureCapacity((int) eventLen);
+                }
+            }
+
             System.arraycopy(buffer, origin, buffer, 0, limit);
+            position -= origin;
+            origin = 0;
             final int len = fin.read(buffer, limit, buffer.length - limit);
             if (len >= 0) {
                 limit += len;
-                position -= origin;
-                origin = 0;
 
                 /* More binlog to fetch */
                 return true;
@@ -153,7 +168,9 @@ public final class FileLogFetcher extends LogFetcher {
      * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
      */
     public void close() throws IOException {
-        if (fin != null) fin.close();
+        if (fin != null) {
+            fin.close();
+        }
 
         fin = null;
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java

@@ -0,0 +1,418 @@
+package com.taobao.tddl.dbsync.binlog;
+
+/**
+ * 处理下MySQL json二进制转化为可读的字符串
+ * 
+ * @author agapple 2016年6月30日 上午11:26:17
+ * @since 1.0.22
+ */
+public class JsonConversion {
+
+    // JSON TYPE
+    public static final int  JSONB_TYPE_SMALL_OBJECT = 0x0;
+    public static final int  JSONB_TYPE_LARGE_OBJECT = 0x1;
+    public static final int  JSONB_TYPE_SMALL_ARRAY  = 0x2;
+    public static final int  JSONB_TYPE_LARGE_ARRAY  = 0x3;
+    public static final int  JSONB_TYPE_LITERAL      = 0x4;
+    public static final int  JSONB_TYPE_INT16        = 0x5;
+    public static final int  JSONB_TYPE_UINT16       = 0x6;
+    public static final int  JSONB_TYPE_INT32        = 0x7;
+    public static final int  JSONB_TYPE_UINT32       = 0x8;
+    public static final int  JSONB_TYPE_INT64        = 0x9;
+    public static final int  JSONB_TYPE_UINT64       = 0xA;
+    public static final int  JSONB_TYPE_DOUBLE       = 0xB;
+    public static final int  JSONB_TYPE_STRING       = 0xC;
+    public static final int  JSONB_TYPE_OPAQUE       = 0xF;
+    public static final char JSONB_NULL_LITERAL      = '\0';
+    public static final char JSONB_TRUE_LITERAL      = '\1';
+    public static final char JSONB_FALSE_LITERAL     = '\2';
+
+    /*
+     * The size of offset or size fields in the small and the large storage
+     * format for JSON objects and JSON arrays.
+     */
+    public static final int  SMALL_OFFSET_SIZE       = 2;
+    public static final int  LARGE_OFFSET_SIZE       = 4;
+
+    /*
+     * The size of key entries for objects when using the small storage format
+     * or the large storage format. In the small format it is 4 bytes (2 bytes
+     * for key length and 2 bytes for key offset). In the large format it is 6
+     * (2 bytes for length, 4 bytes for offset).
+     */
+    public static final int  KEY_ENTRY_SIZE_SMALL    = (2 + SMALL_OFFSET_SIZE);
+    public static final int  KEY_ENTRY_SIZE_LARGE    = (2 + LARGE_OFFSET_SIZE);
+
+    /*
+     * The size of value entries for objects or arrays. When using the small
+     * storage format, the entry size is 3 (1 byte for type, 2 bytes for
+     * offset). When using the large storage format, it is 5 (1 byte for type, 4
+     * bytes for offset).
+     */
+    public static final int  VALUE_ENTRY_SIZE_SMALL  = (1 + SMALL_OFFSET_SIZE);
+    public static final int  VALUE_ENTRY_SIZE_LARGE  = (1 + LARGE_OFFSET_SIZE);
+
+    public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
+        buffer = buffer.duplicate(buffer.position(), (int) len);
+        switch (type) {
+            case JSONB_TYPE_SMALL_OBJECT:
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
+            case JSONB_TYPE_LARGE_OBJECT:
+                return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
+            case JSONB_TYPE_SMALL_ARRAY:
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
+            case JSONB_TYPE_LARGE_ARRAY:
+                return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
+            default:
+                return parse_scalar(type, buffer, len, charsetName);
+        }
+    }
+
+    private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
+                                                    String charsetName) {
+        long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+        if (len < 2 * offset_size) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        long element_count = read_offset_or_size(buffer, large);
+        long bytes = read_offset_or_size(buffer, large);
+
+        if (bytes > len) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        long header_size = 2 * offset_size;
+        if (type == Json_enum_type.OBJECT) {
+            header_size += element_count * (large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL);
+        }
+
+        header_size += element_count * (large ? VALUE_ENTRY_SIZE_LARGE : VALUE_ENTRY_SIZE_SMALL);
+        if (header_size > bytes) {
+            throw new IllegalArgumentException("illegal json data");
+        }
+        return new Json_Value(type, buffer.rewind(), element_count, bytes, large);
+    }
+
+    private static long read_offset_or_size(LogBuffer buffer, boolean large) {
+        return large ? buffer.getUint32() : buffer.getUint16();
+    }
+
+    private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
+        switch (type) {
+            case JSONB_TYPE_LITERAL:
+                /* purecov: inspected */
+                int data = buffer.getUint8();
+                switch (data) {
+                    case JSONB_NULL_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_NULL);
+                    case JSONB_TRUE_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_TRUE);
+                    case JSONB_FALSE_LITERAL:
+                        return new Json_Value(Json_enum_type.LITERAL_FALSE);
+                    default:
+                        throw new IllegalArgumentException("illegal json data");
+                }
+            case JSONB_TYPE_INT16:
+                return new Json_Value(Json_enum_type.INT, buffer.getInt16());
+            case JSONB_TYPE_INT32:
+                return new Json_Value(Json_enum_type.INT, buffer.getInt32());
+            case JSONB_TYPE_INT64:
+                return new Json_Value(Json_enum_type.INT, buffer.getLong64());
+            case JSONB_TYPE_UINT16:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUint16());
+            case JSONB_TYPE_UINT32:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUint32());
+            case JSONB_TYPE_UINT64:
+                return new Json_Value(Json_enum_type.UINT, buffer.getUlong64());
+            case JSONB_TYPE_DOUBLE:
+                return new Json_Value(Json_enum_type.DOUBLE, Double.valueOf(buffer.getDouble64()));
+            case JSONB_TYPE_STRING:
+                int max_bytes = (int) Math.min(len, 5);
+                long tlen = 0;
+                long str_len = 0;
+                long n = 0;
+                byte[] datas = buffer.getData(max_bytes);
+                for (int i = 0; i < max_bytes; i++) {
+                    // Get the next 7 bits of the length.
+                    tlen |= (datas[i] & 0x7f) << (7 * i);
+                    if ((datas[i] & 0x80) == 0) {
+                        // The length shouldn't exceed 32 bits.
+                        if (tlen > 4294967296L) {
+                            throw new IllegalArgumentException("illegal json data");
+                        }
+
+                        // This was the last byte. Return successfully.
+                        n = i + 1;
+                        str_len = tlen;
+                        break;
+                    }
+                }
+
+                if (len < n + str_len) {
+                    throw new IllegalArgumentException("illegal json data");
+                }
+                return new Json_Value(Json_enum_type.STRING, buffer.rewind()
+                    .forward((int) n)
+                    .getFixString((int) str_len, charsetName));
+            case JSONB_TYPE_OPAQUE:
+                /*
+                 * There should always be at least one byte, which tells the
+                 * field type of the opaque value.
+                 */
+                // The type is encoded as a uint8 that maps to an
+                // enum_field_types.
+                int type_byte = buffer.getUint8();
+                int position = buffer.position();
+                // Then there's the length of the value.
+                int q_max_bytes = (int) Math.min(len, 5);
+                long q_tlen = 0;
+                long q_str_len = 0;
+                long q_n = 0;
+                byte[] q_datas = buffer.getData(q_max_bytes);
+                for (int i = 0; i < q_max_bytes; i++) {
+                    // Get the next 7 bits of the length.
+                    q_tlen |= (q_datas[i] & 0x7f) << (7 * i);
+                    if ((q_datas[i] & 0x80) == 0) {
+                        // The length shouldn't exceed 32 bits.
+                        if (q_tlen > 4294967296L) {
+                            throw new IllegalArgumentException("illegal json data");
+                        }
+
+                        // This was the last byte. Return successfully.
+                        q_n = i + 1;
+                        q_str_len = q_tlen;
+                        break;
+                    }
+                }
+
+                if (q_str_len == 0 || len < q_n + q_str_len) {
+                    throw new IllegalArgumentException("illegal json data");
+                }
+                return new Json_Value(type_byte, buffer.position(position).forward((int) q_n), q_str_len);
+            default:
+                throw new IllegalArgumentException("illegal json data");
+        }
+    }
+
+    public static class Json_Value {
+
+        Json_enum_type m_type;
+        int            m_field_type;
+        LogBuffer      m_data;
+        long           m_element_count;
+        long           m_length;
+        String         m_string_value;
+        Number         m_int_value;
+        double         m_double_value;
+        boolean        m_large;
+
+        public Json_Value(Json_enum_type t){
+            this.m_type = t;
+        }
+
+        public Json_Value(Json_enum_type t, Number val){
+            this.m_type = t;
+            if (t == Json_enum_type.DOUBLE) {
+                this.m_double_value = val.doubleValue();
+            } else {
+                this.m_int_value = val;
+            }
+        }
+
+        public Json_Value(Json_enum_type t, String value){
+            this.m_type = t;
+            this.m_string_value = value;
+        }
+
+        public Json_Value(int field_type, LogBuffer data, long bytes){
+            this.m_type = Json_enum_type.OPAQUE; // 不确定类型
+            this.m_field_type = field_type;
+            this.m_data = data;
+            this.m_length = bytes;
+        }
+
+        public Json_Value(Json_enum_type t, LogBuffer data, long element_count, long bytes, boolean large){
+            this.m_type = t;
+            this.m_data = data;
+            this.m_element_count = element_count;
+            this.m_length = bytes;
+            this.m_large = large;
+        }
+
+        public String key(int i, String charsetName) {
+            m_data.rewind();
+            int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+            int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
+            int entry_offset = 2 * offset_size + key_entry_size * i;
+            // The offset of the key is the first part of the key
+            // entry.
+            m_data.forward(entry_offset);
+            long key_offset = read_offset_or_size(m_data, m_large);
+            // The length of the key is the second part of the
+            // entry, always two
+            // bytes.
+            long key_length = m_data.getUint16();
+            return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
+        }
+
+        public Json_Value element(int i, String charsetName) {
+            m_data.rewind();
+            int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
+            int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
+            int value_entry_size = m_large ? VALUE_ENTRY_SIZE_LARGE : VALUE_ENTRY_SIZE_SMALL;
+            int first_entry_offset = 2 * offset_size;
+            if (m_type == Json_enum_type.OBJECT) {
+                first_entry_offset += m_element_count * key_entry_size;
+            }
+            int entry_offset = first_entry_offset + value_entry_size * i;
+            int type = m_data.forward(entry_offset).getUint8();
+            if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
+                || (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
+                return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
+            }
+            int value_offset = (int) read_offset_or_size(m_data, m_large);
+            return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
+        }
+
+        public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
+            switch (m_type) {
+                case OBJECT:
+                    buf.append("{");
+                    for (int i = 0; i < m_element_count; ++i) {
+                        if (i > 0) {
+                            buf.append(", ");
+                        }
+                        buf.append('"').append(key(i, charsetName)).append('"');
+                        buf.append(": ");
+                        element(i, charsetName).toJsonString(buf, charsetName);
+                    }
+                    buf.append("}");
+                    break;
+                case ARRAY:
+                    buf.append("[");
+                    for (int i = 0; i < m_element_count; ++i) {
+                        if (i > 0) {
+                            buf.append(", ");
+                        }
+                        element(i, charsetName).toJsonString(buf, charsetName);
+                    }
+                    buf.append("]");
+                    break;
+                case DOUBLE:
+                    buf.append(Double.valueOf(m_double_value).toString());
+                    break;
+                case INT:
+                    buf.append(m_int_value.toString());
+                    break;
+                case UINT:
+                    buf.append(m_int_value.toString());
+                    break;
+                case LITERAL_FALSE:
+                    buf.append("false");
+                    break;
+                case LITERAL_TRUE:
+                    buf.append("true");
+                    break;
+                case LITERAL_NULL:
+                    buf.append("NULL");
+                    break;
+                case OPAQUE:
+                    String text = null;
+                    if (m_field_type == LogEvent.MYSQL_TYPE_NEWDECIMAL) {
+                        int precision = m_data.getInt8();
+                        int scale = m_data.getInt8();
+                        text = m_data.getDecimal(precision, scale).toPlainString();
+                        buf.append(text);
+                    } else if (m_field_type == LogEvent.MYSQL_TYPE_TIME) {
+                        long packed_value = m_data.getLong64();
+                        if (packed_value == 0) {
+                            text = "00:00:00";
+                        } else {
+                            long ultime = Math.abs(packed_value);
+                            long intpart = ultime >> 24;
+                            int frac = (int) (ultime % (1L << 24));
+                            text = String.format("%s%02d:%02d:%02d",
+                                packed_value >= 0 ? "" : "-",
+                                (int) ((intpart >> 12) % (1 << 10)),
+                                (int) ((intpart >> 6) % (1 << 6)),
+                                (int) (intpart % (1 << 6)));
+                            text = text + "." + usecondsToStr(frac, 6);
+                        }
+                        buf.append('"').append(text).append('"');
+                    } else if (m_field_type == LogEvent.MYSQL_TYPE_DATE || m_field_type == LogEvent.MYSQL_TYPE_DATETIME
+                               || m_field_type == LogEvent.MYSQL_TYPE_TIMESTAMP) {
+                        long packed_value = m_data.getLong64();
+                        if (packed_value == 0) {
+                            text = "0000-00-00 00:00:00";
+                        } else {
+                            // 构造TimeStamp只处理到秒
+                            long ultime = Math.abs(packed_value);
+                            long intpart = ultime >> 24;
+                            int frac = (int) (ultime % (1L << 24));
+                            long ymd = intpart >> 17;
+                            long ym = ymd >> 5;
+                            long hms = intpart % (1 << 17);
+                            text = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                                (int) (ym / 13),
+                                (int) (ym % 13),
+                                (int) (ymd % (1 << 5)),
+                                (int) (hms >> 12),
+                                (int) ((hms >> 6) % (1 << 6)),
+                                (int) (hms % (1 << 6)));
+                            text = text + "." + usecondsToStr(frac, 6);
+                        }
+                        buf.append('"').append(text).append('"');
+                    } else {
+                        text = m_data.getFixString((int) m_length, charsetName);
+                        buf.append('"').append(escapse(text)).append('"');
+                    }
+
+                    break;
+                case STRING:
+                    buf.append('"').append(escapse(m_string_value)).append('"');
+                    break;
+                case ERROR:
+                    throw new IllegalArgumentException("illegal json data");
+            }
+
+            return buf;
+        }
+    }
+
+    private static StringBuilder escapse(String data) {
+        StringBuilder sb = new StringBuilder(data.length());
+        int endIndex = data.length();
+        for (int i = 0; i < endIndex; ++i) {
+            char c = data.charAt(i);
+            if (c == '"') {
+                sb.append('\\');
+            } else if (c == '\\') {
+                sb.append("\\");
+            }
+            sb.append(c);
+        }
+        return sb;
+    }
+
+    public static enum Json_enum_type {
+        OBJECT, ARRAY, STRING, INT, UINT, DOUBLE, LITERAL_NULL, LITERAL_TRUE, LITERAL_FALSE, OPAQUE, ERROR
+    }
+
+    private static String usecondsToStr(int frac, int meta) {
+        String sec = String.valueOf(frac);
+        if (meta > 6) {
+            throw new IllegalArgumentException("unknow useconds meta : " + meta);
+        }
+
+        if (sec.length() < 6) {
+            StringBuilder result = new StringBuilder(6);
+            int len = 6 - sec.length();
+            for (; len > 0; len--) {
+                result.append('0');
+            }
+            result.append(sec);
+            sec = result.toString();
+        }
+
+        return sec.substring(0, meta);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java

@@ -3,75 +3,72 @@ package com.taobao.tddl.dbsync.binlog;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
 
 /**
- * TODO: Document Me!!
+ * TODO: Document Me!! NOTE: Log context will NOT write multi-threaded.
- * 
- * NOTE: Log context will NOT write multi-threaded.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogContext
+public final class LogContext {
-{
+
     private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
 
     private FormatDescriptionLogEvent         formatDescription;
 
     private LogPosition                       logPosition;
 
-    public LogContext()
+    private GTIDSet                           gtidSet;
-    {
+
+    public LogContext(){
         this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
     }
 
-    public LogContext(FormatDescriptionLogEvent descriptionEvent)
+    public LogContext(FormatDescriptionLogEvent descriptionEvent){
-    {
         this.formatDescription = descriptionEvent;
     }
 
-    public final LogPosition getLogPosition()
+    public final LogPosition getLogPosition() {
-    {
         return logPosition;
     }
 
-    public final void setLogPosition(LogPosition logPosition)
+    public final void setLogPosition(LogPosition logPosition) {
-    {
         this.logPosition = logPosition;
     }
 
-    public final FormatDescriptionLogEvent getFormatDescription()
+    public final FormatDescriptionLogEvent getFormatDescription() {
-    {
         return formatDescription;
     }
 
-    public final void setFormatDescription(
+    public final void setFormatDescription(FormatDescriptionLogEvent formatDescription) {
-            FormatDescriptionLogEvent formatDescription)
-    {
         this.formatDescription = formatDescription;
     }
 
-    public final void putTable(TableMapLogEvent mapEvent)
+    public final void putTable(TableMapLogEvent mapEvent) {
-    {
         mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
     }
 
-    public final TableMapLogEvent getTable(final long tableId)
+    public final TableMapLogEvent getTable(final long tableId) {
-    {
         return mapOfTable.get(Long.valueOf(tableId));
     }
 
-    public final void clearAllTables()
+    public final void clearAllTables() {
-    {
         mapOfTable.clear();
     }
 
-    public void reset()
+    public void reset() {
-    {
         formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
-
         mapOfTable.clear();
     }
+
+    public GTIDSet getGtidSet() {
+        return gtidSet;
+    }
+
+    public void setGtidSet(GTIDSet gtidSet) {
+        this.gtidSet = gtidSet;
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -6,6 +6,7 @@ import java.util.BitSet;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.taobao.tddl.dbsync.binlog.event.AppendBlockLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.BeginLoadQueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.CreateFileLogEvent;
@@ -30,19 +31,23 @@ import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.StartLogEventV3;
 import com.taobao.tddl.dbsync.binlog.event.StopLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TransactionContextLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.ViewChangeEvent;
 import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XaPrepareLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.StartEncryptionLogEvent;
 
 /**
  * Implements a binary-log decoder.
- * 
+ *
  * <pre>
  * LogDecoder decoder = new LogDecoder();
  * decoder.handle(...);
@@ -57,81 +62,68 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
  * while (event != null);
  * // no more events in buffer.
  * </pre>
- * 
+ *
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class LogDecoder
+public final class LogDecoder {
-{
+
     protected static final Log logger    = LogFactory.getLog(LogDecoder.class);
 
     protected final BitSet     handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
 
-    public LogDecoder()
+    public LogDecoder(){
-    {
     }
 
-    public LogDecoder(final int fromIndex, final int toIndex)
+    public LogDecoder(final int fromIndex, final int toIndex){
-    {
         handleSet.set(fromIndex, toIndex);
     }
 
-    public final void handle(final int fromIndex, final int toIndex)
+    public final void handle(final int fromIndex, final int toIndex) {
-    {
         handleSet.set(fromIndex, toIndex);
     }
 
-    public final void handle(final int flagIndex)
+    public final void handle(final int flagIndex) {
-    {
         handleSet.set(flagIndex);
     }
 
     /**
      * Decoding an event from binary-log buffer.
-     * 
+     *
      * @return <code>UknownLogEvent</code> if event type is unknown or skipped,
-     *         <code>null</code> if buffer is not including a full event.
+     * <code>null</code> if buffer is not including a full event.
      */
-    public LogEvent decode(LogBuffer buffer, LogContext context)
+    public LogEvent decode(LogBuffer buffer, LogContext context) throws IOException {
-            throws IOException
-    {
         final int limit = buffer.limit();
 
-        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
+        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
-        {
+            LogHeader header = new LogHeader(buffer, context.getFormatDescription());
-            LogHeader header = new LogHeader(buffer,
-                    context.getFormatDescription());
 
             final int len = header.getEventLen();
-            if (limit >= len)
+            if (limit >= len) {
-            {
                 LogEvent event;
 
                 /* Checking binary-log's header */
-                if (handleSet.get(header.getType()))
+                if (handleSet.get(header.getType())) {
-                {
                     buffer.limit(len);
-                    try
+                    try {
-                    {
                         /* Decoding binary-log to event */
                         event = decode(buffer, header, context);
-                    }
+                        if (event != null) {
-                    catch (IOException e)
+                            // set logFileName
-                    {
+                            event.getHeader().setLogFileName(context.getLogPosition().getFileName());
-                        if (logger.isWarnEnabled())
+                            event.setSemival(buffer.semival);
-                            logger.warn("Decoding "
+                        }
-                                    + LogEvent.getTypeName(header.getType())
+                    } catch (IOException e) {
-                                    + " failed from: "
+                        if (logger.isWarnEnabled()) {
-                                    + context.getLogPosition(), e);
+                            logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: "
+                                        + context.getLogPosition(), e);
+                        }
                         throw e;
-                    }
+                    } finally {
-                    finally
-                    {
                         buffer.limit(limit); /* Restore limit */
                     }
-                }
+                } else {
-                else
-                {
                     /* Ignore unsupported binary-log. */
                     event = new UnknownLogEvent(header);
                 }
@@ -149,341 +141,318 @@ public final class LogDecoder
 
     /**
      * Deserialize an event from buffer.
-     * 
+     *
      * @return <code>UknownLogEvent</code> if event type is unknown or skipped.
      */
-    public static LogEvent decode(LogBuffer buffer, LogHeader header,
+    public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext context) throws IOException {
-            LogContext context) throws IOException
-    {
         FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
         LogPosition logPosition = context.getLogPosition();
-        
+
         int checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
         if (header.getType() != LogEvent.FORMAT_DESCRIPTION_EVENT) {
             checksumAlg = descriptionEvent.header.getChecksumAlg();
-        }else {
+        } else {
             // 如果是format事件自己，也需要处理checksum
             checksumAlg = header.getChecksumAlg();
         }
-        
+
         if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
             // remove checksum bytes
             buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
         }
-        
+        GTIDSet gtidSet = context.getGtidSet();
-        switch (header.getType())
+        switch (header.getType()) {
-        {
+            case LogEvent.QUERY_EVENT: {
-        case LogEvent.QUERY_EVENT:
+                QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
-            {
-                QueryLogEvent event = new QueryLogEvent(header, buffer,
-                        descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
                 return event;
             }
-        case LogEvent.XID_EVENT:
+            case LogEvent.XID_EVENT: {
-            {
+                XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
-                XidLogEvent event = new XidLogEvent(header, buffer,
-                        descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
                 return event;
             }
-        case LogEvent.TABLE_MAP_EVENT:
+            case LogEvent.TABLE_MAP_EVENT: {
-            {
+                TableMapLogEvent mapEvent = new TableMapLogEvent(header, buffer, descriptionEvent);
-                TableMapLogEvent mapEvent = new TableMapLogEvent(header,
-                        buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 context.putTable(mapEvent);
                 return mapEvent;
             }
-        case LogEvent.WRITE_ROWS_EVENT_V1:
+            case LogEvent.WRITE_ROWS_EVENT_V1: {
-            {
-                RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                event.fillTable(context);
-                return event;
-            }
-        case LogEvent.UPDATE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                event.fillTable(context);
-                return event;
-            }
-        case LogEvent.DELETE_ROWS_EVENT_V1:
-            {
-                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                event.fillTable(context);
-                return event;
-            }
-        case LogEvent.ROTATE_EVENT:
-            {
-                RotateLogEvent event = new RotateLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition = new LogPosition(event.getFilename(),
-                        event.getPosition());
-                context.setLogPosition(logPosition);
-                return event;
-            }
-        case LogEvent.LOAD_EVENT:
-        case LogEvent.NEW_LOAD_EVENT:
-            {
-                LoadLogEvent event = new LoadLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported SLAVE_EVENT from: "
-                            + context.getLogPosition());
-                break;
-            }
-        case LogEvent.CREATE_FILE_EVENT:
-            {
-                CreateFileLogEvent event = new CreateFileLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.APPEND_BLOCK_EVENT:
-            {
-                AppendBlockLogEvent event = new AppendBlockLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.DELETE_FILE_EVENT:
-            {
-                DeleteFileLogEvent event = new DeleteFileLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.EXEC_LOAD_EVENT:
-            {
-                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
-                        buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.START_EVENT_V3:
-            {
-                /* This is sent only by MySQL <=4.x */
-                StartLogEventV3 event = new StartLogEventV3(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.STOP_EVENT:
-            {
-                StopLogEvent event = new StopLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.INTVAR_EVENT:
-            {
-                IntvarLogEvent event = new IntvarLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.RAND_EVENT:
-            {
-                RandLogEvent event = new RandLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.USER_VAR_EVENT:
-            {
-                UserVarLogEvent event = new UserVarLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.FORMAT_DESCRIPTION_EVENT:
-            {
-                descriptionEvent = new FormatDescriptionLogEvent(header,
-                        buffer, descriptionEvent);
-                context.setFormatDescription(descriptionEvent);
-                return descriptionEvent;
-            }
-        case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
-                            + context.getLogPosition());
-                // ev = new Write_rows_log_event_old(buf, event_len,
-                // description_event);
-                break;
-            }
-        case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
-                            + context.getLogPosition());
-                // ev = new Update_rows_log_event_old(buf, event_len,
-                // description_event);
-                break;
-            }
-        case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
-            {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
-                            + context.getLogPosition());
-                // ev = new Delete_rows_log_event_old(buf, event_len,
-                // description_event);
-                break;
-            }
-        case LogEvent.BEGIN_LOAD_QUERY_EVENT:
-            {
-                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
-                        header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
-            {
-                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
-                        header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.INCIDENT_EVENT:
-            {
-                IncidentLogEvent event = new IncidentLogEvent(header, buffer,
-                        descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.HEARTBEAT_LOG_EVENT:
-        {
-            HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
-                    descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-        }
-        case LogEvent.IGNORABLE_LOG_EVENT:
-            {
-                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.ROWS_QUERY_LOG_EVENT:
-            {
-                RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
-                /* updating position in context */
-                logPosition.position = header.getLogPos();
-                return event;
-            }
-        case LogEvent.WRITE_ROWS_EVENT: {
                 RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.UPDATE_ROWS_EVENT: {
+            case LogEvent.UPDATE_ROWS_EVENT_V1: {
                 RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.DELETE_ROWS_EVENT: {
+            case LogEvent.DELETE_ROWS_EVENT_V1: {
                 RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
                 return event;
             }
-        case LogEvent.GTID_LOG_EVENT:
+            case LogEvent.ROTATE_EVENT: {
-        case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
+                RotateLogEvent event = new RotateLogEvent(header, buffer, descriptionEvent);
-            {
+                /* updating position in context */
-                GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
+                logPosition = new LogPosition(event.getFilename(), event.getPosition());
+                context.setLogPosition(logPosition);
+                return event;
+            }
+            case LogEvent.LOAD_EVENT:
+            case LogEvent.NEW_LOAD_EVENT: {
+                LoadLogEvent event = new LoadLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
+            case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
             {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: "
+                                                        + context.getLogPosition());
+                break;
+            }
+            case LogEvent.CREATE_FILE_EVENT: {
+                CreateFileLogEvent event = new CreateFileLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.APPEND_BLOCK_EVENT: {
+                AppendBlockLogEvent event = new AppendBlockLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.DELETE_FILE_EVENT: {
+                DeleteFileLogEvent event = new DeleteFileLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.EXEC_LOAD_EVENT: {
+                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.START_EVENT_V3: {
+                /* This is sent only by MySQL <=4.x */
+                StartLogEventV3 event = new StartLogEventV3(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.STOP_EVENT: {
+                StopLogEvent event = new StopLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.INTVAR_EVENT: {
+                IntvarLogEvent event = new IntvarLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.RAND_EVENT: {
+                RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.USER_VAR_EVENT: {
+                UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.FORMAT_DESCRIPTION_EVENT: {
+                descriptionEvent = new FormatDescriptionLogEvent(header, buffer, descriptionEvent);
+                context.setFormatDescription(descriptionEvent);
+                return descriptionEvent;
+            }
+            case LogEvent.PRE_GA_WRITE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+                                                        + context.getLogPosition());
+                // ev = new Write_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+            case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+                                                        + context.getLogPosition());
+                // ev = new Update_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+            case LogEvent.PRE_GA_DELETE_ROWS_EVENT: {
+                if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+                                                        + context.getLogPosition());
+                // ev = new Delete_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+            case LogEvent.BEGIN_LOAD_QUERY_EVENT: {
+                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.EXECUTE_LOAD_QUERY_EVENT: {
+                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.INCIDENT_EVENT: {
+                IncidentLogEvent event = new IncidentLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.HEARTBEAT_LOG_EVENT: {
+                HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.IGNORABLE_LOG_EVENT: {
+                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.ROWS_QUERY_LOG_EVENT: {
+                RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.WRITE_ROWS_EVENT: {
+                RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.UPDATE_ROWS_EVENT: {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.DELETE_ROWS_EVENT: {
+                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.GTID_LOG_EVENT:
+            case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
+                GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                if (gtidSet != null) {
+                    gtidSet.update(event.getGtidStr());
+                    // update latest gtid
+                    header.putGtidStr(gtidSet);
+                }
+                return event;
+            }
+            case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
                 PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
             }
-        case LogEvent.ANNOTATE_ROWS_EVENT:
+            case LogEvent.TRANSACTION_CONTEXT_EVENT: {
-            {
+                TransactionContextLogEvent event = new TransactionContextLogEvent(header, buffer, descriptionEvent);
-            AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        case LogEvent.BINLOG_CHECKPOINT_EVENT:
-            {
-            BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        case LogEvent.GTID_EVENT:
-            {
-            MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        case LogEvent.GTID_LIST_EVENT:
-            {
-            MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
-            /* updating position in context */
-            logPosition.position = header.getLogPos();
-            return event;
-            }
-        default:
-            /*
-                Create an object of Ignorable_log_event for unrecognized sub-class.
-                So that SLAVE SQL THREAD will only update the position and continue.
-             */
-            if((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0){
-                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 return event;
-            }else {
-                if (logger.isWarnEnabled())
-                    logger.warn("Skipping unrecognized binlog event "
-                            + LogEvent.getTypeName(header.getType()) + " from: "
-                            + context.getLogPosition());
             }
+            case LogEvent.VIEW_CHANGE_EVENT: {
+                ViewChangeEvent event = new ViewChangeEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.XA_PREPARE_LOG_EVENT: {
+                XaPrepareLogEvent event = new XaPrepareLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.ANNOTATE_ROWS_EVENT: {
+                AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                header.putGtidStr(context.getGtidSet());
+                return event;
+            }
+            case LogEvent.BINLOG_CHECKPOINT_EVENT: {
+                BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.GTID_EVENT: {
+                MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.GTID_LIST_EVENT: {
+                MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            case LogEvent.START_ENCRYPTION_EVENT: {
+                StartEncryptionLogEvent event = new StartEncryptionLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+            default:
+                /*
+                 * Create an object of Ignorable_log_event for unrecognized
+                 * sub-class. So that SLAVE SQL THREAD will only update the
+                 * position and continue.
+                 */
+                if ((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0) {
+                    IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
+                    /* updating position in context */
+                    logPosition.position = header.getLogPos();
+                    return event;
+                } else {
+                    if (logger.isWarnEnabled()) {
+                        logger.warn("Skipping unrecognized binlog event " + LogEvent.getTypeName(header.getType())
+                                    + " from: " + context.getLogPosition());
+                    }
+                }
         }
 
         /* updating position in context */

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java

@@ -7,239 +7,228 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
 
 /**
  * Binary log event definitions. This includes generic code common to all types
- * of log events, as well as specific code for each type of log event.
+ * of log events, as well as specific code for each type of log event. - All
- * 
+ * numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored in
- * - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
+ * little endian, i.e., the least significant byte first, unless otherwise
- * in little endian, i.e., the least significant byte first, unless otherwise
+ * specified. representation of unsigned integers, called Packed Integer. A
- * specified.
+ * Packed Integer has the capacity of storing up to 8-byte integers, while small
- * 
+ * integers still can use 1, 3, or 4 bytes. The value of the first byte
- * representation of unsigned integers, called Packed Integer. A Packed Integer
+ * determines how to read the number, according to the following table:
- * has the capacity of storing up to 8-byte integers, while small integers still
- * can use 1, 3, or 4 bytes. The value of the first byte determines how to read
- * the number, according to the following table:
- * 
  * <table>
  * <caption>Format of Packed Integer</caption>
- * 
  * <tr>
  * <th>First byte</th>
  * <th>Format</th>
  * </tr>
- * 
  * <tr>
  * <td>0-250</td>
  * <td>The first byte is the number (in the range 0-250), and no more bytes are
  * used.</td>
  * </tr>
- * 
  * <tr>
  * <td>252</td>
  * <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
  * </tr>
- * 
  * <tr>
  * <td>253</td>
  * <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
  * </tr>
- * 
  * <tr>
  * <td>254</td>
  * <td>Eight more bytes are used. The number is in the range
  * 0xffffff-0xffffffffffffffff.</td>
  * </tr>
- * 
  * </table>
- * 
  * - Strings are stored in various formats. The format of each string is
  * documented separately.
  * 
  * @see mysql-5.1.60/sql/log_event.h
- * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class LogEvent
+public abstract class LogEvent {
-{
+
     /*
-     * 3 is MySQL 4.x; 4 is MySQL 5.0.0.
+     * 3 is MySQL 4.x; 4 is MySQL 5.0.0. Compared to version 3, version 4 has: -
-     * Compared to version 3, version 4 has:
+     * a different Start_log_event, which includes info about the binary log
-     * - a different Start_log_event, which includes info about the binary log
      * (sizes of headers); this info is included for better compatibility if the
-     * master's MySQL version is different from the slave's.
+     * master's MySQL version is different from the slave's. - all events have a
-     * - all events have a unique ID (the triplet (server_id, timestamp at server
+     * unique ID (the triplet (server_id, timestamp at server start, other) to
-     * start, other) to be sure an event is not executed more than once in a
+     * be sure an event is not executed more than once in a multimaster setup,
-     * multimaster setup, example:
+     * example: M1 / \ v v M2 M3 \ / v v S if a query is run on M1, it will
-     *              M1
+     * arrive twice on S, so we need that S remembers the last unique ID it has
-     *            /   \
+     * processed, to compare and know if the event should be skipped or not.
-     *           v     v
+     * Example of ID: we already have the server id (4 bytes), plus:
-     *           M2    M3
-     *           \     /
-     *            v   v
-     *              S
-     * if a query is run on M1, it will arrive twice on S, so we need that S
-     * remembers the last unique ID it has processed, to compare and know if the
-     * event should be skipped or not. Example of ID: we already have the server id
-     * (4 bytes), plus:
      * timestamp_when_the_master_started (4 bytes), a counter (a sequence number
      * which increments every time we write an event to the binlog) (3 bytes).
      * Q: how do we handle when the counter is overflowed and restarts from 0 ?
-     * 
      * - Query and Load (Create or Execute) events may have a more precise
-     *   timestamp (with microseconds), number of matched/affected/warnings rows
+     * timestamp (with microseconds), number of matched/affected/warnings rows
-     * and fields of session variables: SQL_MODE,
+     * and fields of session variables: SQL_MODE, FOREIGN_KEY_CHECKS,
-     * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
+     * UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and charsets, the
-     * charsets, the PASSWORD() version (old/new/...).
+     * PASSWORD() version (old/new/...).
      */
-    public static final int    BINLOG_VERSION           = 4;
+    public static final int    BINLOG_VERSION                           = 4;
 
     /* Default 5.0 server version */
-    public static final String SERVER_VERSION           = "5.0";
+    public static final String SERVER_VERSION                           = "5.0";
 
     /**
      * Event header offsets; these point to places inside the fixed header.
      */
-    public static final int    EVENT_TYPE_OFFSET        = 4;
+    public static final int    EVENT_TYPE_OFFSET                        = 4;
-    public static final int    SERVER_ID_OFFSET         = 5;
+    public static final int    SERVER_ID_OFFSET                         = 5;
-    public static final int    EVENT_LEN_OFFSET         = 9;
+    public static final int    EVENT_LEN_OFFSET                         = 9;
-    public static final int    LOG_POS_OFFSET           = 13;
+    public static final int    LOG_POS_OFFSET                           = 13;
-    public static final int    FLAGS_OFFSET             = 17;
+    public static final int    FLAGS_OFFSET                             = 17;
 
     /* event-specific post-header sizes */
     // where 3.23, 4.x and 5.0 agree
-    public static final int    QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2);
+    public static final int    QUERY_HEADER_MINIMAL_LEN                 = (4 + 4 + 1 + 2);
     // where 5.0 differs: 2 for len of N-bytes vars.
-    public static final int    QUERY_HEADER_LEN         = (QUERY_HEADER_MINIMAL_LEN + 2);
+    public static final int    QUERY_HEADER_LEN                         = (QUERY_HEADER_MINIMAL_LEN + 2);
 
     /* Enumeration type for the different types of log events. */
-    public static final int    UNKNOWN_EVENT            = 0;
+    public static final int    UNKNOWN_EVENT                            = 0;
-    public static final int    START_EVENT_V3           = 1;
+    public static final int    START_EVENT_V3                           = 1;
-    public static final int    QUERY_EVENT              = 2;
+    public static final int    QUERY_EVENT                              = 2;
-    public static final int    STOP_EVENT               = 3;
+    public static final int    STOP_EVENT                               = 3;
-    public static final int    ROTATE_EVENT             = 4;
+    public static final int    ROTATE_EVENT                             = 4;
-    public static final int    INTVAR_EVENT             = 5;
+    public static final int    INTVAR_EVENT                             = 5;
-    public static final int    LOAD_EVENT               = 6;
+    public static final int    LOAD_EVENT                               = 6;
-    public static final int    SLAVE_EVENT              = 7;
+    public static final int    SLAVE_EVENT                              = 7;
-    public static final int    CREATE_FILE_EVENT        = 8;
+    public static final int    CREATE_FILE_EVENT                        = 8;
-    public static final int    APPEND_BLOCK_EVENT       = 9;
+    public static final int    APPEND_BLOCK_EVENT                       = 9;
-    public static final int    EXEC_LOAD_EVENT          = 10;
+    public static final int    EXEC_LOAD_EVENT                          = 10;
-    public static final int    DELETE_FILE_EVENT        = 11;
+    public static final int    DELETE_FILE_EVENT                        = 11;
 
     /**
      * NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer sql_ex,
      * allowing multibyte TERMINATED BY etc; both types share the same class
      * (Load_log_event)
      */
-    public static final int    NEW_LOAD_EVENT           = 12;
+    public static final int    NEW_LOAD_EVENT                           = 12;
-    public static final int    RAND_EVENT               = 13;
+    public static final int    RAND_EVENT                               = 13;
-    public static final int    USER_VAR_EVENT           = 14;
+    public static final int    USER_VAR_EVENT                           = 14;
-    public static final int    FORMAT_DESCRIPTION_EVENT = 15;
+    public static final int    FORMAT_DESCRIPTION_EVENT                 = 15;
-    public static final int    XID_EVENT                = 16;
+    public static final int    XID_EVENT                                = 16;
-    public static final int    BEGIN_LOAD_QUERY_EVENT   = 17;
+    public static final int    BEGIN_LOAD_QUERY_EVENT                   = 17;
-    public static final int    EXECUTE_LOAD_QUERY_EVENT = 18;
+    public static final int    EXECUTE_LOAD_QUERY_EVENT                 = 18;
 
-    public static final int    TABLE_MAP_EVENT          = 19;
+    public static final int    TABLE_MAP_EVENT                          = 19;
 
     /**
      * These event numbers were used for 5.1.0 to 5.1.15 and are therefore
      * obsolete.
      */
-    public static final int    PRE_GA_WRITE_ROWS_EVENT  = 20;
+    public static final int    PRE_GA_WRITE_ROWS_EVENT                  = 20;
-    public static final int    PRE_GA_UPDATE_ROWS_EVENT = 21;
+    public static final int    PRE_GA_UPDATE_ROWS_EVENT                 = 21;
-    public static final int    PRE_GA_DELETE_ROWS_EVENT = 22;
+    public static final int    PRE_GA_DELETE_ROWS_EVENT                 = 22;
 
     /**
      * These event numbers are used from 5.1.16 and forward
      */
-    public static final int    WRITE_ROWS_EVENT_V1      = 23;
+    public static final int    WRITE_ROWS_EVENT_V1                      = 23;
-    public static final int    UPDATE_ROWS_EVENT_V1     = 24;
+    public static final int    UPDATE_ROWS_EVENT_V1                     = 24;
-    public static final int    DELETE_ROWS_EVENT_V1     = 25;
+    public static final int    DELETE_ROWS_EVENT_V1                     = 25;
 
     /**
      * Something out of the ordinary happened on the master
      */
-    public static final int    INCIDENT_EVENT           = 26;
+    public static final int    INCIDENT_EVENT                           = 26;
 
     /**
-     * Heartbeat event to be send by master at its idle time to ensure master's online status to slave
+     * Heartbeat event to be send by master at its idle time to ensure master's
+     * online status to slave
      */
-    public static final int    HEARTBEAT_LOG_EVENT      = 27;
+    public static final int    HEARTBEAT_LOG_EVENT                      = 27;
 
     /**
-     * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
+     * In some situations, it is necessary to send over ignorable data to the
-     * case there is code for handling it, but which can be ignored if it is not recognized.
+     * slave: data that a slave can handle in case there is code for handling
+     * it, but which can be ignored if it is not recognized.
      */
-    public static final int    IGNORABLE_LOG_EVENT      = 28;
+    public static final int    IGNORABLE_LOG_EVENT                      = 28;
-    public static final int    ROWS_QUERY_LOG_EVENT     = 29;
+    public static final int    ROWS_QUERY_LOG_EVENT                     = 29;
 
     /** Version 2 of the Row events */
-    public static final int    WRITE_ROWS_EVENT         = 30;
+    public static final int    WRITE_ROWS_EVENT                         = 30;
-    public static final int    UPDATE_ROWS_EVENT        = 31;
+    public static final int    UPDATE_ROWS_EVENT                        = 31;
-    public static final int    DELETE_ROWS_EVENT        = 32;
+    public static final int    DELETE_ROWS_EVENT                        = 32;
 
-    public static final int    GTID_LOG_EVENT           = 33;
+    public static final int    GTID_LOG_EVENT                           = 33;
-    public static final int    ANONYMOUS_GTID_LOG_EVENT = 34;
+    public static final int    ANONYMOUS_GTID_LOG_EVENT                 = 34;
+
+    public static final int    PREVIOUS_GTIDS_LOG_EVENT                 = 35;
+
+    /* MySQL 5.7 events */
+    public static final int    TRANSACTION_CONTEXT_EVENT                = 36;
+
+    public static final int    VIEW_CHANGE_EVENT                        = 37;
+
+    /* Prepared XA transaction terminal event similar to Xid */
+    public static final int    XA_PREPARE_LOG_EVENT                     = 38;
 
-    public static final int    PREVIOUS_GTIDS_LOG_EVENT = 35;
-    
     // mariaDb 5.5.34
     /* New MySQL/Sun events are to be added right above this comment */
-    public static final int    MYSQL_EVENTS_END         = 36;
+    public static final int    MYSQL_EVENTS_END                         = 39;
 
-    public static final int    MARIA_EVENTS_BEGIN       = 160;
+    public static final int    MARIA_EVENTS_BEGIN                       = 160;
     /* New Maria event numbers start from here */
-    public static final int    ANNOTATE_ROWS_EVENT      = 160;
+    public static final int    ANNOTATE_ROWS_EVENT                      = 160;
     /*
-    Binlog checkpoint event. Used for XA crash recovery on the master, not used
+     * Binlog checkpoint event. Used for XA crash recovery on the master, not
-    in replication.
+     * used in replication. A binlog checkpoint event specifies a binlog file
-    A binlog checkpoint event specifies a binlog file such that XA crash
+     * such that XA crash recovery can start from that file - and it is
-    recovery can start from that file - and it is guaranteed to find all XIDs
+     * guaranteed to find all XIDs that are prepared in storage engines but not
-    that are prepared in storage engines but not yet committed.
+     * yet committed.
      */
-    public static final int     BINLOG_CHECKPOINT_EVENT = 161;
+    public static final int    BINLOG_CHECKPOINT_EVENT                  = 161;
     /*
-    Gtid event. For global transaction ID, used to start a new event group,
+     * Gtid event. For global transaction ID, used to start a new event group,
-    instead of the old BEGIN query event, and also to mark stand-alone
+     * instead of the old BEGIN query event, and also to mark stand-alone
-    events.
+     * events.
      */
-    public static final int    GTID_EVENT               = 162;
+    public static final int    GTID_EVENT                               = 162;
     /*
-    Gtid list event. Logged at the start of every binlog, to record the
+     * Gtid list event. Logged at the start of every binlog, to record the
-    current replication state. This consists of the last GTID seen for
+     * current replication state. This consists of the last GTID seen for each
-    each replication domain.
+     * replication domain.
      */
-    public static final int    GTID_LIST_EVENT          = 163;
+    public static final int    GTID_LIST_EVENT                          = 163;
+
+    public static final int    START_ENCRYPTION_EVENT                   = 164;
 
     /** end marker */
-    public static final int    ENUM_END_EVENT           = 164;
+    public static final int    ENUM_END_EVENT                           = 165;
 
     /**
-    1 byte length, 1 byte format
+     * 1 byte length, 1 byte format Length is total length in bytes, including 2
-    Length is total length in bytes, including 2 byte header
+     * byte header Length values 0 and 1 are currently invalid and reserved.
-    Length values 0 and 1 are currently invalid and reserved.
+     */
-    */
+    public static final int    EXTRA_ROW_INFO_LEN_OFFSET                = 0;
-    public static final int    EXTRA_ROW_INFO_LEN_OFFSET    = 0;
+    public static final int    EXTRA_ROW_INFO_FORMAT_OFFSET             = 1;
-    public static final int    EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
+    public static final int    EXTRA_ROW_INFO_HDR_BYTES                 = 2;
-    public static final int    EXTRA_ROW_INFO_HDR_BYTES     = 2;
+    public static final int    EXTRA_ROW_INFO_MAX_PAYLOAD               = (255 - EXTRA_ROW_INFO_HDR_BYTES);
-    public static final int    EXTRA_ROW_INFO_MAX_PAYLOAD   = (255 - EXTRA_ROW_INFO_HDR_BYTES);
+
-    
     // Events are without checksum though its generator
-    public static final int    BINLOG_CHECKSUM_ALG_OFF      = 0;
+    public static final int    BINLOG_CHECKSUM_ALG_OFF                  = 0;
     // is checksum-capable New Master (NM).
     // CRC32 of zlib algorithm.
-    public static final int    BINLOG_CHECKSUM_ALG_CRC32    = 1;
+    public static final int    BINLOG_CHECKSUM_ALG_CRC32                = 1;
     // the cut line: valid alg range is [1, 0x7f].
-    public static final int    BINLOG_CHECKSUM_ALG_ENUM_END = 2;
+    public static final int    BINLOG_CHECKSUM_ALG_ENUM_END             = 2;
     // special value to tag undetermined yet checksum
-    public static final int    BINLOG_CHECKSUM_ALG_UNDEF    = 255;
+    public static final int    BINLOG_CHECKSUM_ALG_UNDEF                = 255;
     // or events from checksum-unaware servers
 
-    public static final int    CHECKSUM_CRC32_SIGNATURE_LEN = 4;
+    public static final int    CHECKSUM_CRC32_SIGNATURE_LEN             = 4;
-    public static final int    BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
+    public static final int    BINLOG_CHECKSUM_ALG_DESC_LEN             = 1;
     /**
      * defined statically while there is just one alg implemented
      */
-    public static final int    BINLOG_CHECKSUM_LEN          = CHECKSUM_CRC32_SIGNATURE_LEN;
+    public static final int    BINLOG_CHECKSUM_LEN                      = CHECKSUM_CRC32_SIGNATURE_LEN;
-    
+
     /* MySQL or old MariaDB slave with no announced capability. */
     public static final int    MARIA_SLAVE_CAPABILITY_UNKNOWN           = 0;
 
@@ -258,124 +247,122 @@ public abstract class LogEvent
 
     /* Our capability. */
     public static final int    MARIA_SLAVE_CAPABILITY_MINE              = MARIA_SLAVE_CAPABILITY_GTID;
-    
-    /**
-        For an event, 'e', carrying a type code, that a slave,
-        's', does not recognize, 's' will check 'e' for
-        LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
-        is ignored. Otherwise, 's' acknowledges that it has
-        found an unknown event in the relay log.
-     */
-    public static final int   LOG_EVENT_IGNORABLE_F    = 0x80;
-    
-    /** enum_field_types */
-    public static final int    MYSQL_TYPE_DECIMAL       = 0;
-    public static final int    MYSQL_TYPE_TINY          = 1;
-    public static final int    MYSQL_TYPE_SHORT         = 2;
-    public static final int    MYSQL_TYPE_LONG          = 3;
-    public static final int    MYSQL_TYPE_FLOAT         = 4;
-    public static final int    MYSQL_TYPE_DOUBLE        = 5;
-    public static final int    MYSQL_TYPE_NULL          = 6;
-    public static final int    MYSQL_TYPE_TIMESTAMP     = 7;
-    public static final int    MYSQL_TYPE_LONGLONG      = 8;
-    public static final int    MYSQL_TYPE_INT24         = 9;
-    public static final int    MYSQL_TYPE_DATE          = 10;
-    public static final int    MYSQL_TYPE_TIME          = 11;
-    public static final int    MYSQL_TYPE_DATETIME      = 12;
-    public static final int    MYSQL_TYPE_YEAR          = 13;
-    public static final int    MYSQL_TYPE_NEWDATE       = 14;
-    public static final int    MYSQL_TYPE_VARCHAR       = 15;
-    public static final int    MYSQL_TYPE_BIT           = 16;
-    public static final int    MYSQL_TYPE_TIMESTAMP2    = 17;
-    public static final int    MYSQL_TYPE_DATETIME2     = 18;
-    public static final int    MYSQL_TYPE_TIME2         = 19;
-    public static final int    MYSQL_TYPE_NEWDECIMAL    = 246;
-    public static final int    MYSQL_TYPE_ENUM          = 247;
-    public static final int    MYSQL_TYPE_SET           = 248;
-    public static final int    MYSQL_TYPE_TINY_BLOB     = 249;
-    public static final int    MYSQL_TYPE_MEDIUM_BLOB   = 250;
-    public static final int    MYSQL_TYPE_LONG_BLOB     = 251;
-    public static final int    MYSQL_TYPE_BLOB          = 252;
-    public static final int    MYSQL_TYPE_VAR_STRING    = 253;
-    public static final int    MYSQL_TYPE_STRING        = 254;
-    public static final int    MYSQL_TYPE_GEOMETRY      = 255;
 
-    public static String getTypeName(final int type)
+    /**
-    {
+     * For an event, 'e', carrying a type code, that a slave, 's', does not
-        switch (type)
+     * recognize, 's' will check 'e' for LOG_EVENT_IGNORABLE_F, and if the flag
-        {
+     * is set, then 'e' is ignored. Otherwise, 's' acknowledges that it has
-        case START_EVENT_V3:
+     * found an unknown event in the relay log.
-            return "Start_v3";
+     */
-        case STOP_EVENT:
+    public static final int    LOG_EVENT_IGNORABLE_F                    = 0x80;
-            return "Stop";
+
-        case QUERY_EVENT:
+    /** enum_field_types */
-            return "Query";
+    public static final int    MYSQL_TYPE_DECIMAL                       = 0;
-        case ROTATE_EVENT:
+    public static final int    MYSQL_TYPE_TINY                          = 1;
-            return "Rotate";
+    public static final int    MYSQL_TYPE_SHORT                         = 2;
-        case INTVAR_EVENT:
+    public static final int    MYSQL_TYPE_LONG                          = 3;
-            return "Intvar";
+    public static final int    MYSQL_TYPE_FLOAT                         = 4;
-        case LOAD_EVENT:
+    public static final int    MYSQL_TYPE_DOUBLE                        = 5;
-            return "Load";
+    public static final int    MYSQL_TYPE_NULL                          = 6;
-        case NEW_LOAD_EVENT:
+    public static final int    MYSQL_TYPE_TIMESTAMP                     = 7;
-            return "New_load";
+    public static final int    MYSQL_TYPE_LONGLONG                      = 8;
-        case SLAVE_EVENT:
+    public static final int    MYSQL_TYPE_INT24                         = 9;
-            return "Slave";
+    public static final int    MYSQL_TYPE_DATE                          = 10;
-        case CREATE_FILE_EVENT:
+    public static final int    MYSQL_TYPE_TIME                          = 11;
-            return "Create_file";
+    public static final int    MYSQL_TYPE_DATETIME                      = 12;
-        case APPEND_BLOCK_EVENT:
+    public static final int    MYSQL_TYPE_YEAR                          = 13;
-            return "Append_block";
+    public static final int    MYSQL_TYPE_NEWDATE                       = 14;
-        case DELETE_FILE_EVENT:
+    public static final int    MYSQL_TYPE_VARCHAR                       = 15;
-            return "Delete_file";
+    public static final int    MYSQL_TYPE_BIT                           = 16;
-        case EXEC_LOAD_EVENT:
+    public static final int    MYSQL_TYPE_TIMESTAMP2                    = 17;
-            return "Exec_load";
+    public static final int    MYSQL_TYPE_DATETIME2                     = 18;
-        case RAND_EVENT:
+    public static final int    MYSQL_TYPE_TIME2                         = 19;
-            return "RAND";
+    public static final int    MYSQL_TYPE_JSON                          = 245;
-        case XID_EVENT:
+    public static final int    MYSQL_TYPE_NEWDECIMAL                    = 246;
-            return "Xid";
+    public static final int    MYSQL_TYPE_ENUM                          = 247;
-        case USER_VAR_EVENT:
+    public static final int    MYSQL_TYPE_SET                           = 248;
-            return "User var";
+    public static final int    MYSQL_TYPE_TINY_BLOB                     = 249;
-        case FORMAT_DESCRIPTION_EVENT:
+    public static final int    MYSQL_TYPE_MEDIUM_BLOB                   = 250;
-            return "Format_desc";
+    public static final int    MYSQL_TYPE_LONG_BLOB                     = 251;
-        case TABLE_MAP_EVENT:
+    public static final int    MYSQL_TYPE_BLOB                          = 252;
-            return "Table_map";
+    public static final int    MYSQL_TYPE_VAR_STRING                    = 253;
-        case PRE_GA_WRITE_ROWS_EVENT:
+    public static final int    MYSQL_TYPE_STRING                        = 254;
-            return "Write_rows_event_old";
+    public static final int    MYSQL_TYPE_GEOMETRY                      = 255;
-        case PRE_GA_UPDATE_ROWS_EVENT:
+
-            return "Update_rows_event_old";
+    public static String getTypeName(final int type) {
-        case PRE_GA_DELETE_ROWS_EVENT:
+        switch (type) {
-            return "Delete_rows_event_old";
+            case START_EVENT_V3:
-        case WRITE_ROWS_EVENT_V1:
+                return "Start_v3";
-            return "Write_rows_v1";
+            case STOP_EVENT:
-        case UPDATE_ROWS_EVENT_V1:
+                return "Stop";
-            return "Update_rows_v1";
+            case QUERY_EVENT:
-        case DELETE_ROWS_EVENT_V1:
+                return "Query";
-            return "Delete_rows_v1";
+            case ROTATE_EVENT:
-        case BEGIN_LOAD_QUERY_EVENT:
+                return "Rotate";
-            return "Begin_load_query";
+            case INTVAR_EVENT:
-        case EXECUTE_LOAD_QUERY_EVENT:
+                return "Intvar";
-            return "Execute_load_query";
+            case LOAD_EVENT:
-        case INCIDENT_EVENT:
+                return "Load";
-            return "Incident";
+            case NEW_LOAD_EVENT:
-        case HEARTBEAT_LOG_EVENT:
+                return "New_load";
-            return "Heartbeat";
+            case SLAVE_EVENT:
-        case IGNORABLE_LOG_EVENT:
+                return "Slave";
-            return "Ignorable";
+            case CREATE_FILE_EVENT:
-        case ROWS_QUERY_LOG_EVENT:
+                return "Create_file";
-            return "Rows_query";
+            case APPEND_BLOCK_EVENT:
-        case WRITE_ROWS_EVENT:
+                return "Append_block";
-            return "Write_rows";
+            case DELETE_FILE_EVENT:
-        case UPDATE_ROWS_EVENT:
+                return "Delete_file";
-            return "Update_rows";
+            case EXEC_LOAD_EVENT:
-        case DELETE_ROWS_EVENT:
+                return "Exec_load";
-            return "Delete_rows";
+            case RAND_EVENT:
-        case GTID_LOG_EVENT:
+                return "RAND";
-            return "Gtid";
+            case XID_EVENT:
-        case ANONYMOUS_GTID_LOG_EVENT:
+                return "Xid";
-            return "Anonymous_Gtid";
+            case USER_VAR_EVENT:
-        case PREVIOUS_GTIDS_LOG_EVENT:
+                return "User var";
-            return "Previous_gtids";
+            case FORMAT_DESCRIPTION_EVENT:
-        default:
+                return "Format_desc";
-            return "Unknown"; /* impossible */
+            case TABLE_MAP_EVENT:
+                return "Table_map";
+            case PRE_GA_WRITE_ROWS_EVENT:
+                return "Write_rows_event_old";
+            case PRE_GA_UPDATE_ROWS_EVENT:
+                return "Update_rows_event_old";
+            case PRE_GA_DELETE_ROWS_EVENT:
+                return "Delete_rows_event_old";
+            case WRITE_ROWS_EVENT_V1:
+                return "Write_rows_v1";
+            case UPDATE_ROWS_EVENT_V1:
+                return "Update_rows_v1";
+            case DELETE_ROWS_EVENT_V1:
+                return "Delete_rows_v1";
+            case BEGIN_LOAD_QUERY_EVENT:
+                return "Begin_load_query";
+            case EXECUTE_LOAD_QUERY_EVENT:
+                return "Execute_load_query";
+            case INCIDENT_EVENT:
+                return "Incident";
+            case HEARTBEAT_LOG_EVENT:
+                return "Heartbeat";
+            case IGNORABLE_LOG_EVENT:
+                return "Ignorable";
+            case ROWS_QUERY_LOG_EVENT:
+                return "Rows_query";
+            case WRITE_ROWS_EVENT:
+                return "Write_rows";
+            case UPDATE_ROWS_EVENT:
+                return "Update_rows";
+            case DELETE_ROWS_EVENT:
+                return "Delete_rows";
+            case GTID_LOG_EVENT:
+                return "Gtid";
+            case ANONYMOUS_GTID_LOG_EVENT:
+                return "Anonymous_Gtid";
+            case PREVIOUS_GTIDS_LOG_EVENT:
+                return "Previous_gtids";
+            default:
+                return "Unknown"; /* impossible */
         }
     }
 
@@ -383,16 +370,32 @@ public abstract class LogEvent
 
     protected final LogHeader  header;
 
-    protected LogEvent(LogHeader header)
+    /**
-    {
+     * mysql半同步semi标识
+     * 
+     * <pre>
+     * 0不需要semi ack 给mysql
+     * 1需要semi ack给mysql
+     * </pre>
+     */
+    protected int              semival;
+
+    public int getSemival() {
+        return semival;
+    }
+
+    public void setSemival(int semival) {
+        this.semival = semival;
+    }
+
+    protected LogEvent(LogHeader header){
         this.header = header;
     }
 
     /**
      * Return event header.
      */
-    public final LogHeader getHeader()
+    public final LogHeader getHeader() {
-    {
         return header;
     }
 
@@ -400,16 +403,14 @@ public abstract class LogEvent
      * The total size of this event, in bytes. In other words, this is the sum
      * of the sizes of Common-Header, Post-Header, and Body.
      */
-    public final int getEventLen()
+    public final int getEventLen() {
-    {
         return header.getEventLen();
     }
 
     /**
      * Server ID of the server that created the event.
      */
-    public final long getServerId()
+    public final long getServerId() {
-    {
         return header.getServerId();
     }
 
@@ -420,18 +421,15 @@ public abstract class LogEvent
      * file. In a relay log, this is the position of the next event in the
      * master's binlog.
      */
-    public final long getLogPos()
+    public final long getLogPos() {
-    {
         return header.getLogPos();
     }
 
     /**
      * The time when the query started, in seconds since 1970.
      */
-    public final long getWhen()
+    public final long getWhen() {
-    {
         return header.getWhen();
     }
-    
+
-    
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java

@@ -29,8 +29,8 @@ import java.util.Arrays;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public abstract class LogFetcher extends LogBuffer implements Closeable
+public abstract class LogFetcher extends LogBuffer implements Closeable {
-{
+
     /** Default initial capacity. */
     public static final int   DEFAULT_INITIAL_CAPACITY = 8192;
 
@@ -42,18 +42,15 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
 
     protected final float     factor;
 
-    public LogFetcher()
+    public LogFetcher(){
-    {
         this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
     }
 
-    public LogFetcher(final int initialCapacity)
+    public LogFetcher(final int initialCapacity){
-    {
         this(initialCapacity, DEFAULT_GROWTH_FACTOR);
     }
 
-    public LogFetcher(final int initialCapacity, final float growthFactor)
+    public LogFetcher(final int initialCapacity, final float growthFactor){
-    {
         this.buffer = new byte[initialCapacity];
         this.factor = growthFactor;
     }
@@ -65,15 +62,12 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
      * 
      * @param minCapacity the desired minimum capacity
      */
-    protected final void ensureCapacity(final int minCapacity)
+    protected final void ensureCapacity(final int minCapacity) {
-    {
         final int oldCapacity = buffer.length;
 
-        if (minCapacity > oldCapacity)
+        if (minCapacity > oldCapacity) {
-        {
             int newCapacity = (int) (oldCapacity * factor);
-            if (newCapacity < minCapacity)
+            if (newCapacity < minCapacity) newCapacity = minCapacity;
-                newCapacity = minCapacity;
 
             buffer = Arrays.copyOf(buffer, newCapacity);
         }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java

@@ -6,8 +6,8 @@ package com.taobao.tddl.dbsync.binlog;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class LogPosition implements Cloneable, Comparable<LogPosition>
+public class LogPosition implements Cloneable, Comparable<LogPosition> {
-{
+
     /* binlog file's name */
     protected String fileName;
 
@@ -19,8 +19,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @param fileName file name for binlog files: mysql-bin.000001
      */
-    public LogPosition(String fileName)
+    public LogPosition(String fileName){
-    {
         this.fileName = fileName;
         this.position = 0L;
     }
@@ -30,8 +29,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @param fileName file name for binlog files: mysql-bin.000001
      */
-    public LogPosition(String fileName, final long position)
+    public LogPosition(String fileName, final long position){
-    {
         this.fileName = fileName;
         this.position = position;
     }
@@ -39,31 +37,24 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
     /**
      * Binlog position copy init.
      */
-    public LogPosition(LogPosition source)
+    public LogPosition(LogPosition source){
-    {
         this.fileName = source.fileName;
         this.position = source.position;
     }
 
-    public final String getFileName()
+    public final String getFileName() {
-    {
         return fileName;
     }
 
-    public final long getPosition()
+    public final long getPosition() {
-    {
         return position;
     }
 
     /* Clone binlog position without CloneNotSupportedException */
-    public LogPosition clone()
+    public LogPosition clone() {
-    {
+        try {
-        try
-        {
             return (LogPosition) super.clone();
-        }
+        } catch (CloneNotSupportedException e) {
-        catch (CloneNotSupportedException e)
-        {
             // Never happend
             return null;
         }
@@ -72,12 +63,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
     /**
      * Compares with the specified fileName and position.
      */
-    public final int compareTo(String fileName, final long position)
+    public final int compareTo(String fileName, final long position) {
-    {
         final int val = this.fileName.compareTo(fileName);
 
-        if (val == 0)
+        if (val == 0) {
-        {
             return (int) (this.position - position);
         }
         return val;
@@ -88,12 +77,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Comparable#compareTo(java.lang.Object)
      */
-    public int compareTo(LogPosition o)
+    public int compareTo(LogPosition o) {
-    {
         final int val = fileName.compareTo(o.fileName);
 
-        if (val == 0)
+        if (val == 0) {
-        {
             return (int) (position - o.position);
         }
         return val;
@@ -104,13 +91,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Object#equals(java.lang.Object)
      */
-    public boolean equals(Object obj)
+    public boolean equals(Object obj) {
-    {
+        if (obj instanceof LogPosition) {
-        if (obj instanceof LogPosition)
-        {
             LogPosition pos = ((LogPosition) obj);
-            return fileName.equals(pos.fileName)
+            return fileName.equals(pos.fileName) && (this.position == pos.position);
-                    && (this.position == pos.position);
         }
         return false;
     }
@@ -120,8 +104,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
      * 
      * @see java.lang.Object#toString()
      */
-    public String toString()
+    public String toString() {
-    {
         return fileName + ':' + position;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java

@@ -9,8 +9,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class AppendBlockLogEvent extends LogEvent
+public class AppendBlockLogEvent extends LogEvent {
-{
+
     private final LogBuffer blockBuf;
     private final int       blockLen;
 
@@ -19,9 +19,7 @@ public class AppendBlockLogEvent extends LogEvent
     /* AB = "Append Block" */
     public static final int AB_FILE_ID_OFFSET = 0;
 
-    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
+    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -36,18 +34,15 @@ public class AppendBlockLogEvent extends LogEvent
         blockBuf = buffer.duplicate(blockLen);
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 
-    public final LogBuffer getBuffer()
+    public final LogBuffer getBuffer() {
-    {
         return blockBuf;
     }
 
-    public final byte[] getData()
+    public final byte[] getData() {
-    {
         return blockBuf.getData();
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java

@@ -10,11 +10,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
+public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent {
-{
+
-    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
+    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java

@@ -8,8 +8,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class CreateFileLogEvent extends LoadLogEvent
+public final class CreateFileLogEvent extends LoadLogEvent {
-{
+
     protected LogBuffer     blockBuf;
     protected int           blockLen;
     protected long          fileId;
@@ -20,9 +20,7 @@ public final class CreateFileLogEvent extends LoadLogEvent
     public static final int CF_FILE_ID_OFFSET = 0;
     public static final int CF_DATA_OFFSET    = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
 
-    public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
+    public CreateFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
 
         final int headerLen = descriptionEvent.commonHeaderLen;
@@ -30,44 +28,37 @@ public final class CreateFileLogEvent extends LoadLogEvent
         final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
 
         copyLogEvent(buffer,
-                ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
+            ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen) : (headerLen + loadHeaderLen + createFileHeaderLen)),
-                        : (headerLen + loadHeaderLen + createFileHeaderLen)),
+            descriptionEvent);
-                descriptionEvent);
 
-        if (descriptionEvent.binlogVersion != 1)
+        if (descriptionEvent.binlogVersion != 1) {
-        {
+            fileId = buffer.getUint32(headerLen + loadHeaderLen + CF_FILE_ID_OFFSET);
-            fileId = buffer.getUint32(headerLen + loadHeaderLen
-                    + CF_FILE_ID_OFFSET);
             /*
-              Note that it's ok to use get_data_size() below, because it is computed
+             * Note that it's ok to use get_data_size() below, because it is
-              with values we have already read from this event (because we called
+             * computed with values we have already read from this event
-              copy_log_event()); we are not using slave's format info to decode
+             * (because we called copy_log_event()); we are not using slave's
-              master's format, we are really using master's format info.
+             * format info to decode master's format, we are really using
-              Anyway, both formats should be identical (except the common_header_len)
+             * master's format info. Anyway, both formats should be identical
-              as these Load events are not changed between 4.0 and 5.0 (as logging of
+             * (except the common_header_len) as these Load events are not
-              LOAD DATA INFILE does not use Load_log_event in 5.0).
+             * changed between 4.0 and 5.0 (as logging of LOAD DATA INFILE does
-            */
+             * not use Load_log_event in 5.0).
+             */
             blockLen = buffer.limit() - buffer.position();
             blockBuf = buffer.duplicate(blockLen);
-        }
+        } else {
-        else
-        {
             initedFromOld = true;
         }
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 
-    public final LogBuffer getBuffer()
+    public final LogBuffer getBuffer() {
-    {
         return blockBuf;
     }
 
-    public final byte[] getData()
+    public final byte[] getData() {
-    {
         return blockBuf.getData();
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java

@@ -9,25 +9,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteFileLogEvent extends LogEvent
+public final class DeleteFileLogEvent extends LogEvent {
-{
+
     private final long      fileId;
 
     /* DF = "Delete File" */
     public static final int DF_FILE_ID_OFFSET = 0;
 
-    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
+    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET);
-        fileId = buffer.getUint32(); //  DF_FILE_ID_OFFSET
+        fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class DeleteRowsLogEvent extends RowsLogEvent
+public final class DeleteRowsLogEvent extends RowsLogEvent {
-{
+
-    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
+    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header, buffer, descriptionEvent);
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java

@@ -9,25 +9,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class ExecuteLoadLogEvent extends LogEvent
+public final class ExecuteLoadLogEvent extends LogEvent {
-{
+
     private final long      fileId;
 
     /* EL = "Execute Load" */
     public static final int EL_FILE_ID_OFFSET = 0;
 
-    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
+    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
         buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET);
-        fileId = buffer.getUint32(); //  EL_FILE_ID_OFFSET
+        fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java

@@ -8,7 +8,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * Event responsible for LOAD DATA execution, it similar to Query_log_event but
  * before executing the query it substitutes original filename in LOAD DATA
  * query with name of temporary file.
- * 
  * <ul>
  * <li>4 bytes. The ID of the file to load.</li>
  * <li>4 bytes. The start position within the statement for filename
@@ -22,8 +21,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
+public final class ExecuteLoadQueryLogEvent extends QueryLogEvent {
-{
+
     /** file_id of temporary file */
     private long            fileId;
 
@@ -53,9 +52,8 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
     public static final int ELQ_FN_POS_END_OFFSET   = ELQ_FILE_ID_OFFSET + 8;
     public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
 
-    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
+    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
-            FormatDescriptionLogEvent descriptionEvent) throws IOException
+                                                                                                                   throws IOException{
-    {
         super(header, buffer, descriptionEvent);
 
         buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
@@ -66,36 +64,27 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
         dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
 
         final int len = query.length();
-        if (fnPosStart > len || fnPosEnd > len
+        if (fnPosStart > len || fnPosEnd > len || dupHandling > LOAD_DUP_REPLACE) {
-                || dupHandling > LOAD_DUP_REPLACE)
+            throw new IOException(String.format("Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
-        {
+                                                + "fn_pos_end=%d, dup_handling=%d", fnPosStart, fnPosEnd, dupHandling));
-            throw new IOException(String.format(
-                    "Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
-                            + "fn_pos_end=%d, dup_handling=%d", fnPosStart,
-                    fnPosEnd, dupHandling));
         }
     }
 
-    public final int getFilenamePosStart()
+    public final int getFilenamePosStart() {
-    {
         return fnPosStart;
     }
 
-    public final int getFilenamePosEnd()
+    public final int getFilenamePosEnd() {
-    {
         return fnPosEnd;
     }
 
-    public final String getFilename()
+    public final String getFilename() {
-    {
+        if (query != null) return query.substring(fnPosStart, fnPosEnd).trim();
-        if (query != null)
-            return query.substring(fnPosStart, fnPosEnd).trim();
 
         return null;
     }
 
-    public final long getFileId()
+    public final long getFileId() {
-    {
         return fileId;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java

@@ -5,8 +5,8 @@ import java.io.IOException;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 
 /**
- * For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
+ * For binlog version 4. This event is saved by threads which read it, as they
- * ordinary events).
+ * need it for future use (to decode the ordinary events).
  * 
  * @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
@@ -15,23 +15,24 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
 public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
     /**
-     * The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
+     * The number of types we handle in Format_description_log_event
-     * exist in binlogs, it does not have a format).
+     * (UNKNOWN_EVENT is not to be handled, it does not exist in binlogs, it
+     * does not have a format).
      */
-    public static final int    LOG_EVENT_TYPES                     = (ENUM_END_EVENT - 1);
+    public static final int   LOG_EVENT_TYPES                     = (ENUM_END_EVENT - 1);
 
-    public static final int    ST_COMMON_HEADER_LEN_OFFSET         = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
+    public static final int   ST_COMMON_HEADER_LEN_OFFSET         = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
 
-    public static final int    OLD_HEADER_LEN                      = 13;
+    public static final int   OLD_HEADER_LEN                      = 13;
-    public static final int    LOG_EVENT_HEADER_LEN                = 19;
+    public static final int   LOG_EVENT_HEADER_LEN                = 19;
-    public static final int    LOG_EVENT_MINIMAL_HEADER_LEN        = 19;
+    public static final int   LOG_EVENT_MINIMAL_HEADER_LEN        = 19;
 
     /* event-specific post-header sizes */
-    public static final int    STOP_HEADER_LEN                     = 0;
+    public static final int   STOP_HEADER_LEN                     = 0;
-    public static final int    LOAD_HEADER_LEN                     = (4 + 4 + 4 + 1 + 1 + 4);
+    public static final int   LOAD_HEADER_LEN                     = (4 + 4 + 4 + 1 + 1 + 4);
-    public static final int    SLAVE_HEADER_LEN                    = 0;
+    public static final int   SLAVE_HEADER_LEN                    = 0;
-    public static final int    START_V3_HEADER_LEN                 = (2 + ST_SERVER_VER_LEN + 4);
+    public static final int   START_V3_HEADER_LEN                 = (2 + ST_SERVER_VER_LEN + 4);
-    public static final int    ROTATE_HEADER_LEN                   = 8;                                                       // this
+    public static final int   ROTATE_HEADER_LEN                   = 8;                                                       // this
     // is
     // FROZEN
     // (the
@@ -39,47 +40,53 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
     // post-header
     // is
     // frozen)
-    public static final int    INTVAR_HEADER_LEN                   = 0;
+    public static final int   INTVAR_HEADER_LEN                   = 0;
-    public static final int    CREATE_FILE_HEADER_LEN              = 4;
+    public static final int   CREATE_FILE_HEADER_LEN              = 4;
-    public static final int    APPEND_BLOCK_HEADER_LEN             = 4;
+    public static final int   APPEND_BLOCK_HEADER_LEN             = 4;
-    public static final int    EXEC_LOAD_HEADER_LEN                = 4;
+    public static final int   EXEC_LOAD_HEADER_LEN                = 4;
-    public static final int    DELETE_FILE_HEADER_LEN              = 4;
+    public static final int   DELETE_FILE_HEADER_LEN              = 4;
-    public static final int    NEW_LOAD_HEADER_LEN                 = LOAD_HEADER_LEN;
+    public static final int   NEW_LOAD_HEADER_LEN                 = LOAD_HEADER_LEN;
-    public static final int    RAND_HEADER_LEN                     = 0;
+    public static final int   RAND_HEADER_LEN                     = 0;
-    public static final int    USER_VAR_HEADER_LEN                 = 0;
+    public static final int   USER_VAR_HEADER_LEN                 = 0;
-    public static final int    FORMAT_DESCRIPTION_HEADER_LEN       = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
+    public static final int   FORMAT_DESCRIPTION_HEADER_LEN       = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
-    public static final int    XID_HEADER_LEN                      = 0;
+    public static final int   XID_HEADER_LEN                      = 0;
-    public static final int    BEGIN_LOAD_QUERY_HEADER_LEN         = APPEND_BLOCK_HEADER_LEN;
+    public static final int   BEGIN_LOAD_QUERY_HEADER_LEN         = APPEND_BLOCK_HEADER_LEN;
-    public static final int    ROWS_HEADER_LEN_V1                  = 8;
+    public static final int   ROWS_HEADER_LEN_V1                  = 8;
-    public static final int    TABLE_MAP_HEADER_LEN                = 8;
+    public static final int   TABLE_MAP_HEADER_LEN                = 8;
-    public static final int    EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
+    public static final int   EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
-    public static final int    EXECUTE_LOAD_QUERY_HEADER_LEN       = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
+    public static final int   EXECUTE_LOAD_QUERY_HEADER_LEN       = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
-    public static final int    INCIDENT_HEADER_LEN                 = 2;
+    public static final int   INCIDENT_HEADER_LEN                 = 2;
-    public static final int    HEARTBEAT_HEADER_LEN                = 0;
+    public static final int   HEARTBEAT_HEADER_LEN                = 0;
-    public static final int    IGNORABLE_HEADER_LEN                = 0;
+    public static final int   IGNORABLE_HEADER_LEN                = 0;
-    public static final int    ROWS_HEADER_LEN_V2                  = 10;
+    public static final int   ROWS_HEADER_LEN_V2                  = 10;
-    public static final int    ANNOTATE_ROWS_HEADER_LEN            = 0;
+    public static final int   TRANSACTION_CONTEXT_HEADER_LEN      = 18;
-    public static final int    BINLOG_CHECKPOINT_HEADER_LEN        = 4;
+    public static final int   VIEW_CHANGE_HEADER_LEN              = 52;
-    public static final int    GTID_HEADER_LEN                     = 19;
+    public static final int   XA_PREPARE_HEADER_LEN               = 0;
-    public static final int    GTID_LIST_HEADER_LEN                = 4;
 
-    public static final int    POST_HEADER_LENGTH                  = 11;
+    public static final int   ANNOTATE_ROWS_HEADER_LEN            = 0;
+    public static final int   BINLOG_CHECKPOINT_HEADER_LEN        = 4;
+    public static final int   GTID_HEADER_LEN                     = 19;
+    public static final int   GTID_LIST_HEADER_LEN                = 4;
+    public static final int   START_ENCRYPTION_HEADER_LEN         = 0;
+
+    public static final int   POST_HEADER_LENGTH                  = 11;
 
     public static final int   BINLOG_CHECKSUM_ALG_DESC_LEN        = 1;
     public static final int[] checksumVersionSplit                = { 5, 6, 1 };
     public static final long  checksumVersionProduct              = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
                                                                     * 256 + checksumVersionSplit[2];
     /**
-     * The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
+     * The size of the fixed header which _all_ events have (for binlogs written
-     * LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
+     * by this version, this is equal to LOG_EVENT_HEADER_LEN), except
+     * FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
      * LOG_EVENT_MINIMAL_HEADER_LEN).
      */
-    protected final int        commonHeaderLen;
+    protected final int       commonHeaderLen;
-    protected int              numberOfEventTypes;
+    protected int             numberOfEventTypes;
 
     /** The list of post-headers' lengthes */
-    protected final short[]    postHeaderLen;
+    protected final short[]   postHeaderLen;
-    protected int[]            serverVersionSplit                  = new int[3];
+    protected int[]           serverVersionSplit                  = new int[3];
 
     public FormatDescriptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
                                                                                                                     throws IOException{
@@ -105,12 +112,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         calcServerVersionSplit();
         long calc = getVersionProduct();
         if (calc >= checksumVersionProduct) {
-            /* the last bytes are the checksum alg desc and value (or value's room) */
+            /*
+             * the last bytes are the checksum alg desc and value (or value's
+             * room)
+             */
             numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
         }
-        
+
         if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= "
-                + numberOfEventTypes);
+                                                + numberOfEventTypes);
     }
 
     /** MySQL 5.0 format descriptions. */
@@ -136,6 +146,11 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
         }
     }
 
+    public FormatDescriptionLogEvent(final int binlogVersion, int binlogChecksum){
+        this(binlogVersion);
+        this.header.checksumAlg = binlogChecksum;
+    }
+
     public FormatDescriptionLogEvent(final int binlogVersion){
         this.binlogVersion = binlogVersion;
 
@@ -147,7 +162,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 commonHeaderLen = LOG_EVENT_HEADER_LEN;
                 numberOfEventTypes = LOG_EVENT_TYPES;
 
-                /* Note: all event types must explicitly fill in their lengths here. */
+                /*
+                 * Note: all event types must explicitly fill in their lengths
+                 * here.
+                 */
                 postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
                 postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
                 postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
@@ -171,11 +189,14 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
                 /*
-                 * We here have the possibility to simulate a master of before we changed the table map id to be stored
+                 * We here have the possibility to simulate a master of before
-                 * in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
+                 * we changed the table map id to be stored in 6 bytes: when it
-                 * compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
+                 * was stored in 4 bytes (=> post_header_len was 6). This is
-                 * that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
+                 * used to test backward compatibility. This code can be removed
-                 * test (rpl_row_4_bytes) too.
+                 * after a few months (today is Dec 21st 2005), when we know
+                 * that the 4-byte masters are not deployed anymore (check with
+                 * Tomas Ulin first!), and the accompanying test
+                 * (rpl_row_4_bytes) too.
                  */
                 postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
                 postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
@@ -186,26 +207,34 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 postHeaderLen[GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
                 postHeaderLen[ANONYMOUS_GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
                 postHeaderLen[PREVIOUS_GTIDS_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
+
+                postHeaderLen[TRANSACTION_CONTEXT_EVENT - 1] = TRANSACTION_CONTEXT_HEADER_LEN;
+                postHeaderLen[VIEW_CHANGE_EVENT - 1] = VIEW_CHANGE_HEADER_LEN;
+                postHeaderLen[XA_PREPARE_LOG_EVENT - 1] = XA_PREPARE_HEADER_LEN;
+
                 // mariadb 10
                 postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;
                 postHeaderLen[BINLOG_CHECKPOINT_EVENT - 1] = BINLOG_CHECKPOINT_HEADER_LEN;
                 postHeaderLen[GTID_EVENT - 1] = GTID_HEADER_LEN;
                 postHeaderLen[GTID_LIST_EVENT - 1] = GTID_LIST_HEADER_LEN;
+                postHeaderLen[START_ENCRYPTION_EVENT - 1] = START_ENCRYPTION_HEADER_LEN;
                 break;
 
             case 3: /* 4.0.x x>=2 */
                 /*
-                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
+                 * We build an artificial (i.e. not sent by the master) event,
-                 * versions send.
+                 * which describes what those old master versions send.
                  */
                 serverVersion = "4.0";
                 commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
 
                 /*
-                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
+                 * The first new event in binlog version 4 is Format_desc. So
-                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
+                 * any event type after that does not exist in older versions.
-                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
+                 * We use the events known by version 3, even if version 1 had
-                 * detect less corruptions).
+                 * only a subset of them (this is not a problem: it uses a few
+                 * bytes for nothing but unifies code; it does not make the
+                 * slave detect less corruptions).
                  */
                 numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
 
@@ -222,17 +251,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
 
             case 1: /* 3.23 */
                 /*
-                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
+                 * We build an artificial (i.e. not sent by the master) event,
-                 * versions send.
+                 * which describes what those old master versions send.
                  */
                 serverVersion = "3.23";
                 commonHeaderLen = OLD_HEADER_LEN;
 
                 /*
-                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
+                 * The first new event in binlog version 4 is Format_desc. So
-                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
+                 * any event type after that does not exist in older versions.
-                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
+                 * We use the events known by version 3, even if version 1 had
-                 * detect less corruptions).
+                 * only a subset of them (this is not a problem: it uses a few
+                 * bytes for nothing but unifies code; it does not make the
+                 * slave detect less corruptions).
                  */
                 numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
 
@@ -251,7 +282,7 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
                 commonHeaderLen = 0;
         }
     }
-    
+
     public void calcServerVersionSplit() {
         doServerVersionSplit(serverVersion, serverVersionSplit);
     }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java

@@ -1,5 +1,8 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
@@ -11,11 +14,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 public class GtidLogEvent extends LogEvent {
 
     // / Length of the commit_flag in event encoding
-    public static final int ENCODED_FLAG_LENGTH = 1;
+    public static final int ENCODED_FLAG_LENGTH         = 1;
     // / Length of SID in event encoding
-    public static final int ENCODED_SID_LENGTH  = 16;
+    public static final int ENCODED_SID_LENGTH          = 16;
+    public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2;
 
     private boolean         commitFlag;
+    private UUID            sid;
+    private long            gno;
+    private Long            lastCommitted;
+    private Long            sequenceNumber;
 
     public GtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
         super(header);
@@ -27,6 +35,21 @@ public class GtidLogEvent extends LogEvent {
         buffer.position(commonHeaderLen);
         commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH
 
+        byte[] bs = buffer.getData(ENCODED_SID_LENGTH);
+        ByteBuffer bb = ByteBuffer.wrap(bs);
+        long high = bb.getLong();
+        long low = bb.getLong();
+        sid = new UUID(high, low);
+
+        gno = buffer.getLong64();
+
+        // support gtid lastCommitted and sequenceNumber
+        // fix bug #776
+        if (buffer.hasRemaining() && buffer.remaining() > 16 && buffer.getUint8() == LOGICAL_TIMESTAMP_TYPE_CODE) {
+            lastCommitted = buffer.getLong64();
+            sequenceNumber = buffer.getLong64();
+        }
+
         // ignore gtid info read
         // sid.copy_from((uchar *)ptr_buffer);
         // ptr_buffer+= ENCODED_SID_LENGTH;
@@ -42,4 +65,26 @@ public class GtidLogEvent extends LogEvent {
         return commitFlag;
     }
 
+    public UUID getSid() {
+        return sid;
+    }
+
+    public long getGno() {
+        return gno;
+    }
+
+    public Long getLastCommitted() {
+        return lastCommitted;
+    }
+
+    public Long getSequenceNumber() {
+        return sequenceNumber;
+    }
+
+    public String getGtidStr() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(sid.toString()).append(":");
+        sb.append(gno);
+        return sb.toString();
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java

@@ -6,19 +6,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * <pre>
  *   Base class for ignorable log events. Events deriving from
-  this class can be safely ignored by slaves that cannot
+ *   this class can be safely ignored by slaves that cannot
-  recognize them. Newer slaves, will be able to read and
+ *   recognize them. Newer slaves, will be able to read and
-  handle them. This has been designed to be an open-ended
+ *   handle them. This has been designed to be an open-ended
-  architecture, so adding new derived events shall not harm
+ *   architecture, so adding new derived events shall not harm
-  the old slaves that support ignorable log event mechanism
+ *   the old slaves that support ignorable log event mechanism
-  (they will just ignore unrecognized ignorable events).
+ *   (they will just ignore unrecognized ignorable events).
-
+ * 
-  @note The only thing that makes an event ignorable is that it has
+ *   @note The only thing that makes an event ignorable is that it has
-  the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
+ *   the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
-  that ignorable event types derive from Ignorable_log_event; they may
+ *   that ignorable event types derive from Ignorable_log_event; they may
-  just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
+ *   just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
-  argument to the Log_event constructor.
+ *   argument to the Log_event constructor.
-  </pre>
+ * </pre>
  * 
  * @author jianghang 2013-4-8 上午12:36:29
  * @version 1.0.3

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java

@@ -5,12 +5,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 
 /**
  * Class representing an incident, an occurance out of the ordinary, that
- * happened on the master.
+ * happened on the master. The event is used to inform the slave that something
- * 
+ * out of the ordinary happened on the master that might cause the database to
- * The event is used to inform the slave that something out of the ordinary
+ * be in an inconsistent state.
- * happened on the master that might cause the database to be in an inconsistent
- * state.
- * 
  * <table id="IncidentFormat">
  * <caption>Incident event format</caption>
  * <tr>
@@ -38,8 +35,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class IncidentLogEvent extends LogEvent
+public final class IncidentLogEvent extends LogEvent {
-{
+
     public static final int INCIDENT_NONE        = 0;
 
     /** There are possibly lost events in the replication stream */
@@ -51,9 +48,7 @@ public final class IncidentLogEvent extends LogEvent
     private final int       incident;
     private final String    message;
 
-    public IncidentLogEvent(LogHeader header, LogBuffer buffer,
+    public IncidentLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -61,10 +56,9 @@ public final class IncidentLogEvent extends LogEvent
 
         buffer.position(commonHeaderLen);
         final int incidentNumber = buffer.getUint16();
-        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
+        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE) {
-        {
             // If the incident is not recognized, this binlog event is
-            // invalid.  If we set incident_number to INCIDENT_NONE, the
+            // invalid. If we set incident_number to INCIDENT_NONE, the
             // invalidity will be detected by is_valid().
             incident = INCIDENT_NONE;
             message = null;
@@ -76,13 +70,11 @@ public final class IncidentLogEvent extends LogEvent
         message = buffer.getString();
     }
 
-    public final int getIncident()
+    public final int getIncident() {
-    {
         return incident;
     }
 
-    public final String getMessage()
+    public final String getMessage() {
-    {
         return message;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java

@@ -6,54 +6,43 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 /**
  * An Intvar_log_event will be created just before a Query_log_event, if the
  * query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
- * Intvar_log_event holds the value of one of these variables.
+ * Intvar_log_event holds the value of one of these variables. Binary Format The
- * 
+ * Post-Header for this event type is empty. The Body has two components:
- * Binary Format
- * 
- * The Post-Header for this event type is empty. The Body has two components:
- * 
  * <table>
  * <caption>Body for Intvar_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>type</td>
  * <td>1 byte enumeration</td>
  * <td>One byte identifying the type of variable stored. Currently, two
  * identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
  * </tr>
- * 
  * <tr>
  * <td>value</td>
  * <td>8 byte unsigned integer</td>
  * <td>The value of the variable.</td>
  * </tr>
- * 
  * </table>
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public final class IntvarLogEvent extends LogEvent
+public final class IntvarLogEvent extends LogEvent {
-{
+
     /**
      * Fixed data part: Empty
-     * 
      * <p>
      * Variable data part:
-     * 
      * <ul>
      * <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
      * 1 or INSERT_ID_EVENT = 2.</li>
      * <li>8 bytes. An unsigned integer indicating the value to be used for the
      * LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
      * </ul>
-     * 
      * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
      */
     private final long      value;
@@ -68,31 +57,25 @@ public final class IntvarLogEvent extends LogEvent
     public static final int LAST_INSERT_ID_EVENT = 1;
     public static final int INSERT_ID_EVENT      = 2;
 
-    public IntvarLogEvent(LogHeader header, LogBuffer buffer,
+    public IntvarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         /* The Post-Header is empty. The Varible Data part begins immediately. */
-        buffer.position(descriptionEvent.commonHeaderLen
+        buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
-                + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+                        + I_TYPE_OFFSET);
-                + I_TYPE_OFFSET);
         type = buffer.getInt8(); // I_TYPE_OFFSET
         value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
     }
 
-    public final int getType()
+    public final int getType() {
-    {
         return type;
     }
 
-    public final long getValue()
+    public final long getValue() {
-    {
         return value;
     }
 
-    public final String getQuery()
+    public final String getQuery() {
-    {
         return "SET INSERT_ID = " + value;
     }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java

@@ -26,16 +26,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * </pre>
  * 
  * Binary Format: The Post-Header consists of the following six components.
- * 
  * <table>
  * <caption>Post-Header for Load_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>slave_proxy_id</td>
  * <td>4 byte unsigned integer</td>
@@ -45,71 +42,57 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * temporary table local to the client. The slave_proxy_id is used to
  * distinguish temporary tables that belong to different clients.</td>
  * </tr>
- * 
  * <tr>
  * <td>exec_time</td>
  * <td>4 byte unsigned integer</td>
  * <td>The time from when the query started to when it was logged in the binlog,
  * in seconds.</td>
  * </tr>
- * 
  * <tr>
  * <td>skip_lines</td>
  * <td>4 byte unsigned integer</td>
  * <td>The number on line (14) above, if present, or 0 if line (14) is left out.
  * </td>
  * </tr>
- * 
  * <tr>
  * <td>table_name_len</td>
  * <td>1 byte unsigned integer</td>
  * <td>The length of 'table_name' on line (4) above.</td>
  * </tr>
- * 
  * <tr>
  * <td>db_len</td>
  * <td>1 byte unsigned integer</td>
  * <td>The length of 'db' on line (1) above.</td>
  * </tr>
- * 
  * <tr>
  * <td>num_fields</td>
  * <td>4 byte unsigned integer</td>
  * <td>The number n of fields on line (15) above.</td>
  * </tr>
  * </table>
- * 
  * The Body contains the following components.
- * 
  * <table>
  * <caption>Body of Load_log_event</caption>
- * 
  * <tr>
  * <th>Name</th>
  * <th>Format</th>
  * <th>Description</th>
  * </tr>
- * 
  * <tr>
  * <td>sql_ex</td>
  * <td>variable length</td>
- * 
  * <td>Describes the part of the query on lines (3) and (5)&ndash;(13) above.
  * More precisely, it stores the five strings (on lines) field_term (6),
  * enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
  * bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
- * OPTIONALLY (7).
+ * OPTIONALLY (7). The data is stored in one of two formats, called "old" and
- * 
+ * "new". The type field of Common-Header determines which of these two formats
- * The data is stored in one of two formats, called "old" and "new". The type
+ * is used: type LOAD_EVENT means that the old format is used, and type
- * field of Common-Header determines which of these two formats is used: type
+ * NEW_LOAD_EVENT means that the new format is used. When MySQL writes a
- * LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
+ * Load_log_event, it uses the new format if at least one of the five strings is
- * that the new format is used. When MySQL writes a Load_log_event, it uses the
+ * two or more bytes long. Otherwise (i.e., if all strings are 0 or 1 bytes
- * new format if at least one of the five strings is two or more bytes long.
+ * long), the old format is used. The new and old format differ in the way the
- * Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
+ * five strings are stored.
- * used.
- * 
- * The new and old format differ in the way the five strings are stored.
- * 
  * <ul>
  * <li>In the new format, the strings are stored in the order field_term,
  * enclosed, escaped, line_term, line_start. Each string consists of a length (1
@@ -117,7 +100,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
  * IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
  * the presence of the corresponding keyword in the SQL query.
- * 
  * <li>In the old format, we know that each string has length 0 or 1. Therefore,
  * only the first byte of each string is stored. The order of the strings is the
  * same as in the new format. These five bytes are followed by the same 1 byte
@@ -128,18 +110,15 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
  * LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
  * </ul>
- * 
  * Thus, the size of the new format is 6 bytes + the sum of the sizes of the
  * five strings. The size of the old format is always 7 bytes.</td>
  * </tr>
- * 
  * <tr>
  * <td>field_lens</td>
  * <td>num_fields 1 byte unsigned integers</td>
  * <td>An array of num_fields integers representing the length of each field in
  * the query. (num_fields is from the Post-Header).</td>
  * </tr>
- * 
  * <tr>
  * <td>fields</td>
  * <td>num_fields null-terminated strings</td>
@@ -148,7 +127,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * the num_fields array.) The total length of all strings equals to the sum of
  * all field_lens, plus num_fields bytes for all the trailing zeros.</td>
  * </tr>
- * 
  * <tr>
  * <td>table_name</td>
  * <td>null-terminated string of length table_len+1 bytes</td>
@@ -156,31 +134,27 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
  * trailing zero is actually redundant since the table_len is known from
  * Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>db</td>
  * <td>null-terminated string of length db_len+1 bytes</td>
  * <td>The 'db' from the query, as a null-terminated string. (The trailing zero
  * is actually redundant since the db_len is known from Post-Header.)</td>
  * </tr>
- * 
  * <tr>
  * <td>file_name</td>
  * <td>variable length string without trailing zero, extending to the end of the
  * event (determined by the length field of the Common-Header)</td>
  * <td>The 'file_name' from the query.</td>
  * </tr>
- * 
  * </table>
- * 
  * This event type is understood by current versions, but only generated by
  * MySQL 3.23 and earlier.
  * 
  * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
  * @version 1.0
  */
-public class LoadLogEvent extends LogEvent
+public class LoadLogEvent extends LogEvent {
-{
+
     private String          table;
     private String          db;
     private String          fname;
@@ -210,11 +184,10 @@ public class LoadLogEvent extends LogEvent
     public static final int L_DATA_OFFSET       = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
 
     /*
-      These are flags and structs to handle all the LOAD DATA INFILE options (LINES
+     * These are flags and structs to handle all the LOAD DATA INFILE options
-      TERMINATED etc).
+     * (LINES TERMINATED etc). DUMPFILE_FLAG is probably useless (DUMPFILE is a
-      DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
+     * clause of SELECT, not of LOAD DATA).
-      DATA).
+     */
-    */
     public static final int DUMPFILE_FLAG       = 0x1;
     public static final int OPT_ENCLOSED_FLAG   = 0x2;
     public static final int REPLACE_FLAG        = 0x4;
@@ -226,9 +199,7 @@ public class LoadLogEvent extends LogEvent
     public static final int LINE_START_EMPTY    = 0x8;
     public static final int ESCAPED_EMPTY       = 0x10;
 
-    public LoadLogEvent(LogHeader header, LogBuffer buffer,
+    public LoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         super(header);
 
         final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
@@ -236,18 +207,16 @@ public class LoadLogEvent extends LogEvent
          * I (Guilhem) manually tested replication of LOAD DATA INFILE for
          * 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
          */
-        copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
+        copyLogEvent(buffer,
-                + descriptionEvent.commonHeaderLen : loadHeaderLen
+            ((header.type == LOAD_EVENT) ? loadHeaderLen + descriptionEvent.commonHeaderLen : loadHeaderLen
-                + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
+                                                                                              + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
-                descriptionEvent);
+            descriptionEvent);
     }
 
     /**
      * @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
      */
-    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
+    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset, FormatDescriptionLogEvent descriptionEvent) {
-            FormatDescriptionLogEvent descriptionEvent)
-    {
         /* this is the beginning of the post-header */
         buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
 
@@ -262,8 +231,7 @@ public class LoadLogEvent extends LogEvent
          * Sql_ex.init() on success returns the pointer to the first byte after
          * the sql_ex structure, which is the start of field lengths array.
          */
-        if (header.type != LOAD_EVENT /* use_new_format */)
+        if (header.type != LOAD_EVENT /* use_new_format */) {
-        {
             /*
              * The code below assumes that buf will not disappear from under our
              * feet during the lifetime of the event. This assumption holds true
@@ -279,9 +247,7 @@ public class LoadLogEvent extends LogEvent
             escaped = buffer.getString();
             optFlags = buffer.getInt8();
             emptyFlags = 0;
-        }
+        } else {
-        else
-        {
             fieldTerm = buffer.getFixString(1);
             enclosed = buffer.getFixString(1);
             lineTerm = buffer.getFixString(1);
@@ -290,23 +256,17 @@ public class LoadLogEvent extends LogEvent
             optFlags = buffer.getUint8();
             emptyFlags = buffer.getUint8();
 
-            if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
+            if ((emptyFlags & FIELD_TERM_EMPTY) != 0) fieldTerm = null;
-                fieldTerm = null;
+            if ((emptyFlags & ENCLOSED_EMPTY) != 0) enclosed = null;
-            if ((emptyFlags & ENCLOSED_EMPTY) != 0)
+            if ((emptyFlags & LINE_TERM_EMPTY) != 0) lineTerm = null;
-                enclosed = null;
+            if ((emptyFlags & LINE_START_EMPTY) != 0) lineStart = null;
-            if ((emptyFlags & LINE_TERM_EMPTY) != 0)
+            if ((emptyFlags & ESCAPED_EMPTY) != 0) escaped = null;
-                lineTerm = null;
-            if ((emptyFlags & LINE_START_EMPTY) != 0)
-                lineStart = null;
-            if ((emptyFlags & ESCAPED_EMPTY) != 0)
-                escaped = null;
         }
 
         final int fieldLenPos = buffer.position();
         buffer.forward(numFields);
         fields = new String[numFields];
-        for (int i = 0; i < numFields; i++)
+        for (int i = 0; i < numFields; i++) {
-        {
             final int fieldLen = buffer.getUint8(fieldLenPos + i);
             fields[i] = buffer.getFixString(fieldLen + 1);
         }
@@ -314,78 +274,65 @@ public class LoadLogEvent extends LogEvent
         table = buffer.getFixString(tableNameLen + 1);
         db = buffer.getFixString(dbLen + 1);
 
-        // null termination is accomplished by the caller 
+        // null termination is accomplished by the caller
         final int from = buffer.position();
         final int end = from + buffer.limit();
         int found = from;
         for (; (found < end) && buffer.getInt8(found) != '\0'; found++)
             /* empty loop */;
         fname = buffer.getString(found);
-        buffer.forward(1); // The + 1 is for \0 terminating fname 
+        buffer.forward(1); // The + 1 is for \0 terminating fname
     }
 
-    public final String getTable()
+    public final String getTable() {
-    {
         return table;
     }
 
-    public final String getDb()
+    public final String getDb() {
-    {
         return db;
     }
 
-    public final String getFname()
+    public final String getFname() {
-    {
         return fname;
     }
 
-    public final int getSkipLines()
+    public final int getSkipLines() {
-    {
         return skipLines;
     }
 
-    public final String[] getFields()
+    public final String[] getFields() {
-    {
         return fields;
     }
 
-    public final String getFieldTerm()
+    public final String getFieldTerm() {
-    {
         return fieldTerm;
     }
 
-    public final String getLineTerm()
+    public final String getLineTerm() {
-    {
         return lineTerm;
     }
 
-    public final String getLineStart()
+    public final String getLineStart() {
-    {
         return lineStart;
     }
 
-    public final String getEnclosed()
+    public final String getEnclosed() {
-    {
         return enclosed;
     }
 
-    public final String getEscaped()
+    public final String getEscaped() {
-    {
         return escaped;
     }
 
-    public final int getOptFlags()
+    public final int getOptFlags() {
-    {
         return optFlags;
     }
 
-    public final int getEmptyFlags()
+    public final int getEmptyFlags() {
-    {
         return emptyFlags;
     }
 
-    public final long getExecTime()
+    public final long getExecTime() {
-    {
         return execTime;
     }
 }