Merge canal repo master

README.md

@@ -1,14 +1,6 @@
 <div class="blog_content">
     <div class="iteye-blog-content-contain">
     	
-<h3>最新更新</h3>
-<ol>
-<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
-<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
-<li>Canal已在阿里云推出商业化版本 <a href="https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7">数据传输服务DTS</a>， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。<a href="https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E">DTS产品使用文档</a></li>
-DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，<a href="https://common-buy.aliyun.com/?commodityCode=dtspre&request=%7b%22dts_function%22%3a%22data_subscribe%22%7d">立即体验>>></a>
-</ol>
-
 <h1>背景</h1>
 <p style="font-size: 14px;">   早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&amp;消费的业务，从此开启了一段新纪元。</p>
 <p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.7及以下的版本(阿里内部mysql 5.7.13, 5.6.10, mysql 5.5.18和5.1.40/48)</p>
@@ -46,24 +38,54 @@ DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体
 <li>canal解析binary log对象(原始为byte流)</li>
 </ol>
 
+<h1>重要版本更新说明</h1>
+
+canal 1.1.x系列，参考release文档：<a href="https://github.com/alibaba/canal/releases">版本发布信息</a>
+
+1. 整体性能测试&优化,提升了150%. #726 参考: 【[Performance](https://github.com/alibaba/canal/wiki/Performance)】
+2. 原生支持prometheus监控 #765 【[Prometheus QuickStart](https://github.com/alibaba/canal/wiki/Prometheus-QuickStart)】
+3. 原生支持kafka消息投递 #695 【[Canal Kafka QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-QuickStart)】
+4. 原生支持aliyun rds的binlog订阅 (解决自动主备切换/oss binlog离线解析) 参考: 【[Aliyun RDS QuickStart](https://github.com/alibaba/canal/wiki/aliyun-RDS-QuickStart)】
+5. 原生支持docker镜像 #801 参考:  【[Docker QuickStart](https://github.com/alibaba/canal/wiki/Docker-QuickStart)】
+
 <h1>相关文档</h1>
 
 See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文档</a>
 
 <h3><a name="table-of-contents" class="anchor" href="#table-of-contents"><span class="mini-icon mini-icon-link"></span></a>wiki文档列表</h3>
 <ul>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Home">Home</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/Home">Home</a></li>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Introduction">Introduction</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/Introduction">Introduction</a></li>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/QuickStart">QuickStart</a></li>
+<li>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientExample">ClientExample</a></li>
+<a class="internal present" href="/alibaba/canal/wiki/QuickStart">QuickStart</a>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/AdminGuide">AdminGuide</a></li>
+<ul>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientAPI">ClientAPI</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/Docker-QuickStart">Docker QuickStart</a></li>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/DevGuide">DevGuide</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/Canal-Kafka-QuickStart">Canal Kafka QuickStart</a></li>
-<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/BinlogChange%28mysql5.6%29">BinlogChange(Mysql5.6)</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/aliyun-RDS-QuickStart">Aliyun RDS QuickStart</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/Prometheus-QuickStart">Prometheus QuickStart</a></li>
+</ul>
+</li>
+<li><a class="internal present" href="/alibaba/canal/wiki/AdminGuide">AdminGuide</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/ClientExample">ClientExample</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/ClientAPI">ClientAPI</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/Performance">Performance</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/DevGuide">DevGuide</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/BinlogChange%28mysql5.6%29">BinlogChange(Mysql5.6)</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/BinlogChange%28MariaDB%29">BinlogChange(MariaDB)</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/TableMetaTSDB">TableMetaTSDB</a></li>
 <li><a href="http://alibaba.github.com/canal/release.html">ReleaseNotes</a></li>
 <li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
+<li><a class="internal present" href="/alibaba/canal/wiki/FAQ">FAQ</a></li>
 </ul>
 
+<h1>多语言业务</h1>
+
+1. canal整体交互协议设计上使用了protobuf3.0，理论上可以支持绝大部分的多语言场景，欢迎大家提交多客户端的PR
+    * canal java客户端: <a href="https://github.com/alibaba/canal/wiki/ClientExample"> https://github.com/alibaba/canal/wiki/ClientExample </a>
+    * canal c#客户端开源项目地址：<a href="https://github.com/CanalSharp/CanalSharp"> https://github.com/CanalSharp/CanalSharp </a>
+    * canal go客户端，开发进行中
+2. canal作为MySQL binlog的增量获取工具，可以将数据投递到MQ系统中，比如Kafka/RocketMQ，可以借助于MQ的多语言能力 
+
 <h1>相关资料</h1>
 
 * ADC阿里技术嘉年华分享ppt (放在google docs上，可能需要翻墙): <a href="https://docs.google.com/presentation/d/1MkszUPYRDkfVPz9IqOT1LLT5d9tuwde_WC8GZvjaDRg/edit?usp=sharing">ppt下载</href>  
@@ -91,9 +113,13 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
 <li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
 </ol>
 
-<pre>
+<h3>最新更新</h3>
-【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
+<ol>
-岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
+<li>canal发布重大版本更新1.1.0，具体releaseNode参考：<a href="https://github.com/alibaba/canal/releases/tag/canal-1.1.0">https://github.com/alibaba/canal/releases/tag/canal-1.1.0</a></li>
-工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊. 
+<li>canal c#客户端开源项目地址：<a href="https://github.com/CanalSharp/CanalSharp"> https://github.com/CanalSharp/CanalSharp </a>，推荐! </li>
-具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
+<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
-</pre>
+<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
+
+<li>Canal已在阿里云推出商业化版本 <a href="https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7">数据传输服务DTS</a>， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。<a href="https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E">DTS产品使用文档</a></li>
+DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，<a href="https://common-buy.aliyun.com/?commodityCode=dtspre&request=%7b%22dts_function%22%3a%22data_subscribe%22%7d">立即体验>>></a>
+</ol>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java

@@ -1,7 +1,9 @@
 package com.alibaba.otter.canal.client.adapter;
 
 import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -23,9 +25,11 @@ public interface CanalOuterAdapter {
     /**
      * 往适配器中写入数据
      *
-     * @param message message数据包
+     * @param dml 数据包
      */
-    void writeOut(Message message);
+    void writeOut(Dml dml);
+
+    // void writeOut(FlatMessage flatMessage);
 
     /**
      * 外部适配器销毁接口

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -22,6 +22,9 @@ public class CanalClientConfig {
 
     private List<MQTopic>       mqTopics;
 
+    private Boolean             flatMessage = true;
+
+
     private List<CanalInstance> canalInstances;
 
     public String getCanalServerHost() {
@@ -60,6 +63,14 @@ public class CanalClientConfig {
         return mqTopics;
     }
 
+    public Boolean getFlatMessage() {
+        return flatMessage;
+    }
+
+    public void setFlatMessage(Boolean flatMessage) {
+        this.flatMessage = flatMessage;
+    }
+
     public void setMqTopics(List<MQTopic> mqTopics) {
         this.mqTopics = mqTopics;
     }
@@ -166,25 +177,6 @@ public class CanalClientConfig {
             this.outAdapters = outAdapters;
         }
 
-        // public List<Adaptor> getAdapters() {
-        // return adapters;
-        // }
-        //
-        // public void setAdapters(List<Adaptor> adapters) {
-        // this.adapters = adapters;
-        // }
     }
 
-    // public static class Adaptor {
-    // private List<CanalOuterAdapterConfiguration> outAdapters;
-    //
-    // public List<CanalOuterAdapterConfiguration> getOutAdapters() {
-    // return outAdapters;
-    // }
-    //
-    // public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters)
-    // {
-    // this.outAdapters = outAdapters;
-    // }
-    // }
 }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java

@@ -21,7 +21,7 @@ public class JdbcTypeUtil {
 
     private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
 
-    public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
+    public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) {
         if (value == null || value.equals("")) {
             return null;
         }
@@ -96,7 +96,7 @@ public class JdbcTypeUtil {
             }
             return res;
         } catch (Exception e) {
-            logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
+            logger.error("table: {} column: {}, failed convert type {} to {}", columnName, value, sqlType);
             return value;
         }
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
 import java.util.*;
 
 import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -63,8 +64,7 @@ public class MessageUtil {
 
                     for (CanalEntry.Column column : columns) {
                         row.put(column.getName(),
-                            JdbcTypeUtil.typeConvert(dml.getTable(),
+                            JdbcTypeUtil.typeConvert(column.getName(),
-                                column.getName(),
                                 column.getValue(),
                                 column.getSqlType(),
                                 column.getMysqlType()));
@@ -82,8 +82,7 @@ public class MessageUtil {
                         for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
                             if (updateSet.contains(column.getName())) {
                                 rowOld.put(column.getName(),
-                                    JdbcTypeUtil.typeConvert(dml.getTable(),
+                                    JdbcTypeUtil.typeConvert(column.getName(),
-                                        column.getName(),
                                         column.getValue(),
                                         column.getSqlType(),
                                         column.getMysqlType()));
@@ -106,6 +105,57 @@ public class MessageUtil {
         }
     }
 
+    public static Dml flatMessage2Dml(FlatMessage flatMessage) {
+        if (flatMessage == null) {
+            return null;
+        }
+        Dml dml = new Dml();
+        dml.setDatabase(flatMessage.getDatabase());
+        dml.setTable(flatMessage.getTable());
+        dml.setType(flatMessage.getType());
+        dml.setTs(flatMessage.getTs());
+        dml.setSql(flatMessage.getSql());
+        if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
+            throw new RuntimeException("SqlType or mysqlType is null");
+        }
+        List<Map<String, String>> data = flatMessage.getData();
+        if (data != null) {
+            dml.setData(changeRows(data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
+        }
+        List<Map<String, String>> old = flatMessage.getOld();
+        if (old != null) {
+            dml.setOld(changeRows(old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
+        }
+        return dml;
+    }
+
+    private static List<Map<String, Object>> changeRows(List<Map<String, String>> rows, Map<String, Integer> sqlTypes,
+                                                        Map<String, String> mysqlTypes) {
+        List<Map<String, Object>> result = new ArrayList<>();
+        for (Map<String, String> row : rows) {
+            Map<String, Object> resultRow = new LinkedHashMap<>();
+            for (Map.Entry<String, String> entry : row.entrySet()) {
+                String columnName = entry.getKey();
+                String columnValue = entry.getValue();
+
+                Integer sqlType = sqlTypes.get(columnName);
+                if (sqlType == null) {
+                    continue;
+                }
+
+                String mysqlType = mysqlTypes.get(columnName);
+                if (mysqlType == null) {
+                    continue;
+                }
+
+                Object finalValue = JdbcTypeUtil.typeConvert(columnName, columnValue, sqlType, mysqlType);
+                resultRow.put(columnName, finalValue);
+            }
+            result.add(resultRow);
+        }
+        return result;
+    }
+
     public interface Consumer<T> {
 
         void accept(T t);

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -86,20 +87,14 @@ public class HbaseAdapter implements CanalOuterAdapter {
     }
 
     @Override
-    public void writeOut(Message message) {
+    public void writeOut(Dml dml) {
-        MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
+        if (dml == null) {
-
+            return;
-            @Override
+        }
-            public void accept(Dml dml) {
+        String database = dml.getDatabase();
-                if (dml == null) {
+        String table = dml.getTable();
-                    return;
+        MappingConfig config = mappingConfigCache.get(database + "-" + table);
-                }
+        hbaseSyncService.sync(config, dml);
-                String database = dml.getDatabase();
-                String table = dml.getTable();
-                MappingConfig config = mappingConfigCache.get(database + "-" + table);
-                hbaseSyncService.sync(config, dml);
-            }
-        });
     }
 
     @Override

client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java

@@ -6,9 +6,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
 import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
 import com.alibaba.otter.canal.client.adapter.support.SPI;
-import com.alibaba.otter.canal.protocol.Message;
 
 /**
  * 外部适配器示例
@@ -22,20 +20,13 @@ public class LoggerAdapterExample implements CanalOuterAdapter {
     private Logger logger = LoggerFactory.getLogger(this.getClass());
 
     @Override
-    public void writeOut(Message message) {
+    public void init(CanalOuterAdapterConfiguration configuration) {
-        // 直接输出日志信息
-        MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
 
-            @Override
-            public void accept(Dml dml) {
-                logger.info(dml.toString());
-            }
-        });
     }
 
     @Override
-    public void init(CanalOuterAdapterConfiguration configuration) {
+    public void writeOut(Dml dml) {
-
+        logger.info(dml.toString());
     }
 
     @Override

client-launcher/src/main/bin/startup.sh

@@ -74,7 +74,7 @@ else
 fi
 
 JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
-CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
+CANAL_OPTS="-DappName=otter-canal-client -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
 
 if [ -e $canal_conf -a -e $logback_configurationFile ]
 then 

client-launcher/src/main/bin/stop.sh

@@ -45,7 +45,7 @@ fi
 
 pid=`cat $pidfile`
 if [ "$pid" == "" ] ; then
-	pid=`get_pid "appName=otter-canal"`
+	pid=`get_pid "appName=otter-canal-client"`
 fi
 
 echo -e "`hostname`: stopping canal $pid ... "
@@ -54,7 +54,7 @@ kill $pid
 LOOPS=0
 while (true); 
 do 
-	gpid=`get_pid "appName=otter-canal" "$pid"`
+	gpid=`get_pid "appName=otter-canal-client" "$pid"`
     if [ "$gpid" == "" ] ; then
     	echo "Oook! cost:$LOOPS"
     	`rm $pidfile`

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/AbstractCanalAdapterWorker.java

@@ -7,10 +7,13 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
-import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -47,9 +50,16 @@ public abstract class AbstractCanalAdapterWorker {
                 public Boolean call() {
                     try {
                         // 组内适配器穿行运行，尽量不要配置组内适配器
-                        for (CanalOuterAdapter c : adapters) {
+                        for (final CanalOuterAdapter c : adapters) {
                             long begin = System.currentTimeMillis();
-                            c.writeOut(message);
+                            MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
+
+                                @Override
+                                public void accept(Dml dml) {
+                                    c.writeOut(dml);
+                                }
+                            });
+
                             if (logger.isDebugEnabled()) {
                                 logger.debug("{} elapsed time: {}",
                                     c.getClass().getName(),
@@ -71,7 +81,49 @@ public abstract class AbstractCanalAdapterWorker {
                         logger.error("Outer adapter write failed");
                     }
                 } catch (InterruptedException | ExecutionException e) {
-                    logger.error(e.getMessage(), e);
+                    // ignore
+                }
+            }
+        }
+    }
+
+    protected void writeOut(final FlatMessage flatMessage) {
+        List<Future<Boolean>> futures = new ArrayList<>();
+        // 组间适配器并行运行
+        for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
+            final List<CanalOuterAdapter> adapters = outerAdapters;
+            futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
+
+                @Override
+                public Boolean call() {
+                    try {
+                        // 组内适配器穿行运行，尽量不要配置组内适配器
+                        for (CanalOuterAdapter c : adapters) {
+                            long begin = System.currentTimeMillis();
+                            Dml dml = MessageUtil.flatMessage2Dml(flatMessage);
+                            c.writeOut(dml);
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("{} elapsed time: {}",
+                                    c.getClass().getName(),
+                                    (System.currentTimeMillis() - begin));
+                            }
+                        }
+                        return true;
+                    } catch (Exception e) {
+                        return false;
+                    }
+                }
+            }));
+
+            // 等待所有适配器写入完成
+            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
+            for (Future<Boolean> f : futures) {
+                try {
+                    if (!f.get()) {
+                        logger.error("Outer adapter write failed");
+                    }
+                } catch (InterruptedException | ExecutionException e) {
+                    // ignore
                 }
             }
         }

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterKafkaWorker.java

@@ -12,6 +12,7 @@ import org.apache.kafka.common.errors.WakeupException;
 import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
 import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
 import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
+import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
 
 /**
@@ -24,16 +25,19 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
 
     private KafkaCanalConnector connector;
 
-    private String topic;
+    private String              topic;
 
-    public CanalAdapterKafkaWorker(String zkServers, String bootstrapServers, String topic, String groupId,
+    private boolean             flatMessage;
-        List<List<CanalOuterAdapter>> canalOuterAdapters) {
+
+    public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
+        List<List<CanalOuterAdapter>> canalOuterAdapters, boolean flatMessage){
         this.canalOuterAdapters = canalOuterAdapters;
         this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
         this.topic = topic;
         this.canalDestination = topic;
-        connector = KafkaCanalConnectors.newKafkaConnector(zkServers, bootstrapServers, topic, null, groupId);
+        this.flatMessage = flatMessage;
-        // connector.setSessionTimeout(5L, TimeUnit.MINUTES);
+        connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
+        // connector.setSessionTimeout(1L, TimeUnit.MINUTES);
 
         // super.initSwitcher(topic);
     }
@@ -60,11 +64,29 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
             if (!running) {
                 return;
             }
+
             connector.stopRunning();
             running = false;
-            logger.info("Stop topic {} out adapters begin", this.topic);
+
-            stopOutAdapters();
+            // if (switcher != null && !switcher.state()) {
-            logger.info("Stop topic {} out adapters end", this.topic);
+            // switcher.set(true);
+            // }
+
+            if (thread != null) {
+                try {
+                    thread.join();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            groupInnerExecutorService.shutdown();
+            logger.info("topic {} connectors' worker thread dead!", this.topic);
+            for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
+                for (CanalOuterAdapter adapter : outerAdapters) {
+                    adapter.destroy();
+                }
+            }
+            logger.info("topic {} all connectors destroyed!", this.topic);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }
@@ -79,39 +101,54 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
             try {
                 logger.info("=============> Start to connect topic: {} <=============", this.topic);
                 connector.connect();
-                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
+                logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
                 connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
+                logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
                 while (running) {
                     try {
                         // switcher.get(); //等待开关开启
 
-                        final Message message = connector.getWithoutAck();
+                        List<?> messages;
+                        if (!flatMessage) {
+                            messages = connector.getWithoutAck();
+                        } else {
+                            messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS);
+                        }
+                        if (messages != null) {
+                            for (final Object message : messages) {
+                                executing.set(true);
+                                if (message != null) {
+                                    executor.submit(new Runnable() {
 
-                        executing.set(true);
+                                        @Override
-                        if (message != null) {
+                                        public void run() {
-                            executor.submit(new Runnable() {
+                                            try {
+                                                if (message instanceof FlatMessage) {
+                                                    writeOut((FlatMessage) message);
+                                                } else {
+                                                    writeOut((Message) message);
+                                                }
+                                            } catch (Exception e) {
+                                                logger.error(e.getMessage(), e);
+                                            } finally {
+                                                executing.compareAndSet(true, false);
+                                            }
+                                        }
+                                    });
 
-                                @Override
+                                    // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
-                                public void run() {
+                                    long currentTS = System.currentTimeMillis();
-                                    try {
+                                    while (executing.get()) {
-                                        writeOut(message, topic);
+                                        // 大于10秒未消费完ack一次keep alive
-                                    } catch (Exception e) {
+                                        if (System.currentTimeMillis() - currentTS > 10000) {
-                                        logger.error(e.getMessage(), e);
+                                            connector.ack();
-                                    } finally {
+                                            currentTS = System.currentTimeMillis();
-                                        executing.compareAndSet(true, false);
+                                        }
                                     }
                                 }
-                            });
-
-                            // 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
-                            while (executing.get()) {
-                                connector.ack();
-                                Thread.sleep(500);
                             }
-                        } else {
-                            connector.ack();
                         }
+                        connector.ack();
                     } catch (CommitFailedException e) {
                         logger.warn(e.getMessage());
                     } catch (Exception e) {
@@ -131,7 +168,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
         } catch (WakeupException e) {
             // No-op. Continue process
         }
-        connector.disconnnect();
+        connector.disconnect();
         logger.info("=============> Disconnect topic: {} <=============", this.topic);
     }
 }

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterLoader.java

@@ -4,10 +4,6 @@ import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
 import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
 import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
 import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.util.ArrayList;
@@ -16,6 +12,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * MQ外部适配器的加载器
@@ -94,7 +93,6 @@ public class CanalAdapterLoader {
                         loadConnector(config, canalOuterAdapters);
                     }
                     canalOuterAdapterGroups.add(canalOuterAdapters);
-
                     if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
                         CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(
                             canalClientConfig.getBootstrapServers(),
@@ -104,16 +102,17 @@ public class CanalAdapterLoader {
                         canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
                         rocketMQWorker.start();
                     } else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
-                        CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(zkHosts,
+                        CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(
                             canalClientConfig.getBootstrapServers(),
                             topic.getTopic(),
                             group.getGroupId(),
-                            canalOuterAdapterGroups);
+                            canalOuterAdapterGroups,canalClientConfig.getFlatMessage());
                         canalMQWorker.put(topic.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
                         canalKafkaWorker.start();
                     }
                     logger.info("Start adapter for canal-client rocketmq topic: {} succeed",
                         topic.getTopic() + "-" + group.getGroupId());
+
                 }
             }
         }

client-launcher/src/main/java/com/alibaba/otter/canal/client/adapter/loader/CanalAdapterWorker.java

@@ -1,16 +1,15 @@
 package com.alibaba.otter.canal.client.adapter.loader;
 
-import com.alibaba.otter.canal.client.CanalConnector;
-import com.alibaba.otter.canal.client.CanalConnectors;
-import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
-import com.alibaba.otter.canal.client.adapter.loader.AbstractCanalAdapterWorker;
-import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
-import com.alibaba.otter.canal.protocol.Message;
-
 import java.net.SocketAddress;
 import java.util.List;
 import java.util.concurrent.Executors;
 
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalConnectors;
+import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
+import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
+import com.alibaba.otter.canal.protocol.Message;
+
 /**
  * 原生canal-server对应的client适配器工作线程
  *

client-launcher/src/main/resources/canal-client.yml

@@ -1,6 +1,7 @@
-canalServerHost: 127.0.0.1:11111
+#canalServerHost: 127.0.0.1:11111
 #zookeeperHosts: slave1:2181
-#bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers
+bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers
+flatMessage: true
 
 #canalInstances:
 #- instance: example

client/pom.xml

@@ -1,115 +1,265 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
+	<modelVersion>4.0.0</modelVersion>
-    <modelVersion>4.0.0</modelVersion>
+	<parent>
-    <parent>
+		<groupId>com.alibaba.otter</groupId>
-        <groupId>com.alibaba.otter</groupId>
+		<artifactId>canal</artifactId>
-        <artifactId>canal</artifactId>
+		<version>1.1.1-SNAPSHOT</version>
-        <version>1.1.1-SNAPSHOT</version>
+		<relativePath>../pom.xml</relativePath>
-        <relativePath>../pom.xml</relativePath>
+	</parent>
-    </parent>
+	<groupId>com.alibaba.otter</groupId>
-    <groupId>com.alibaba.otter</groupId>
+	<artifactId>canal.client</artifactId>
-    <artifactId>canal.client</artifactId>
+	<packaging>jar</packaging>
-    <packaging>jar</packaging>
+	<name>canal client module for otter ${project.version}</name>
-    <name>canal client module for otter ${project.version}</name>
+	<dependencies>
-    <dependencies>
+		<dependency>
-        <dependency>
+			<groupId>com.alibaba.otter</groupId>
-            <groupId>com.alibaba.otter</groupId>
+			<artifactId>canal.protocol</artifactId>
-            <artifactId>canal.protocol</artifactId>
+			<version>${project.version}</version>
-            <version>${project.version}</version>
+			<optional>true</optional>
-        </dependency>
+		</dependency>
-        <!-- 客户端要使用请单独引入kafka依赖 -->
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-            <version>1.1.1</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.rocketmq</groupId>
-            <artifactId>rocketmq-client</artifactId>
-            <version>4.3.0</version>
-        </dependency>
-        <!-- junit -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-        </dependency>
-    </dependencies>
 
-    <profiles>
+		<dependency>
-        <profile>
+			<groupId>com.google.protobuf</groupId>
-            <id>dev</id>
+			<artifactId>protobuf-java</artifactId>
-            <activation>
+		</dependency>
-                <activeByDefault>true</activeByDefault>
-                <property>
-                    <name>env</name>
-                    <value>!javadoc</value>
-                </property>
-            </activation>
-        </profile>
 
-        <profile>
+		<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
-            <id>javadoc</id>
+		<dependency>
-            <activation>
+			<groupId>io.netty</groupId>
-                <property>
+			<artifactId>netty-all</artifactId>
-                    <name>env</name>
+		</dependency>
-                    <value>javadoc</value>
+		<!-- zk -->
-                </property>
+		<dependency>
-            </activation>
+			<groupId>org.apache.zookeeper</groupId>
-            <build>
+			<artifactId>zookeeper</artifactId>
-                <plugins>
+		</dependency>
-                    <plugin>
+		<dependency>
-                        <groupId>org.apache.maven.plugins</groupId>
+			<groupId>com.101tec</groupId>
-                        <artifactId>maven-javadoc-plugin</artifactId>
+			<artifactId>zkclient</artifactId>
-                        <version>2.9.1</version>
+		</dependency>
-                        <executions>
+		<!-- external -->
-                            <execution>
+		<dependency>
-                                <id>attach-javadocs</id>
+			<groupId>commons-io</groupId>
-                                <phase>package</phase>
+			<artifactId>commons-io</artifactId>
-                                <goals>
+		</dependency>
-                                    <goal>jar</goal>
+		<dependency>
-                                </goals>
+			<groupId>commons-lang</groupId>
-                            </execution>
+			<artifactId>commons-lang</artifactId>
-                        </executions>
+		</dependency>
-                        <configuration>
+		<dependency>
-                            <aggregate>true</aggregate>
+			<groupId>commons-codec</groupId>
-                            <show>public</show>
+			<artifactId>commons-codec</artifactId>
-                            <nohelp>true</nohelp>
+		</dependency>
-                            <header>${project.artifactId}-${project.version}</header>
+		<dependency>
-                            <footer>${project.artifactId}-${project.version}</footer>
+			<groupId>com.alibaba</groupId>
-                            <doctitle>${project.artifactId}-${project.version}</doctitle>
+			<artifactId>fastjson</artifactId>
-                            <links>
+		</dependency>
-                                <link>https://github.com/alibaba/canal</link>
+		<dependency>
-                            </links>
+			<groupId>com.google.guava</groupId>
-                            <outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}
+			<artifactId>guava</artifactId>
-                            </outputDirectory>
+		</dependency>
-                        </configuration>
+		<!-- log -->
-                    </plugin>
+		<dependency>
-                    <plugin>
+			<groupId>ch.qos.logback</groupId>
-                        <groupId>org.apache.maven.plugins</groupId>
+			<artifactId>logback-core</artifactId>
-                        <artifactId>maven-scm-publish-plugin</artifactId>
+		</dependency>
-                        <version>1.0-beta-2</version>
+		<dependency>
-                        <executions>
+			<groupId>ch.qos.logback</groupId>
-                            <execution>
+			<artifactId>logback-classic</artifactId>
-                                <id>attach-javadocs</id>
+		</dependency>
-                                <phase>package</phase>
+		<dependency>
-                                <goals>
+			<groupId>org.slf4j</groupId>
-                                    <goal>publish-scm</goal>
+			<artifactId>jcl-over-slf4j</artifactId>
-                                </goals>
+		</dependency>
-                            </execution>
+		<dependency>
-                        </executions>
+			<groupId>org.slf4j</groupId>
-                        <configuration>
+			<artifactId>slf4j-api</artifactId>
-                            <checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
+		</dependency>
-                            <checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}
+		<dependency>
-                            </checkinComment>
+			<groupId>org.springframework</groupId>
-                            <content>${project.build.directory}/apidocs</content>
+			<artifactId>spring-core</artifactId>
-                            <skipDeletedFiles>true</skipDeletedFiles>
+		</dependency>
-                            <pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
+		<dependency>
-                            <scmBranch>gh-pages</scmBranch>
+			<groupId>org.springframework</groupId>
-                        </configuration>
+			<artifactId>spring-aop</artifactId>
-                    </plugin>
+		</dependency>
-                </plugins>
+		<dependency>
-            </build>
+			<groupId>org.springframework</groupId>
-        </profile>
+			<artifactId>spring-context</artifactId>
-    </profiles>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-jdbc</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-orm</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-test</artifactId>
+			<version>${spring_version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.rocketmq</groupId>
+			<artifactId>rocketmq-client</artifactId>
+			<version>4.3.0</version>
+		</dependency>
+		<!-- 客户端要使用请单独引入kafka依赖 -->
+		<dependency>
+			<groupId>org.apache.kafka</groupId>
+			<artifactId>kafka-clients</artifactId>
+			<version>1.1.1</version>
+			<scope>provided</scope>
+		</dependency>
+
+		<!-- junit -->
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>2.4.1</version>
+				<configuration>
+					<createDependencyReducedPom>false</createDependencyReducedPom>
+					<artifactSet>
+						<excludes>
+							<exclude>org.apache.zookeeper:zookeeper</exclude>
+							<exclude>com.101tec:zkclient</exclude>
+							<exclude>commons-io:commons-io</exclude>
+							<exclude>commons-lang:commons-lang</exclude>
+							<exclude>commons-codec:commons-codec</exclude>
+							<exclude>com.alibaba:fastjson</exclude>
+							<exclude>ch.qos.logback:logback-core</exclude>
+							<exclude>ch.qos.logback:logback-classic</exclude>
+							<exclude>org.slf4j:jcl-over-slf4j</exclude>
+							<exclude>org.slf4j:slf4j-api</exclude>
+							<exclude>org.springframework:*</exclude>
+							<exclude>org.jboss.netty:netty</exclude>
+							<exclude>commons-logging:commons-logging</exclude>
+							<exclude>aopalliance:aopalliance</exclude>
+							<exclude>com.google.protobuf:protobuf-java</exclude>
+							<exclude>io.netty:*</exclude>
+							<exclude>junit:junit</exclude>
+						</excludes>
+					</artifactSet>
+				</configuration>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<relocations>
+								<relocation>
+									<pattern>com.google.guava</pattern>
+									<shadedPattern>com.alibaba.google.guava</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.joda</pattern>
+									<shadedPattern>com.alibaba.google.joda</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.google.common</pattern>
+									<shadedPattern>com.alibaba.google.common</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.google.thirdparty</pattern>
+									<shadedPattern>com.alibaba.google.thirdparty</shadedPattern>
+								</relocation>
+							</relocations>
+							<transformers>
+								<transformer
+										implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+							</transformers>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+
+	<profiles>
+		<profile>
+			<id>dev</id>
+			<activation>
+				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>env</name>
+					<value>!javadoc</value>
+				</property>
+			</activation>
+		</profile>
+
+		<profile>
+			<id>javadoc</id>
+			<activation>
+				<property>
+					<name>env</name>
+					<value>javadoc</value>
+				</property>
+			</activation>
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-javadoc-plugin</artifactId>
+						<version>2.9.1</version>
+						<executions>
+							<execution>
+								<id>attach-javadocs</id>
+								<phase>package</phase>
+								<goals>
+									<goal>jar</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<aggregate>true</aggregate>
+							<show>public</show>
+							<nohelp>true</nohelp>
+							<header>${project.artifactId}-${project.version}</header>
+							<footer>${project.artifactId}-${project.version}</footer>
+							<doctitle>${project.artifactId}-${project.version}</doctitle>
+							<links>
+								<link>https://github.com/alibaba/canal</link>
+							</links>
+							<outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}</outputDirectory>
+						</configuration>
+					</plugin>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-scm-publish-plugin</artifactId>
+						<version>1.0-beta-2</version>
+						<executions>
+							<execution>
+								<id>attach-javadocs</id>
+								<phase>package</phase>
+								<goals>
+									<goal>publish-scm</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
+							<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}</checkinComment>
+							<content>${project.build.directory}/apidocs</content>
+							<skipDeletedFiles>true</skipDeletedFiles>
+							<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
+							<scmBranch>gh-pages</scmBranch>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
 </project>

client/src/main/java/com/alibaba/otter/canal/client/CanalMessageDeserializer.java

@@ -8,8 +8,8 @@ import com.google.protobuf.ByteString;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DeserializerUtil {
+public class CanalMessageDeserializer {
-    private static Logger logger = LoggerFactory.getLogger(DeserializerUtil.class);
+    private static Logger logger = LoggerFactory.getLogger(CanalMessageDeserializer.class);
 
     public static Message deserializer(byte[] data) {
         try {
@@ -19,9 +19,9 @@ public class DeserializerUtil {
                 CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
                 switch (p.getType()) {
                     case MESSAGES: {
-                        if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
+//                        if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
-                            throw new CanalClientException("compression is not supported in this connector");
+//                            throw new CanalClientException("compression is not supported in this connector");
-                        }
+//                        }
 
                         CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
                         Message result = new Message(messages.getBatchId());

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java

@@ -159,7 +159,7 @@ public class SimpleCanalConnector implements CanalConnector {
             }
             //
             Handshake handshake = Handshake.parseFrom(p.getBody());
-            supportedCompressions.addAll(handshake.getSupportedCompressionsList());
+            supportedCompressions.add(handshake.getSupportedCompressions());
             //
             ClientAuth ca = ClientAuth.newBuilder()
                 .setUsername(username != null ? username : "")
@@ -180,8 +180,8 @@ public class SimpleCanalConnector implements CanalConnector {
 
             Ack ackBody = Ack.parseFrom(ack.getBody());
             if (ackBody.getErrorCode() > 0) {
-                throw new CanalClientException("something goes wrong when doing authentication: "
+                throw new CanalClientException(
-                                               + ackBody.getErrorMessage());
+                    "something goes wrong when doing authentication: " + ackBody.getErrorMessage());
             }
 
             connected = true;
@@ -323,7 +323,8 @@ public class SimpleCanalConnector implements CanalConnector {
         Packet p = Packet.parseFrom(data);
         switch (p.getType()) {
             case MESSAGES: {
-                if (!p.getCompression().equals(Compression.NONE)) {
+                if (!p.getCompression().equals(Compression.NONE)
+                    && !p.getCompression().equals(Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
                     throw new CanalClientException("compression is not supported in this connector");
                 }
 
@@ -360,11 +361,8 @@ public class SimpleCanalConnector implements CanalConnector {
             .setBatchId(batchId)
             .build();
         try {
-            writeWithHeader(Packet.newBuilder()
+            writeWithHeader(
-                .setType(PacketType.CLIENTACK)
+                Packet.newBuilder().setType(PacketType.CLIENTACK).setBody(ca.toByteString()).build().toByteArray());
-                .setBody(ca.toByteString())
-                .build()
-                .toByteArray());
         } catch (IOException e) {
             throw new CanalClientException(e);
         }

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java

@@ -1,19 +1,20 @@
 package com.alibaba.otter.canal.client.kafka;
 
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.ConsumerRecords;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.serialization.StringDeserializer;
 
-import com.alibaba.otter.canal.client.kafka.running.ClientRunningData;
+import com.alibaba.fastjson.JSON;
-import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.protocol.FlatMessage;
-import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
 import com.alibaba.otter.canal.protocol.Message;
-import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 
 /**
  * canal kafka 数据操作客户端
@@ -24,18 +25,19 @@ import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 public class KafkaCanalConnector {
 
     private KafkaConsumer<String, Message> kafkaConsumer;
+    private KafkaConsumer<String, String>  kafkaConsumer2;   // 用于扁平message的数据消费
     private String                         topic;
     private Integer                        partition;
     private Properties                     properties;
-    // private ClientRunningMonitor runningMonitor; // 运行控制
-    // private BooleanMutex mutex = new BooleanMutex(false);
-    private ZkClientx                      zkClientx;
     private volatile boolean               connected = false;
     private volatile boolean               running   = false;
+    private boolean                        flatMessage;
 
-    public KafkaCanalConnector(String zkServers, String servers, String topic, Integer partition, String groupId){
+    public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId,
+                               boolean flatMessage){
         this.topic = topic;
         this.partition = partition;
+        this.flatMessage = flatMessage;
 
         properties = new Properties();
         properties.put("bootstrap.servers", servers);
@@ -45,32 +47,13 @@ public class KafkaCanalConnector {
         properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
         properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
         properties.put("session.timeout.ms", "30000"); // 默认为30秒
-        properties.put("max.poll.records", "1"); // 所以一次只取一条数据
+        properties.put("max.poll.records", "100");
         properties.put("key.deserializer", StringDeserializer.class.getName());
-        properties.put("value.deserializer", MessageDeserializer.class.getName());
+        if (!flatMessage) {
-
+            properties.put("value.deserializer", MessageDeserializer.class.getName());
-        if (zkServers != null) {
+        } else {
-            zkClientx = new ZkClientx(zkServers);
+            properties.put("value.deserializer", StringDeserializer.class.getName());
-
-            ClientRunningData clientData = new ClientRunningData();
-            clientData.setGroupId(groupId);
-            clientData.setAddress(AddressUtils.getHostIp());
-
-            // runningMonitor = new ClientRunningMonitor();
-            // runningMonitor.setTopic(topic);
-            // runningMonitor.setZkClient(zkClientx);
-            // runningMonitor.setClientData(clientData);
-            // runningMonitor.setListener(new ClientRunningListener() {
-            // public void processActiveEnter() {
-            // mutex.set(true);
-            // }
-            //
-            // public void processActiveExit() {
-            // mutex.set(false);
-            // }
-            // });
         }
-
     }
 
     /**
@@ -93,57 +76,36 @@ public class KafkaCanalConnector {
             return;
         }
 
-        // if (runningMonitor != null) {
-        // if (!runningMonitor.isStart()) {
-        // runningMonitor.start();
-        // }
-        // }
-
         connected = true;
 
-        if (kafkaConsumer == null) {
+        if (kafkaConsumer == null && !flatMessage) {
             kafkaConsumer = new KafkaConsumer<String, Message>(properties);
         }
+        if (kafkaConsumer2 == null && flatMessage) {
+            kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
+        }
     }
 
     /**
      * 关闭链接
      */
-    public void disconnnect() {
+    public void disconnect() {
-        kafkaConsumer.close();
+        if (kafkaConsumer != null) {
+            kafkaConsumer.close();
+        }
+        if (kafkaConsumer2 != null) {
+            kafkaConsumer2.close();
+        }
 
         connected = false;
-        // if (runningMonitor.isStart()) {
-        // runningMonitor.stop();
-        // }
     }
 
     private void waitClientRunning() {
-        try {
+        running = true;
-            if (zkClientx != null) {
-                if (!connected) {// 未调用connect
-                    throw new CanalClientException("should connect first");
-                }
-
-                running = true;
-                // mutex.get();// 阻塞等待
-            } else {
-                // 单机模式直接设置为running
-                running = true;
-            }
-        } catch (Exception e) {
-            Thread.currentThread().interrupt();
-            throw new CanalClientException(e);
-        }
     }
 
     public boolean checkValid() {
-        if (zkClientx != null) {
+        return true;// 默认都放过
-            // return mutex.state();
-            return true;
-        } else {
-            return true;// 默认都放过
-        }
     }
 
     /**
@@ -156,10 +118,20 @@ public class KafkaCanalConnector {
         }
 
         if (partition == null) {
-            kafkaConsumer.subscribe(Collections.singletonList(topic));
+            if (kafkaConsumer != null) {
+                kafkaConsumer.subscribe(Collections.singletonList(topic));
+            }
+            if (kafkaConsumer2 != null) {
+                kafkaConsumer2.subscribe(Collections.singletonList(topic));
+            }
         } else {
             TopicPartition topicPartition = new TopicPartition(topic, partition);
-            kafkaConsumer.assign(Collections.singletonList(topicPartition));
+            if (kafkaConsumer != null) {
+                kafkaConsumer.assign(Collections.singletonList(topicPartition));
+            }
+            if (kafkaConsumer2 != null) {
+                kafkaConsumer2.assign(Collections.singletonList(topicPartition));
+            }
         }
     }
 
@@ -172,7 +144,12 @@ public class KafkaCanalConnector {
             return;
         }
 
-        kafkaConsumer.unsubscribe();
+        if (kafkaConsumer != null) {
+            kafkaConsumer.unsubscribe();
+        }
+        if (kafkaConsumer2 != null) {
+            kafkaConsumer2.unsubscribe();
+        }
     }
 
     /**
@@ -180,22 +157,22 @@ public class KafkaCanalConnector {
      *
      * @return
      */
-    public Message get() {
+    public List<Message> get() {
         return get(100L, TimeUnit.MILLISECONDS);
     }
 
-    public Message get(Long timeout, TimeUnit unit) {
+    public List<Message> get(Long timeout, TimeUnit unit) {
         waitClientRunning();
         if (!running) {
             return null;
         }
 
-        Message message = getWithoutAck(timeout, unit);
+        List<Message> messages = getWithoutAck(timeout, unit);
         this.ack();
-        return message;
+        return messages;
     }
 
-    public Message getWithoutAck() {
+    public List<Message> getWithoutAck() {
         return getWithoutAck(100L, TimeUnit.MILLISECONDS);
     }
 
@@ -204,7 +181,7 @@ public class KafkaCanalConnector {
      *
      * @return
      */
-    public Message getWithoutAck(Long timeout, TimeUnit unit) {
+    public List<Message> getWithoutAck(Long timeout, TimeUnit unit) {
         waitClientRunning();
         if (!running) {
             return null;
@@ -213,7 +190,33 @@ public class KafkaCanalConnector {
         ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据
 
         if (!records.isEmpty()) {
-            return records.iterator().next().value();
+            // return records.iterator().next().value();
+            List<Message> messages = new ArrayList<>();
+            for (ConsumerRecord<String, Message> record : records) {
+                messages.add(record.value());
+            }
+            return messages;
+        }
+        return null;
+    }
+
+    public List<FlatMessage> getFlatMessageWithoutAck(Long timeout, TimeUnit unit) {
+        waitClientRunning();
+        if (!running) {
+            return null;
+        }
+
+        ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
+
+        if (!records.isEmpty()) {
+            List<FlatMessage> flatMessages = new ArrayList<>();
+            for (ConsumerRecord<String, String> record : records) {
+                String flatMessageJson = record.value();
+                FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
+                flatMessages.add(flatMessage);
+            }
+
+            return flatMessages;
         }
         return null;
     }
@@ -227,7 +230,12 @@ public class KafkaCanalConnector {
             return;
         }
 
-        kafkaConsumer.commitSync();
+        if (kafkaConsumer != null) {
+            kafkaConsumer.commitSync();
+        }
+        if (kafkaConsumer2 != null) {
+            kafkaConsumer2.commitAsync();
+        }
     }
 
     public void stopRunning() {

client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnectors.java

@@ -17,8 +17,9 @@ public class KafkaCanalConnectors {
      * @param groupId
      * @return
      */
-    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) {
+    public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition,
-        return new KafkaCanalConnector(null, servers, topic, partition, groupId);
+                                                        String groupId) {
+        return new KafkaCanalConnector(servers, topic, partition, groupId, false);
     }
 
     /**
@@ -30,33 +31,21 @@ public class KafkaCanalConnectors {
      * @return
      */
     public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
-        return new KafkaCanalConnector(null, servers, topic, null, groupId);
+        return new KafkaCanalConnector(servers, topic, null, groupId, false);
     }
 
     /**
      * 创建kafka客户端链接
      *
-     * @param zkServers
      * @param servers
      * @param topic
      * @param partition
      * @param groupId
+     * @param flatMessage
      * @return
      */
-    public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, Integer partition, String groupId) {
+    public static KafkaCanalConnector newKafkaConnector( String servers, String topic,
-        return new KafkaCanalConnector(zkServers, servers, topic, partition, groupId);
+                                                        Integer partition, String groupId,boolean flatMessage) {
-    }
+        return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage);
-
-    /**
-     * 创建kafka客户端链接
-     *
-     * @param zkServers
-     * @param servers
-     * @param topic
-     * @param groupId
-     * @return
-     */
-    public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, String groupId) {
-        return new KafkaCanalConnector(zkServers, servers, topic, null, groupId);
     }
 }

client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java

@@ -1,6 +1,6 @@
 package com.alibaba.otter.canal.client.kafka;
 
-import com.alibaba.otter.canal.client.DeserializerUtil;
+import com.alibaba.otter.canal.client.CanalMessageDeserializer;
 import com.alibaba.otter.canal.protocol.Message;
 import java.util.Map;
 import org.apache.kafka.common.serialization.Deserializer;
@@ -23,7 +23,7 @@ public class MessageDeserializer implements Deserializer<Message> {
 
     @Override
     public Message deserialize(String topic1, byte[] data) {
-        return DeserializerUtil.deserializer(data);
+        return CanalMessageDeserializer.deserializer(data);
     }
 
     @Override

client/src/main/java/com/alibaba/otter/canal/client/rocketmq/RocketMQCanalConnector.java

@@ -1,7 +1,7 @@
 package com.alibaba.otter.canal.client.rocketmq;
 
 import com.alibaba.otter.canal.client.CanalConnector;
-import com.alibaba.otter.canal.client.DeserializerUtil;
+import com.alibaba.otter.canal.client.CanalMessageDeserializer;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import java.util.List;
@@ -94,7 +94,7 @@ public class RocketMQCanalConnector implements CanalConnector {
         BlockingQueue<Message> messageList = new LinkedBlockingQueue<>();
         for (MessageExt messageExt : messageExts) {
             byte[] data = messageExt.getBody();
-            Message message = DeserializerUtil.deserializer(data);
+            Message message = CanalMessageDeserializer.deserializer(data);
             try {
                 messageList.put(message);
             } catch (InterruptedException ex) {

client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java

@@ -20,117 +20,117 @@ import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
 
 public class ClientRunningTest extends AbstractZkTest {
 
-    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+	private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
-    private short     clientId  = 1001;
+	private short     clientId  = 1001;
 
-    @Before
+	@Before
-    public void setUp() {
+	public void setUp() {
-        String path = ZookeeperPathUtils.getDestinationPath(destination);
+		String path = ZookeeperPathUtils.getDestinationPath(destination);
-        zkclientx.deleteRecursive(path);
+		zkclientx.deleteRecursive(path);
 
-        zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
+		zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
-    }
+	}
 
-    @After
+	@After
-    public void tearDown() {
+	public void tearDown() {
-        String path = ZookeeperPathUtils.getDestinationPath(destination);
+		String path = ZookeeperPathUtils.getDestinationPath(destination);
-        zkclientx.deleteRecursive(path);
+		zkclientx.deleteRecursive(path);
-    }
+	}
 
-    @Test
+	@Test
-    public void testOneServer() {
+	public void testOneServer() {
-        final CountDownLatch countLatch = new CountDownLatch(2);
+		final CountDownLatch countLatch = new CountDownLatch(2);
-        ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
+		ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
-        runningMonitor.start();
+		runningMonitor.start();
-        sleep(2000L);
+		sleep(2000L);
-        runningMonitor.stop();
+		runningMonitor.stop();
-        sleep(2000L);
+		sleep(2000L);
 
-        if (countLatch.getCount() != 0) {
+		if (countLatch.getCount() != 0) {
-            Assert.fail();
+			Assert.fail();
-        }
+		}
-    }
+	}
 
-    @Test
+	@Test
-    public void testMultiServer() {
+	public void testMultiServer() {
-        final CountDownLatch countLatch = new CountDownLatch(30);
+		final CountDownLatch countLatch = new CountDownLatch(30);
-        final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
+		final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
-        final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
+		final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
-        final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
+		final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
-        final ExecutorService executor = Executors.newFixedThreadPool(3);
+		final ExecutorService executor = Executors.newFixedThreadPool(3);
-        executor.submit(new Runnable() {
+		executor.submit(new Runnable() {
 
-            public void run() {
+			public void run() {
-                for (int i = 0; i < 10; i++) {
+				for (int i = 0; i < 10; i++) {
-                    if (!runningMonitor1.isStart()) {
+					if (!runningMonitor1.isStart()) {
-                        runningMonitor1.start();
+						runningMonitor1.start();
-                    }
+					}
-                    sleep(2000L + RandomUtils.nextInt(500));
+					sleep(2000L + RandomUtils.nextInt(500));
-                    runningMonitor1.stop();
+					runningMonitor1.stop();
-                    sleep(2000L + RandomUtils.nextInt(500));
+					sleep(2000L + RandomUtils.nextInt(500));
-                }
+				}
-            }
+			}
 
-        });
+		});
 
-        executor.submit(new Runnable() {
+		executor.submit(new Runnable() {
 
-            public void run() {
+			public void run() {
-                for (int i = 0; i < 10; i++) {
+				for (int i = 0; i < 10; i++) {
-                    if (!runningMonitor2.isStart()) {
+					if (!runningMonitor2.isStart()) {
-                        runningMonitor2.start();
+						runningMonitor2.start();
-                    }
+					}
-                    sleep(2000L + RandomUtils.nextInt(500));
+					sleep(2000L + RandomUtils.nextInt(500));
-                    runningMonitor2.stop();
+					runningMonitor2.stop();
-                    sleep(2000L + RandomUtils.nextInt(500));
+					sleep(2000L + RandomUtils.nextInt(500));
-                }
+				}
-            }
+			}
 
-        });
+		});
 
-        executor.submit(new Runnable() {
+		executor.submit(new Runnable() {
 
-            public void run() {
+			public void run() {
-                for (int i = 0; i < 10; i++) {
+				for (int i = 0; i < 10; i++) {
-                    if (!runningMonitor3.isStart()) {
+					if (!runningMonitor3.isStart()) {
-                        runningMonitor3.start();
+						runningMonitor3.start();
-                    }
+					}
-                    sleep(2000L + RandomUtils.nextInt(500));
+					sleep(2000L + RandomUtils.nextInt(500));
-                    runningMonitor3.stop();
+					runningMonitor3.stop();
-                    sleep(2000L + RandomUtils.nextInt(500));
+					sleep(2000L + RandomUtils.nextInt(500));
-                }
+				}
-            }
+			}
 
-        });
+		});
 
-        sleep(30000L);
+		sleep(30000L);
-    }
+	}
 
-    private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
+	private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
-                                                    final int port) {
+																									final int port) {
-        ClientRunningData clientData = new ClientRunningData();
+		ClientRunningData clientData = new ClientRunningData();
-        clientData.setClientId(clientId);
+		clientData.setClientId(clientId);
-        clientData.setAddress(AddressUtils.getHostIp());
+		clientData.setAddress(AddressUtils.getHostIp());
 
-        ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
+		ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
-        runningMonitor.setDestination(destination);
+		runningMonitor.setDestination(destination);
-        runningMonitor.setZkClient(zkclientx);
+		runningMonitor.setZkClient(zkclientx);
-        runningMonitor.setClientData(clientData);
+		runningMonitor.setClientData(clientData);
-        runningMonitor.setListener(new ClientRunningListener() {
+		runningMonitor.setListener(new ClientRunningListener() {
 
-            public InetSocketAddress processActiveEnter() {
+			public InetSocketAddress processActiveEnter() {
-                System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
+				System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
-                countLatch.countDown();
+				countLatch.countDown();
-                return new InetSocketAddress(AddressUtils.getHostIp(), port);
+				return new InetSocketAddress(AddressUtils.getHostIp(), port);
-            }
+			}
 
-            public void processActiveExit() {
+			public void processActiveExit() {
-                countLatch.countDown();
+				countLatch.countDown();
-                System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
+				System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
-            }
+			}
 
-        });
+		});
-        runningMonitor.setDelayTime(1);
+		runningMonitor.setDelayTime(1);
-        return runningMonitor;
+		return runningMonitor;
-    }
+	}
 }

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/CanalKafkaClientExample.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.client.running.kafka;
 
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.kafka.common.errors.WakeupException;
@@ -19,32 +20,33 @@ import com.alibaba.otter.canal.protocol.Message;
  */
 public class CanalKafkaClientExample {
 
-    protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
+    protected final static Logger           logger  = LoggerFactory.getLogger(CanalKafkaClientExample.class);
 
-    private KafkaCanalConnector connector;
+    private KafkaCanalConnector             connector;
 
-    private static volatile boolean running = false;
+    private static volatile boolean         running = false;
 
-    private Thread thread = null;
+    private Thread                          thread  = null;
 
     private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
 
-        public void uncaughtException(Thread t, Throwable e) {
+                                                        public void uncaughtException(Thread t, Throwable e) {
-            logger.error("parse events has an error", e);
+                                                            logger.error("parse events has an error", e);
-        }
+                                                        }
-    };
+                                                    };
 
-    public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId) {
+    public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
-        connector = KafkaCanalConnectors.newKafkaConnector(zkServers, servers, topic, partition, groupId);
+        connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false);
     }
 
     public static void main(String[] args) {
         try {
-            final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
+            final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
-                    AbstractKafkaTest.servers,
+                AbstractKafkaTest.zkServers,
-                    AbstractKafkaTest.topic,
+                AbstractKafkaTest.servers,
-                    AbstractKafkaTest.partition,
+                AbstractKafkaTest.topic,
-                    AbstractKafkaTest.groupId);
+                AbstractKafkaTest.partition,
+                AbstractKafkaTest.groupId);
             logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
             kafkaCanalClientExample.start();
             logger.info("## the canal kafka consumer is running now ......");
@@ -107,21 +109,24 @@ public class CanalKafkaClientExample {
                 connector.subscribe();
                 while (running) {
                     try {
-                        Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
+                        List<Message> messages = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
-                        if (message == null) {
+
+                        if (messages == null) {
                             continue;
                         }
-                        long batchId = message.getId();
+                        for (Message message : messages) {
-                        int size = message.getEntries().size();
+                            long batchId = message.getId();
-                        if (batchId == -1 || size == 0) {
+                            int size = message.getEntries().size();
-                            // try {
+                            if (batchId == -1 || size == 0) {
-                            // Thread.sleep(1000);
+                                // try {
-                            // } catch (InterruptedException e) {
+                                // Thread.sleep(1000);
-                            // }
+                                // } catch (InterruptedException e) {
-                        } else {
+                                // }
-                            // printSummary(message, batchId, size);
+                            } else {
-                            // printEntry(message.getEntries());
+                                // printSummary(message, batchId, size);
-                            logger.info(message.toString());
+                                // printEntry(message.getEntries());
+                                logger.info(message.toString());
+                            }
                         }
 
                         connector.ack(); // 提交确认
@@ -139,6 +144,6 @@ public class CanalKafkaClientExample {
         } catch (WakeupException e) {
             // No-op. Continue process
         }
-        connector.disconnnect();
+        connector.disconnect();
     }
 }

client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.client.running.kafka;
 
+import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -39,9 +40,9 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
                 connector.subscribe();
                 while (running) {
                     try {
-                        Message message = connector.getWithoutAck(3L, TimeUnit.SECONDS);
+                        List<Message> messages = connector.getWithoutAck(3L, TimeUnit.SECONDS);
-                        if (message != null) {
+                        if (messages != null) {
-                            System.out.println(message);
+                            System.out.println(messages);
                         }
                         connector.ack();
                     } catch (WakeupException e) {
@@ -49,7 +50,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
                     }
                 }
                 connector.unsubscribe();
-                connector.disconnnect();
+                connector.disconnect();
             }
         });
 

dbsync/pom.xml

@@ -11,6 +11,11 @@
 	<packaging>jar</packaging>
 	<name>canal dbsync module for otter ${project.version}</name>
 	<dependencies>
+		<dependency>
+			<groupId>com.alibaba.otter</groupId>
+			<artifactId>canal.parse.driver</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 		<!-- log -->
 		<dependency>
 			<groupId>ch.qos.logback</groupId>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java

@@ -3,7 +3,9 @@ package com.taobao.tddl.dbsync.binlog;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
 
 /**
@@ -20,6 +22,10 @@ public final class LogContext {
 
     private LogPosition                       logPosition;
 
+    private GTIDSet                           gtidSet;
+
+    private GtidLogEvent                      gtidLogEvent; // save current gtid log event
+
     public LogContext(){
         this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
     }
@@ -60,4 +66,20 @@ public final class LogContext {
         formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
         mapOfTable.clear();
     }
+
+    public GTIDSet getGtidSet() {
+        return gtidSet;
+    }
+
+    public void setGtidSet(GTIDSet gtidSet) {
+        this.gtidSet = gtidSet;
+    }
+
+    public GtidLogEvent getGtidLogEvent() {
+        return gtidLogEvent;
+    }
+
+    public void setGtidLogEvent(GtidLogEvent gtidLogEvent) {
+        this.gtidLogEvent = gtidLogEvent;
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -6,6 +6,7 @@ import java.util.BitSet;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.taobao.tddl.dbsync.binlog.event.AppendBlockLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.BeginLoadQueryLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.CreateFileLogEvent;
@@ -159,18 +160,21 @@ public final class LogDecoder {
             // remove checksum bytes
             buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
         }
-
+        GTIDSet gtidSet = context.getGtidSet();
+        GtidLogEvent gtidLogEvent = context.getGtidLogEvent();
         switch (header.getType()) {
             case LogEvent.QUERY_EVENT: {
                 QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.XID_EVENT: {
                 XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.TABLE_MAP_EVENT: {
@@ -268,12 +272,14 @@ public final class LogDecoder {
                 RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.USER_VAR_EVENT: {
                 UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.FORMAT_DESCRIPTION_EVENT: {
@@ -336,6 +342,7 @@ public final class LogDecoder {
                 RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.WRITE_ROWS_EVENT: {
@@ -343,6 +350,7 @@ public final class LogDecoder {
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.UPDATE_ROWS_EVENT: {
@@ -350,6 +358,7 @@ public final class LogDecoder {
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.DELETE_ROWS_EVENT: {
@@ -357,6 +366,7 @@ public final class LogDecoder {
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
                 event.fillTable(context);
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.GTID_LOG_EVENT:
@@ -364,6 +374,13 @@ public final class LogDecoder {
                 GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                if (gtidSet != null) {
+                    gtidSet.update(event.getGtidStr());
+                    // update latest gtid
+                    header.putGtid(gtidSet, event);
+                }
+                // update current gtid event to context
+                context.setGtidLogEvent(event);
                 return event;
             }
             case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
@@ -394,6 +411,7 @@ public final class LogDecoder {
                 AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
                 /* updating position in context */
                 logPosition.position = header.getLogPos();
+                header.putGtid(context.getGtidSet(), gtidLogEvent);
                 return event;
             }
             case LogEvent.BINLOG_CHECKPOINT_EVENT: {

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java

@@ -14,10 +14,10 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
 public class GtidLogEvent extends LogEvent {
 
     // / Length of the commit_flag in event encoding
-    public static final int ENCODED_FLAG_LENGTH = 1;
+    public static final int ENCODED_FLAG_LENGTH         = 1;
     // / Length of SID in event encoding
-    public static final int ENCODED_SID_LENGTH  = 16;
+    public static final int ENCODED_SID_LENGTH          = 16;
-    public static final int LOGICAL_TIMESTAMP_TYPE_CODE  = 2;
+    public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2;
 
     private boolean         commitFlag;
     private UUID            sid;
@@ -50,8 +50,6 @@ public class GtidLogEvent extends LogEvent {
             sequenceNumber = buffer.getLong64();
         }
 
-
-
         // ignore gtid info read
         // sid.copy_from((uchar *)ptr_buffer);
         // ptr_buffer+= ENCODED_SID_LENGTH;
@@ -82,4 +80,11 @@ public class GtidLogEvent extends LogEvent {
     public Long getSequenceNumber() {
         return sequenceNumber;
     }
+
+    public String getGtidStr() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(sid.toString()).append(":");
+        sb.append(gno);
+        return sb.toString();
+    }
 }

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java

@@ -1,8 +1,12 @@
 package com.taobao.tddl.dbsync.binlog.event;
 
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.taobao.tddl.dbsync.binlog.LogBuffer;
 import com.taobao.tddl.dbsync.binlog.LogEvent;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * The Common-Header, documented in the table @ref Table_common_header "below",
  * always has the same form and length within one version of MySQL. Each event
@@ -120,6 +124,13 @@ public final class LogHeader {
      */
     protected String    logFileName;
 
+    protected Map<String, String> gtidMap = new HashMap<>();
+
+    private static final String CURRENT_GTID_STRING = "curt_gtid";
+    private static final String CURRENT_GTID_SN = "curt_gtid_sn";
+    private static final String CURRENT_GTID_LAST_COMMIT = "curt_gtid_lct";
+    private static final String GTID_SET_STRING = "gtid_str";
+
     /* for Start_event_v3 */
     public LogHeader(final int type){
         this.type = type;
@@ -288,4 +299,31 @@ public final class LogHeader {
             crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
         }
     }
+
+    public String getGtidSetStr() {
+        return gtidMap.get(GTID_SET_STRING);
+    }
+
+    public String getCurrentGtid() {
+        return gtidMap.get(CURRENT_GTID_STRING);
+    }
+
+    public String getCurrentGtidSn() {
+        return gtidMap.get(CURRENT_GTID_SN);
+    }
+
+    public String getCurrentGtidLastCommit() {
+        return gtidMap.get(CURRENT_GTID_LAST_COMMIT);
+    }
+
+    public void putGtid(GTIDSet gtidSet, GtidLogEvent event) {
+        if (gtidSet != null) {
+            gtidMap.put(GTID_SET_STRING, gtidSet.toString());
+            if (event != null) {
+                gtidMap.put(CURRENT_GTID_STRING, event.getGtidStr());
+                gtidMap.put(CURRENT_GTID_SN, String.valueOf(event.getSequenceNumber()));
+                gtidMap.put(CURRENT_GTID_LAST_COMMIT, String.valueOf(event.getLastCommitted()));
+            }
+        }
+    }
 }

dbsync/src/test/java/com/taobao/tddl/dbsync/FetcherPerformanceTest.java

@@ -15,9 +15,7 @@ public class FetcherPerformanceTest {
         DirectLogFetcher fetcher = new DirectLogFetcher();
         try {
             Class.forName("com.mysql.jdbc.Driver");
-            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306",
+            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello");
-                "root",
-                "hello");
             Statement statement = connection.createStatement();
             statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
             statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java

@@ -387,7 +387,7 @@ public class CanalController {
         return config;
     }
 
-    private String getProperty(Properties properties, String key) {
+    public String getProperty(Properties properties, String key) {
         key = StringUtils.trim(key);
         String value = System.getProperty(key);
 

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java

@@ -6,7 +6,6 @@ import com.alibaba.otter.canal.server.CanalMQStarter;
 import com.alibaba.otter.canal.spi.CanalMQProducer;
 import java.io.FileInputStream;
 import java.util.Properties;
-
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,7 +56,7 @@ public class CanalLauncher {
             });
 
             CanalMQProducer canalMQProducer = null;
-            String serverMode = properties.getProperty(CanalConstants.CANAL_SERVER_MODE, "tcp");
+            String serverMode = controller.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
             if (serverMode.equalsIgnoreCase("kafka")) {
                 canalMQProducer = new CanalKafkaProducer();
             } else if (serverMode.equalsIgnoreCase("rocketmq")) {
@@ -68,6 +67,7 @@ public class CanalLauncher {
                 if (canalServerStarter != null) {
                     canalServerStarter.init();
                 }
+
             }
         } catch (Throwable e) {
             logger.error("## Something goes wrong when starting up the canal Server:", e);

deployer/src/main/resources/metrics/Canal_instances_tmpl.json

@@ -1,4 +1,34 @@
 {
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "5.2.2"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": "5.0.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "5.0.0"
+    }
+  ],
   "annotations": {
     "list": [
       {
@@ -15,8 +45,8 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 8,
+  "id": null,
-  "iteration": 1534740332233,
+  "iteration": 1536989235272,
   "links": [],
   "panels": [
     {
@@ -189,7 +219,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\"}[2m]) / 1024",
+          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024",
           "format": "time_series",
           "interval": "15s",
           "intervalFactor": 2,
@@ -203,6 +233,22 @@
           "intervalFactor": 2,
           "legendFormat": "outbound",
           "refId": "B"
+        },
+        {
+          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024",
+          "format": "time_series",
+          "interval": "15s",
+          "intervalFactor": 2,
+          "legendFormat": "inbound-1",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024",
+          "format": "time_series",
+          "interval": "15s",
+          "intervalFactor": 2,
+          "legendFormat": "inbound-2",
+          "refId": "D"
         }
       ],
       "thresholds": [],
@@ -399,20 +445,36 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
+          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10",
           "format": "time_series",
           "interval": "15s",
           "intervalFactor": 2,
           "legendFormat": "dump",
-          "refId": "A"
+          "refId": "B"
         },
         {
-          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
+          "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
           "format": "time_series",
           "interval": "15s",
           "intervalFactor": 2,
           "legendFormat": "sink",
-          "refId": "B"
+          "refId": "A"
+        },
+        {
+          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10",
+          "format": "time_series",
+          "interval": "15s",
+          "intervalFactor": 2,
+          "legendFormat": "dump-1",
+          "refId": "C"
+        },
+        {
+          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10",
+          "format": "time_series",
+          "interval": "15s",
+          "intervalFactor": 2,
+          "legendFormat": "dump-2",
+          "refId": "D"
         }
       ],
       "thresholds": [],
@@ -465,7 +527,7 @@
       },
       "id": 32,
       "panels": [],
-      "title": "Server",
+      "title": "Throughput",
       "type": "row"
     },
     {
@@ -660,180 +722,6 @@
         "alignLevel": null
       }
     },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "$datasource",
-      "description": "Canal instance ringbuffer内未释放的events数量。",
-      "fill": 1,
-      "gridPos": {
-        "h": 5,
-        "w": 6,
-        "x": 12,
-        "y": 7
-      },
-      "id": 20,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",
-          "format": "time_series",
-          "interval": "15s",
-          "intervalFactor": 2,
-          "legendFormat": "events",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Store remain events",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "decimals": null,
-          "format": "none",
-          "label": "",
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "decimals": null,
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "$datasource",
-      "description": "Canal instance ringbuffer 内未释放events占用内存。",
-      "fill": 1,
-      "gridPos": {
-        "h": 5,
-        "w": 6,
-        "x": 18,
-        "y": 7
-      },
-      "id": 22,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",
-          "format": "time_series",
-          "interval": "15s",
-          "intervalFactor": 2,
-          "legendFormat": "memsize",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Store remain mem",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "deckbytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
     {
       "collapsed": false,
       "gridPos": {
@@ -938,7 +826,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "$datasource",
+      "datasource": "${DS_PROMETHEUS}",
       "description": "client 请求的GET与ACK包的QPS。",
       "fill": 1,
       "gridPos": {
@@ -1218,6 +1106,193 @@
         "align": false,
         "alignLevel": null
       }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 18
+      },
+      "id": 36,
+      "panels": [],
+      "title": "Store",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "description": "Canal instance ringbuffer内未释放的events数量。",
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 0,
+        "y": 19
+      },
+      "id": 20,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",
+          "format": "time_series",
+          "interval": "15s",
+          "intervalFactor": 2,
+          "legendFormat": "events",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Store remain events",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "none",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "description": "Canal instance ringbuffer 内未释放events占用内存。",
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 6,
+        "y": 19
+      },
+      "id": 22,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",
+          "format": "time_series",
+          "interval": "15s",
+          "intervalFactor": 2,
+          "legendFormat": "memsize",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Store remain mem",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "deckbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
     }
   ],
   "refresh": false,
@@ -1230,9 +1305,8 @@
     "list": [
       {
         "current": {
-          "tags": [],
+          "text": "prometheus",
-          "text": "Prometheus",
+          "value": "prometheus"
-          "value": "Prometheus"
         },
         "hide": 0,
         "label": "datasource",
@@ -1245,10 +1319,7 @@
       },
       {
         "allValue": null,
-        "current": {
+        "current": {},
-          "text": "example",
-          "value": "example"
-        },
         "datasource": "$datasource",
         "hide": 0,
         "includeAll": false,
@@ -1269,7 +1340,7 @@
     ]
   },
   "time": {
-    "from": "now-1h",
+    "from": "now-6h",
     "to": "now"
   },
   "timepicker": {
@@ -1300,5 +1371,5 @@
   "timezone": "",
   "title": "Canal instances",
   "uid": "8vh8NGpiz",
-  "version": 1
+  "version": 103
 }

deployer/src/main/resources/mq.yml

@@ -3,17 +3,21 @@ retries: 0
 batchSize: 16384
 lingerMs: 1
 bufferMemory: 33554432
-# canal的批次大小，单位 k，量大建议改为1M
+
+# Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下)
 canalBatchSize: 50
-filterTransactionEntry: true
+# Canal get数据的超时时间, 单位: 毫秒, 空为不限超时
+canalGetTimeout: 100
+flatMessage: true
 
 canalDestinations:
   - canalDestination: example
     topic: example
     partition: 1
-    # 一个destination可以对应多个topic
+#  #对应topic分区数量
-    #topics:
+#  partitionsNum: 3
-    #  - topic: example
+#  partitionHash:
-    #    partition:
+#    #库名.表名: 唯一主键
+#    mytest.person: id
 
 

deployer/src/main/resources/spring/tsdb/sql/create_table.sql

@@ -26,10 +26,10 @@ CREATE TABLE IF NOT EXISTS `meta_history` (
   `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
   `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
   `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
-  `schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
+  `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
-  `table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
+  `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
-  `sql` longtext DEFAULT NULL COMMENT '执行的sql',
+  `sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
-  `type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
+  `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
   `extra` text DEFAULT NULL COMMENT '额外的扩展信息',
   PRIMARY KEY (`id`),
   UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannel.java

@@ -138,6 +138,16 @@ public class BioSocketChannel implements SocketChannel {
         if (socket != null) {
             return socket.getRemoteSocketAddress();
         }
+
+        return null;
+    }
+
+    public SocketAddress getLocalSocketAddress() {
+        Socket socket = this.socket;
+        if (socket != null) {
+            return socket.getLocalSocketAddress();
+        }
+
         return null;
     }
 

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/NettySocketChannel.java

@@ -216,6 +216,10 @@ public class NettySocketChannel implements SocketChannel {
         return channel != null ? channel.remoteAddress() : null;
     }
 
+    public SocketAddress getLocalSocketAddress() {
+        return channel != null ? channel.localAddress() : null;
+    }
+
     public void close() {
         if (channel != null) {
             channel.close();

driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java

@@ -21,5 +21,7 @@ public interface SocketChannel {
 
     public SocketAddress getRemoteSocketAddress();
 
+    public SocketAddress getLocalSocketAddress();
+
     public void close();
 }

example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java

@@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
 
+import com.alibaba.otter.canal.protocol.CanalEntry;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.SystemUtils;
 import org.slf4j.Logger;
@@ -299,4 +300,55 @@ public class AbstractCanalClientTest {
         this.connector = connector;
     }
 
+    /**
+     * 获取当前Entry的 GTID信息示例
+     * @param header
+     * @return
+     */
+    public static String getCurrentGtid(CanalEntry.Header header) {
+        List<CanalEntry.Pair> props = header.getPropsList();
+        if (props != null && props.size() > 0) {
+            for (CanalEntry.Pair pair : props) {
+                if ("curtGtid".equals(pair.getKey())) {
+                    return pair.getValue();
+                }
+            }
+        }
+        return "";
+    }
+
+    /**
+     * 获取当前Entry的 GTID Sequence No信息示例
+     * @param header
+     * @return
+     */
+    public static String getCurrentGtidSn(CanalEntry.Header header) {
+        List<CanalEntry.Pair> props = header.getPropsList();
+        if (props != null && props.size() > 0) {
+            for (CanalEntry.Pair pair : props) {
+                if ("curtGtidSn".equals(pair.getKey())) {
+                    return pair.getValue();
+                }
+            }
+        }
+        return "";
+    }
+
+    /**
+     * 获取当前Entry的 GTID Last Committed信息示例
+     * @param header
+     * @return
+     */
+    public static String getCurrentGtidLct(CanalEntry.Header header) {
+        List<CanalEntry.Pair> props = header.getPropsList();
+        if (props != null && props.size() > 0) {
+            for (CanalEntry.Pair pair : props) {
+                if ("curtGtidLct".equals(pair.getKey())) {
+                    return pair.getValue();
+                }
+            }
+        }
+        return "";
+    }
+
 }

parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java

@@ -8,23 +8,23 @@ public class PositionNotFoundException extends CanalParseException {
 
     private static final long serialVersionUID = -7382448928116244017L;
 
-    public PositionNotFoundException(String errorCode) {
+    public PositionNotFoundException(String errorCode){
         super(errorCode);
     }
 
-    public PositionNotFoundException(String errorCode, Throwable cause) {
+    public PositionNotFoundException(String errorCode, Throwable cause){
         super(errorCode, cause);
     }
 
-    public PositionNotFoundException(String errorCode, String errorDesc) {
+    public PositionNotFoundException(String errorCode, String errorDesc){
         super(errorCode, errorDesc);
     }
 
-    public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause) {
+    public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause){
         super(errorCode, errorDesc, cause);
     }
 
-    public PositionNotFoundException(Throwable cause) {
+    public PositionNotFoundException(Throwable cause){
         super(cause);
     }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java

@@ -6,27 +6,27 @@ import com.alibaba.otter.canal.common.CanalException;
  * @author chengjin.lyf on 2018/8/8 下午1:07
  * @since 1.0.25
  */
-public class ServerIdNotMatchException extends CanalException{
+public class ServerIdNotMatchException extends CanalException {
 
     private static final long serialVersionUID = -6124989280379293953L;
 
-    public ServerIdNotMatchException(String errorCode) {
+    public ServerIdNotMatchException(String errorCode){
         super(errorCode);
     }
 
-    public ServerIdNotMatchException(String errorCode, Throwable cause) {
+    public ServerIdNotMatchException(String errorCode, Throwable cause){
         super(errorCode, cause);
     }
 
-    public ServerIdNotMatchException(String errorCode, String errorDesc) {
+    public ServerIdNotMatchException(String errorCode, String errorDesc){
         super(errorCode, errorDesc);
     }
 
-    public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause) {
+    public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause){
         super(errorCode, errorDesc, cause);
     }
 
-    public ServerIdNotMatchException(Throwable cause) {
+    public ServerIdNotMatchException(Throwable cause){
         super(cause);
     }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java

@@ -19,11 +19,13 @@ import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
 import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
 import com.alibaba.otter.canal.filter.CanalEventFilter;
 import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
 import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
 import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlMultiStageCoprocessor;
 import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
 import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
 import com.alibaba.otter.canal.protocol.CanalEntry;
@@ -235,12 +237,14 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
                         if (parallel) {
                             // build stage processor
                             multiStageCoprocessor = buildMultiStageCoprocessor();
-                            multiStageCoprocessor.start();
-
                             if (isGTIDMode()) {
                                 // 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
-                                erosaConnection.dump(MysqlGTIDSet.parse(startPosition.getGtid()), multiStageCoprocessor);
+                                GTIDSet gtidSet = MysqlGTIDSet.parse(startPosition.getGtid());
+                                ((MysqlMultiStageCoprocessor) multiStageCoprocessor).setGtidSet(gtidSet);
+                                multiStageCoprocessor.start();
+                                erosaConnection.dump(gtidSet, multiStageCoprocessor);
                             } else {
+                                multiStageCoprocessor.start();
                                 if (StringUtils.isEmpty(startPosition.getJournalName())
                                     && startPosition.getTimestamp() != null) {
                                     erosaConnection.dump(startPosition.getTimestamp(), multiStageCoprocessor);

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java

@@ -5,5 +5,6 @@ package com.alibaba.otter.canal.parse.inbound;
  * @since 1.0.25
  */
 public interface ParserExceptionHandler {
+
     void handle(Throwable e);
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java

@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.otter.canal.filter.CanalEventFilter;
 import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
 import com.alibaba.otter.canal.parse.CanalEventParser;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
 import com.alibaba.otter.canal.parse.inbound.BinlogParser;
@@ -39,6 +38,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
     protected boolean              filterRows                = false;
     protected boolean              filterTableError          = false;
     protected boolean              useDruidDdlFilter         = true;
+    // instance received binlog bytes
+    protected final AtomicLong     receivedBinlogBytes       = new AtomicLong(0L);
     private final AtomicLong       eventsPublishBlockingTime = new AtomicLong(0L);
 
     protected BinlogParser buildParser() {
@@ -76,13 +77,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
      * @return
      */
     protected boolean processTableMeta(EntryPosition position) {
-        if (isGTIDMode()) {
-            if (binlogParser instanceof LogEventConvert) {
-                // 记录gtid
-                ((LogEventConvert) binlogParser).setGtidSet(MysqlGTIDSet.parse(position.getGtid()));
-            }
-        }
-
         if (tableMetaTSDB != null) {
             if (position.getTimestamp() == null || position.getTimestamp() <= 0) {
                 throw new CanalParseException("use gtid and TableMeta TSDB should be config timestamp > 0");
@@ -212,4 +206,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
         return this.eventsPublishBlockingTime;
     }
 
+    public AtomicLong getReceivedBinlogBytes() {
+        return this.receivedBinlogBytes;
+    }
+
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java

@@ -4,6 +4,7 @@ import static com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetche
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.net.SocketAddress;
 import java.nio.charset.Charset;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -179,6 +180,8 @@ public class MysqlConnection implements ErosaConnection {
             fetcher.start(connector.getChannel());
             LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             LogContext context = new LogContext();
+            // fix bug: #890 将gtid传输至context中，供decode使用
+            context.setGtidSet(gtidSet);
             while (fetcher.fetch()) {
                 accumulateReceivedBytes(fetcher.limit());
                 LogEvent event = null;
@@ -252,10 +255,18 @@ public class MysqlConnection implements ErosaConnection {
 
     private void sendRegisterSlave() throws IOException {
         RegisterSlaveCommandPacket cmd = new RegisterSlaveCommandPacket();
-        cmd.reportHost = authInfo.getAddress().getAddress().getHostAddress();
+        SocketAddress socketAddress = connector.getChannel().getLocalSocketAddress();
+        if (socketAddress == null || !(socketAddress instanceof InetSocketAddress)) {
+            return;
+        }
+
+        InetSocketAddress address = (InetSocketAddress) socketAddress;
+        String host = address.getHostString();
+        int port = address.getPort();
+        cmd.reportHost = host;
+        cmd.reportPort = port;
         cmd.reportPasswd = authInfo.getPassword();
         cmd.reportUser = authInfo.getUsername();
-        cmd.reportPort = authInfo.getAddress().getPort(); // 暂时先用master节点的port
         cmd.serverId = this.slaveId;
         byte[] cmdBody = cmd.toBytes();
 

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java

@@ -8,7 +8,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.lang.StringUtils;
 import org.springframework.util.CollectionUtils;
@@ -49,30 +48,27 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
 
     private CanalHAController    haController                      = null;
 
-    private int                  defaultConnectionTimeoutInSeconds = 30;                // sotimeout
+    private int                  defaultConnectionTimeoutInSeconds = 30;       // sotimeout
     private int                  receiveBufferSize                 = 64 * 1024;
     private int                  sendBufferSize                    = 64 * 1024;
     // 数据库信息
-    protected AuthenticationInfo masterInfo;                                            // 主库
+    protected AuthenticationInfo masterInfo;                                   // 主库
-    protected AuthenticationInfo standbyInfo;                                           // 备库
+    protected AuthenticationInfo standbyInfo;                                  // 备库
     // binlog信息
     protected EntryPosition      masterPosition;
     protected EntryPosition      standbyPosition;
-    private long                 slaveId;                                               // 链接到mysql的slave
+    private long                 slaveId;                                      // 链接到mysql的slave
     // 心跳检查信息
-    private String               detectingSQL;                                          // 心跳sql
+    private String               detectingSQL;                                 // 心跳sql
-    private MysqlConnection      metaConnection;                                        // 查询meta信息的链接
+    private MysqlConnection      metaConnection;                               // 查询meta信息的链接
-    private TableMetaCache       tableMetaCache;                                        // 对应meta
+    private TableMetaCache       tableMetaCache;                               // 对应meta
-    private int                  fallbackIntervalInSeconds         = 60;                // 切换回退时间
+    private int                  fallbackIntervalInSeconds         = 60;       // 切换回退时间
-    private BinlogFormat[]       supportBinlogFormats;                                  // 支持的binlogFormat,如果设置会执行强校验
+    private BinlogFormat[]       supportBinlogFormats;                         // 支持的binlogFormat,如果设置会执行强校验
-    private BinlogImage[]        supportBinlogImages;                                   // 支持的binlogImage,如果设置会执行强校验
+    private BinlogImage[]        supportBinlogImages;                          // 支持的binlogImage,如果设置会执行强校验
 
     // update by yishun.chen,特殊异常处理参数
-    private int                  dumpErrorCount                    = 0;                 // binlogDump失败异常计数
+    private int                  dumpErrorCount                    = 0;        // binlogDump失败异常计数
-    private int                  dumpErrorCountThreshold           = 2;                 // binlogDump失败异常计数阀值
+    private int                  dumpErrorCountThreshold           = 2;        // binlogDump失败异常计数阀值
-
-    // instance received binlog bytes
-    private final AtomicLong     receivedBinlogBytes               = new AtomicLong(0L);
 
     protected ErosaConnection buildErosaConnection() {
         return buildMysqlConnection(this.runningInfo);
@@ -907,8 +903,4 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
         this.dumpErrorCountThreshold = dumpErrorCountThreshold;
     }
 
-    public AtomicLong getReceivedBinlogBytes() {
-        return this.receivedBinlogBytes;
-    }
-
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java

@@ -2,11 +2,13 @@ package com.alibaba.otter.canal.parse.inbound.mysql;
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.LockSupport;
 
 import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
 import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
 import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer;
@@ -35,6 +37,7 @@ import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
 import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
 
+
 /**
  * 针对解析器提供一个多阶段协同的处理
  * 
@@ -63,6 +66,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
     private String                       destination;
     private volatile CanalParseException exception;
     private AtomicLong                   eventsPublishBlockingTime;
+    private GTIDSet                      gtidSet;
+    private WorkerPool<MessageEvent>     workerPool;
+    private BatchEventProcessor<MessageEvent> simpleParserStage;
+    private BatchEventProcessor<MessageEvent> sinkStoreStage;
 
     public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
                                       EventTransactionBuffer transactionBuffer, String destination){
@@ -81,15 +88,15 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
             ringBufferSize,
             new BlockingWaitStrategy());
         int tc = parserThreadCount > 0 ? parserThreadCount : 1;
-        this.parserExecutor = Executors.newFixedThreadPool(tc,
+        this.parserExecutor = Executors.newFixedThreadPool(tc, new NamedThreadFactory("MultiStageCoprocessor-Parser-"
-            new NamedThreadFactory("MultiStageCoprocessor-Parser-" + destination));
+                                                                                      + destination));
 
         this.stageExecutor = Executors.newFixedThreadPool(2, new NamedThreadFactory("MultiStageCoprocessor-other-"
                                                                                     + destination));
         SequenceBarrier sequenceBarrier = disruptorMsgBuffer.newBarrier();
         ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
         // stage 2
-        BatchEventProcessor<MessageEvent> simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
+        simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
             sequenceBarrier,
             new SimpleParserStage());
         simpleParserStage.setExceptionHandler(exceptionHandler);
@@ -101,7 +108,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         for (int i = 0; i < parserThreadCount; i++) {
             workHandlers[i] = new DmlParserStage();
         }
-        WorkerPool<MessageEvent> workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer,
+        workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer,
             dmlParserSequenceBarrier,
             exceptionHandler,
             workHandlers);
@@ -110,7 +117,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
         // stage 4
         SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence);
-        BatchEventProcessor<MessageEvent> sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
+        sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
             sinkSequenceBarrier,
             new SinkStoreStage());
         sinkStoreStage.setExceptionHandler(exceptionHandler);
@@ -124,14 +131,24 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
 
     @Override
     public void stop() {
+        // fix bug #968，对于pool与
+        workerPool.halt();
+        simpleParserStage.halt();
+        sinkStoreStage.halt();
         try {
             parserExecutor.shutdownNow();
+            while (!parserExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
+                parserExecutor.shutdownNow();
+            }
         } catch (Throwable e) {
             // ignore
         }
 
         try {
             stageExecutor.shutdownNow();
+            while (!stageExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
+                stageExecutor.shutdownNow();
+            }
         } catch (Throwable e) {
             // ignore
         }
@@ -227,6 +244,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         public SimpleParserStage(){
             decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
             context = new LogContext();
+            if (gtidSet != null) {
+                context.setGtidSet(gtidSet);
+            }
         }
 
         public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
@@ -447,4 +467,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
         this.eventsPublishBlockingTime = eventsPublishBlockingTime;
     }
 
+    public void setGtidSet(GTIDSet gtidSet) {
+        this.gtidSet = gtidSet;
+    }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java

@@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
 import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
-import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
 import com.alibaba.otter.canal.parse.exception.CanalParseException;
 import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
 import com.alibaba.otter.canal.parse.inbound.BinlogParser;
@@ -99,13 +98,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     private boolean                     filterRows          = false;
     private boolean                     useDruidDdlFilter   = true;
 
-    // latest gtid
-    private GTIDSet                     gtidSet;
-
-    public LogEventConvert(GTIDSet gtidSet){
-        this.gtidSet = gtidSet;
-    }
-
     public LogEventConvert(){
 
     }
@@ -172,13 +164,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
 
     private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
         LogHeader logHeader = logEvent.getHeader();
-        String value = logEvent.getSid().toString() + ":" + logEvent.getGno();
         Pair.Builder builder = Pair.newBuilder();
         builder.setKey("gtid");
-        builder.setValue(value);
+        builder.setValue(logEvent.getGtidStr());
-        if (gtidSet != null) {
+
-            gtidSet.update(value);
-        }
         if (logEvent.getLastCommitted() != null) {
             builder.setKey("lastCommitted");
             builder.setValue(String.valueOf(logEvent.getLastCommitted()));
@@ -556,7 +545,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
                 rowsCount++;
                 rowChangeBuider.addRowDatas(rowDataBuilder.build());
             }
-
             TableMapLogEvent table = event.getTable();
             Header header = createHeader(event.getHeader(),
                 table.getDbName(),
@@ -845,9 +833,24 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
         }
         headerBuilder.setEventLength(logHeader.getEventLen());
         // enable gtid position
-        if (gtidSet != null) {
+        if (StringUtils.isNotEmpty(logHeader.getGtidSetStr())) {
-            String gtid = gtidSet.toString();
+            headerBuilder.setGtid(logHeader.getGtidSetStr());
-            headerBuilder.setGtid(gtid);
+        }
+        // add current gtid
+        if (StringUtils.isNotEmpty(logHeader.getCurrentGtid())) {
+            Pair pair = createSpecialPair("curtGtid", logHeader.getCurrentGtid());
+            headerBuilder.addProps(pair);
+        }
+        // add current gtid sequence no
+        if (StringUtils.isNotEmpty(logHeader.getCurrentGtidSn())) {
+            Pair pair = createSpecialPair("curtGtidSn", logHeader.getCurrentGtidSn());
+            headerBuilder.addProps(pair);
+        }
+
+        // add current gtid last committed
+        if (StringUtils.isNotEmpty(logHeader.getCurrentGtidLastCommit())) {
+            Pair pair = createSpecialPair("curtGtidLct", logHeader.getCurrentGtidLastCommit());
+            headerBuilder.addProps(pair);
         }
 
         // add rowsCount suppport
@@ -977,8 +980,4 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
     public void setFilterRows(boolean filterRows) {
         this.filterRows = filterRows;
     }
-
-    public void setGtidSet(GTIDSet gtidSet) {
-        this.gtidSet = gtidSet;
-    }
 }

parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/SimpleDdlParser.java

@@ -44,8 +44,8 @@ public class SimpleDdlParser {
      * http://dev.mysql.com/doc/refman/5.6/en/create-index.html
      * </pre>
      */
-    public static final String CREATE_INDEX_PATTERN   = "^\\s*CREATE\\s*(UNIQUE)?(FULLTEXT)?(SPATIAL)?\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
+    public static final String CREATE_INDEX_PATTERN   = "^\\s*CREATE\\s*(UNIQUE)?(FULLTEXT)?(SPATIAL)?\\s*INDEX\\s*(.*?)\\s+ON\\s+(.*?)$";
-    public static final String DROP_INDEX_PATTERN     = "^\\s*DROP\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
+    public static final String DROP_INDEX_PATTERN     = "^\\s*DROP\\s*INDEX\\s*(.*?)\\s+ON\\s+(.*?)$";
 
     public static DdlResult parse(String queryString, String schmeaName) {
         queryString = removeComment(queryString); // 去除/* */的sql注释内容

parse/src/test/java/com/alibaba/otter/canal/parse/MysqlBinlogDumpPerformanceTest.java

@@ -20,12 +20,12 @@ public class MysqlBinlogDumpPerformanceTest {
 
     public static void main(String args[]) {
         final MysqlEventParser controller = new MysqlEventParser();
-        final EntryPosition startPosition = new EntryPosition("mysql-bin.001699", 120L, 100L);
+        final EntryPosition startPosition = new EntryPosition("mysql-bin.000007", 89796293L, 100L);
         controller.setConnectionCharset(Charset.forName("UTF-8"));
         controller.setSlaveId(3344L);
         controller.setDetectingEnable(false);
         controller.setFilterQueryDml(true);
-        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3328), "root", "hello"));
+        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("100.81.154.142", 3306), "canal", "canal"));
         controller.setMasterPosition(startPosition);
         controller.setEnableTsdb(false);
         controller.setDestination("example");

parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java

@@ -225,5 +225,19 @@ public class SimpleDdlParserTest {
         Assert.assertNotNull(result);
         Assert.assertEquals("retl", result.getSchemaName());
         Assert.assertEquals("retl_mark", result.getTableName());
+
+        // test index name contains 'on' -- version
+        queryString = "create index schema_new_index_version_s_idx on q_contract_account (contract_id,main_contract_id)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("q_contract_account", result.getTableName());
+
+        queryString = "drop index schema_new_index_version_s_idx on q_contract_account";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("q_contract_account", result.getTableName());
+
     }
 }

pom.xml

@@ -237,7 +237,7 @@
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-java</artifactId>
-                <version>2.6.1</version>
+                <version>3.6.1</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.ibatis</groupId>

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/EntryCollector.java

@@ -53,9 +53,9 @@ public class EntryCollector extends Collector implements InstanceRegistry {
         for (EntryMetricsHolder emh : instances.values()) {
             long now = System.currentTimeMillis();
             long latest = emh.latestExecTime.get();
-            if (now >= latest) {
+            // execTime > now，delay显示为0
-                delay.addMetric(emh.destLabelValues, (now - latest));
+            long d = (now >= latest) ? (now - latest) : 0;
-            }
+            delay.addMetric(emh.destLabelValues, d);
             transactions.addMetric(emh.destLabelValues, emh.transactionCounter.doubleValue());
         }
         mfs.add(delay);

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/ParserCollector.java

@@ -1,14 +1,15 @@
 package com.alibaba.otter.canal.prometheus.impl;
 
 import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST;
-import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST;
+
+import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.AbstractMysqlEventParser;
 import io.prometheus.client.Collector;
 import io.prometheus.client.CounterMetricFamily;
 import io.prometheus.client.GaugeMetricFamily;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -19,7 +20,6 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.instance.core.CanalInstance;
 import com.alibaba.otter.canal.parse.CanalEventParser;
-import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
 import com.alibaba.otter.canal.prometheus.InstanceRegistry;
 import com.google.common.base.Preconditions;
 
@@ -34,11 +34,13 @@ public class ParserCollector extends Collector implements InstanceRegistry {
     private static final String                              RECEIVED_BINLOG       = "canal_instance_received_binlog_bytes";
     private static final String                              PARSER_MODE           = "canal_instance_parser_mode";
     private static final String                              MODE_LABEL            = "parallel";
+    private static final String                              PARSER_LABEL          = "parser";
     private static final String                              PUBLISH_BLOCKING_HELP = "Publish blocking time of dump thread in milliseconds";
     private static final String                              RECEIVED_BINLOG_HELP  = "Received binlog bytes";
     private static final String                              MODE_HELP             = "Parser mode(parallel/serial) of instance";
     private final List<String>                               modeLabels            = Arrays.asList(DEST, MODE_LABEL);
-    private final ConcurrentMap<String, ParserMetricsHolder> instances             = new ConcurrentHashMap<String, ParserMetricsHolder>();
+    private final List<String>                               parserLabels          = Arrays.asList(DEST, PARSER_LABEL);
+    private final ConcurrentMap<String, ParserMetricsHolder> instances             = new ConcurrentHashMap<>();
 
     private ParserCollector() {}
 
@@ -52,53 +54,87 @@ public class ParserCollector extends Collector implements InstanceRegistry {
 
     @Override
     public List<MetricFamilySamples> collect() {
-        List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
+        List<MetricFamilySamples> mfs = new ArrayList<>();
-        boolean hasParallel = false;
         CounterMetricFamily bytesCounter = new CounterMetricFamily(RECEIVED_BINLOG,
-                RECEIVED_BINLOG_HELP, DEST_LABELS_LIST);
+                RECEIVED_BINLOG_HELP, parserLabels);
         GaugeMetricFamily modeGauge = new GaugeMetricFamily(PARSER_MODE,
                 MODE_HELP, modeLabels);
         CounterMetricFamily blockingCounter = new CounterMetricFamily(PUBLISH_BLOCKING,
-                PUBLISH_BLOCKING_HELP, DEST_LABELS_LIST);
+                PUBLISH_BLOCKING_HELP, parserLabels);
         for (ParserMetricsHolder emh : instances.values()) {
-            if (emh.isParallel) {
+            if (emh instanceof GroupParserMetricsHolder) {
-                blockingCounter.addMetric(emh.destLabelValues, (emh.eventsPublishBlockingTime.doubleValue() / NANO_PER_MILLI));
+                GroupParserMetricsHolder group = (GroupParserMetricsHolder) emh;
-                hasParallel = true;
+                for (ParserMetricsHolder semh :  group.holders) {
+                    singleCollect(bytesCounter, blockingCounter, modeGauge, semh);
+                }
+            }
+            else {
+                singleCollect(bytesCounter, blockingCounter, modeGauge, emh);
             }
-            modeGauge.addMetric(emh.modeLabelValues, 1);
-            bytesCounter.addMetric(emh.destLabelValues, emh.receivedBinlogBytes.doubleValue());
-
         }
         mfs.add(bytesCounter);
         mfs.add(modeGauge);
-        if (hasParallel) {
+        if (!blockingCounter.samples.isEmpty()) {
             mfs.add(blockingCounter);
         }
         return mfs;
     }
 
+    private void singleCollect(CounterMetricFamily bytesCounter, CounterMetricFamily blockingCounter, GaugeMetricFamily modeGauge, ParserMetricsHolder holder) {
+        if (holder.isParallel) {
+            blockingCounter.addMetric(holder.parserLabelValues, (holder.eventsPublishBlockingTime.doubleValue() / NANO_PER_MILLI));
+        }
+        modeGauge.addMetric(holder.modeLabelValues, 1);
+        bytesCounter.addMetric(holder.parserLabelValues, holder.receivedBinlogBytes.doubleValue());
+    }
+
     @Override
     public void register(CanalInstance instance) {
         final String destination = instance.getDestination();
-        ParserMetricsHolder holder = new ParserMetricsHolder();
+        ParserMetricsHolder holder;
         CanalEventParser parser = instance.getEventParser();
-        if (!(parser instanceof MysqlEventParser)) {
+        if (parser instanceof AbstractMysqlEventParser) {
-            throw new IllegalArgumentException("CanalEventParser must be MysqlEventParser");
+            holder = singleHolder(destination, (AbstractMysqlEventParser)parser, "0");
+        } else if (parser instanceof GroupEventParser) {
+            holder = groupHolder(destination, (GroupEventParser)parser);
+        } else {
+            throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
         }
-        MysqlEventParser mysqlParser = (MysqlEventParser) parser;
+        Preconditions.checkNotNull(holder);
-        holder.destLabelValues = Collections.singletonList(destination);
-        holder.modeLabelValues = Arrays.asList(destination, Boolean.toString(mysqlParser.isParallel()));
-        holder.eventsPublishBlockingTime = mysqlParser.getEventsPublishBlockingTime();
-        holder.receivedBinlogBytes = mysqlParser.getReceivedBinlogBytes();
-        holder.isParallel = mysqlParser.isParallel();
-        Preconditions.checkNotNull(holder.eventsPublishBlockingTime);
-        Preconditions.checkNotNull(holder.receivedBinlogBytes);
         ParserMetricsHolder old = instances.put(destination, holder);
         if (old != null) {
             logger.warn("Remove stale ParserCollector for instance {}.", destination);
         }
     }
 
+    private ParserMetricsHolder singleHolder(String destination, AbstractMysqlEventParser parser, String id) {
+        ParserMetricsHolder holder = new ParserMetricsHolder();
+        holder.parserLabelValues = Arrays.asList(destination, id);
+        holder.modeLabelValues = Arrays.asList(destination, Boolean.toString(parser.isParallel()));
+        holder.eventsPublishBlockingTime = parser.getEventsPublishBlockingTime();
+        holder.receivedBinlogBytes = parser.getReceivedBinlogBytes();
+        holder.isParallel = parser.isParallel();
+        Preconditions.checkNotNull(holder.eventsPublishBlockingTime);
+        Preconditions.checkNotNull(holder.receivedBinlogBytes);
+        return holder;
+    }
+
+    private GroupParserMetricsHolder groupHolder(String destination, GroupEventParser group) {
+        List<CanalEventParser> parsers = group.getEventParsers();
+        GroupParserMetricsHolder groupHolder = new GroupParserMetricsHolder();
+        int num = parsers.size();
+        for (int i = 0; i < num; i ++) {
+            CanalEventParser parser = parsers.get(i);
+            if (parser instanceof AbstractMysqlEventParser) {
+                ParserMetricsHolder single = singleHolder(destination, (AbstractMysqlEventParser)parser, Integer.toString(i + 1));
+                groupHolder.holders.add(single);
+            } else {
+                logger.warn("Null or non AbstractMysqlEventParser, ignore.");
+            }
+        }
+        return groupHolder;
+    }
+
     @Override
     public void unregister(CanalInstance instance) {
         final String destination = instance.getDestination();
@@ -106,11 +142,17 @@ public class ParserCollector extends Collector implements InstanceRegistry {
     }
 
     private class ParserMetricsHolder {
-        private List<String> destLabelValues;
+        private List<String> parserLabelValues;
         private List<String> modeLabelValues;
+        // metrics for single parser
         private AtomicLong   receivedBinlogBytes;
         private AtomicLong   eventsPublishBlockingTime;
+        // parser mode
         private boolean      isParallel;
     }
 
+    private class GroupParserMetricsHolder extends ParserMetricsHolder {
+        private final List<ParserMetricsHolder> holders = new ArrayList<>();
+    }
+
 }

prometheus/src/main/java/com/alibaba/otter/canal/prometheus/impl/StoreCollector.java

@@ -99,15 +99,13 @@ public class StoreCollector extends Collector implements InstanceRegistry {
             long get = Math.min(smh.getExecTime.get(), pet);
             long aet = Math.min(smh.ackExecTime.get(), get);
             long now = System.currentTimeMillis();
-            if (now >= pet) {
+            // execTime > now，delay显示为0
-                putDelay.addMetric(smh.destLabelValues, (now - pet));
+            long pd = (now >= pet) ? (now - pet) : 0;
-            }
+            putDelay.addMetric(smh.destLabelValues, pd);
-            if (now >= get) {
+            long gd = (now >= get) ? (now - get) : 0;
-                getDelay.addMetric(smh.destLabelValues, (now - get));
+            getDelay.addMetric(smh.destLabelValues, gd);
-            }
+            long ad = (now >= aet) ? (now - aet) : 0;
-            if (now >= aet) {
+            ackDelay.addMetric(smh.destLabelValues, ad);
-                ackDelay.addMetric(smh.destLabelValues, (now - aet));
-            }
             putRows.addMetric(smh.destLabelValues, smh.putTableRows.doubleValue());
             getRows.addMetric(smh.destLabelValues, smh.getTableRows.doubleValue());
             ackRows.addMetric(smh.destLabelValues, smh.ackTableRows.doubleValue());

protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalProtocol.proto

@@ -1,3 +1,4 @@
+syntax = "proto3";
 package com.alibaba.otter.canal.protocol;
 
 option java_package = "com.alibaba.otter.canal.protocol";
@@ -5,6 +6,7 @@ option java_outer_classname = "CanalPacket";
 option optimize_for = SPEED;
 
 enum Compression {
+    COMPRESSIONCOMPATIBLEPROTO2 = 0;
     NONE = 1;
     ZLIB = 2;
     GZIP = 3;
@@ -12,6 +14,8 @@ enum Compression {
 }
 
 enum PacketType {
+    //compatible
+    PACKAGETYPECOMPATIBLEPROTO2 = 0;
     HANDSHAKE = 1;
     CLIENTAUTHENTICATION = 2;
     ACK = 3;
@@ -29,86 +33,122 @@ enum PacketType {
 }
 
 message Packet {
-    optional int32 magic_number = 1 [default = 17];
+     //[default = 17];
-    optional int32 version = 2 [default = 1];
+     oneof magic_number_present {
-    optional PacketType type = 3;
+         int32 magic_number = 1;
-    optional Compression compression = 4 [default = NONE];
+     }
-    optional bytes body = 5;
+     //[default = 1];
+     oneof version_present {
+          int32 version = 2;
+     };
+     PacketType type = 3;
+     //[default = NONE];
+     oneof compression_present {
+          Compression compression = 4;
+     }
+
+     bytes body = 5;
 }
 
 message HeartBeat {
-    optional int64 send_timestamp = 1;
+     int64 send_timestamp = 1;
-    optional int64 start_timestamp = 2;
+     int64 start_timestamp = 2;
 }
 
 message Handshake {
-    optional string communication_encoding = 1 [default = "utf8"];
+    //  [default = "utf8"];
-    optional bytes seeds = 2;
+    oneof communication_encoding_present {
-    repeated Compression supported_compressions = 3;
+        string communication_encoding = 1;
+    }
+     bytes seeds = 2;
+     Compression supported_compressions = 3;
 }
 
 // client authentication
 message ClientAuth {
-    optional string username = 1;
+    string username = 1;
-    optional bytes password = 2; // hashed password with seeds from Handshake message
+    bytes password = 2; // hashed password with seeds from Handshake message
-    optional int32 net_read_timeout = 3 [default = 0]; // in seconds
+    // [default = 0]
-    optional int32 net_write_timeout = 4 [default = 0]; // in seconds
+    oneof net_read_timeout_present {
-    optional string destination = 5;
+         int32 net_read_timeout = 3; // in seconds
-    optional string client_id = 6;
+    }
-    optional string filter = 7;
+    // [default = 0];
-    optional int64 start_timestamp = 8;
+    oneof net_write_timeout_present {
+        int32 net_write_timeout = 4; // in seconds
+    }
+    string destination = 5;
+    string client_id = 6;
+    string filter = 7;
+    int64 start_timestamp = 8;
 }
 
 message Ack {
-    optional int32 error_code = 1 [default = 0];
+    //[default = 0]
-    optional string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
+    oneof error_code_present {
+        int32 error_code = 1;
+    }
+    string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
 }
 
 message ClientAck {
-    optional string destination = 1;
+    string destination = 1;
-    optional string client_id = 2;
+    string client_id = 2;
-    optional int64 batch_id = 3;
+    int64 batch_id = 3;
 }
 
 // subscription
 message Sub {
-    optional string destination = 1;
+    string destination = 1;
-    optional string client_id = 2;
+    string client_id = 2;
-    optional string filter = 7;
+    string filter = 7;
 }
 
 // Unsubscription
 message Unsub {
-    optional string destination = 1;
+    string destination = 1;
-    optional string client_id = 2;
+    string client_id = 2;
-    optional string filter = 7;
+    string filter = 7;
 }
 
 //  PullRequest
 message Get {
-    optional string destination = 1;
+    string destination = 1;
-    optional string client_id = 2;
+    string client_id = 2;
-    optional int32 fetch_size = 3;
+    int32 fetch_size = 3;
-    optional int64 timeout = 4 [default = -1]; // 默认-1时代表不控制
+    //[default = -1]
-    optional int32 unit = 5 [default = 2];// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
+    oneof timeout_present {
-    optional bool auto_ack = 6 [default = false]; // 是否自动ack
+        int64 timeout = 4; // 默认-1时代表不控制
+    }
+    //[default = 2]
+    oneof unit_present {
+        int32 unit = 5;// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
+    }
+    //[default = false]
+    oneof auto_ack_present {
+        bool auto_ack = 6; // 是否自动ack
+    }
+
 }
 
 //
 message Messages {
-	optional int64 batch_id = 1;
+	int64 batch_id = 1;
     repeated bytes messages = 2;
 }
 
 // TBD when new packets are required
 message Dump{
-    optional string journal = 1;
+    string journal = 1;
-    optional int64  position = 2;
+    int64  position = 2;
-    optional int64 timestamp = 3 [default = 0];
+    // [default = 0]
+    oneof timestamp_present {
+        int64 timestamp = 3;
+    }
+
 }
 
 message ClientRollback{
-    optional string destination = 1;
+    string destination = 1;
-    optional string client_id = 2;
+    string client_id = 2;
-    optional int64 batch_id = 3;
+    int64 batch_id = 3;
 }

protocol/src/main/java/com/alibaba/otter/canal/protocol/EntryProtocol.proto

@@ -1,3 +1,4 @@
+syntax = "proto3";
 package com.alibaba.otter.canal.protocol;
 
 option java_package = "com.alibaba.otter.canal.protocol";
@@ -10,88 +11,101 @@ option optimize_for = SPEED;
  ****************************************************************/
 message Entry {
 	/**协议头部信息**/
-	optional Header						header 				= 1;
+     Header						header 				= 1;
-	
+	///**打散后的事件类型**/ [default = ROWDATA]
-	/**打散后的事件类型**/
+	oneof entryType_present{
-	optional EntryType					entryType			= 2 [default = ROWDATA];
+		EntryType					entryType			= 2;
+	}
 
 	/**传输的二进制数组**/
-	optional bytes						storeValue			= 3;
+	bytes						storeValue			= 3;
 }
 
 /**message Header**/
 message Header {
-	/**协议的版本号**/
+	/**协议的版本号**/  //[default = 1]
-	optional int32 					version				= 1 [default = 1];	
+	oneof version_present {
+		int32 					version				= 1;
+	}
+
 
 	/**binlog/redolog 文件名**/
-	optional string					logfileName			= 2;
+	string					logfileName			= 2;
 
 	/**binlog/redolog 文件的偏移位置**/
-	optional int64 					logfileOffset		= 3;
+	int64 					logfileOffset		= 3;
 
 	/**服务端serverId**/
-	optional 	int64				serverId         	= 4;	
+	int64				serverId         	= 4;
 
 	/** 变更数据的编码 **/
-	optional string					serverenCode		= 5;
+	string					serverenCode		= 5;
 
 	/**变更数据的执行时间 **/
-	optional int64					executeTime			= 6;
+	int64					executeTime			= 6;
+
+	/** 变更数据的来源**/ //[default = MYSQL]
+	oneof sourceType_present {
+		Type					sourceType			= 7;
+	}
 
-	/** 变更数据的来源**/
-	optional Type					sourceType			= 7 [default = MYSQL];
 
 	/** 变更数据的schemaname**/
-	optional string					schemaName			= 8;
+	string					schemaName			= 8;
 
 	/**变更数据的tablename**/
-	optional string					tableName			= 9;	
+	string					tableName			= 9;
 
 	/**每个event的长度**/
-	optional int64					eventLength         = 10;
+	int64					eventLength         = 10;
+
+	/**数据变更类型**/  // [default = UPDATE]
+	oneof eventType_present {
+		EventType 				eventType			= 11;
+	}
 
-	/**数据变更类型**/
-	optional EventType 				eventType			= 11 [default = UPDATE];
 
 	/**预留扩展**/
 	repeated Pair					props				= 12;
 
     /**当前事务的gitd**/
-	optional string                 gtid                = 13;
+	string                 gtid                = 13;
 }
 
 /**每个字段的数据结构**/
 message Column {
 	/**字段下标**/
-	optional int32		index			= 		1;
+	int32		index			= 		1;
 
 	/**字段java中类型**/
-	optional int32 		sqlType			= 		2;
+	int32 		sqlType			= 		2;
 
 	/**字段名称(忽略大小写)，在mysql中是没有的**/
-	optional string		name			=		3;
+	string		name			=		3;
 
 	/**是否是主键**/
-	optional bool 		isKey			= 		4;
+	bool 		isKey			= 		4;
 
 	/**如果EventType=UPDATE,用于标识这个字段值是否有修改**/
-	optional bool		updated			= 		5;
+	bool		updated			= 		5;
+
+	/** 标识是否为空  **/ //[default = false]
+	oneof isNull_present {
+		bool		isNull			= 		6;
+	}
 
-	/** 标识是否为空  **/
-	optional bool		isNull			= 		6 [default = false];
 
 	/**预留扩展**/
 	repeated Pair		props			=		7;
 
 	/** 字段值,timestamp,Datetime是一个时间格式的文本 **/
-	optional string		value			= 		8;
+	string		value			= 		8;
 
 	/** 对应数据对象原始长度 **/
-	optional int32		length			= 		9;
+	int32		length			= 		9;
 
 	/**字段mysql类型**/
-	optional string		mysqlType		= 		10;
+	string		mysqlType		= 		10;
 }
 
 message RowData {
@@ -110,16 +124,23 @@ message RowData {
 message RowChange {
 
 	/**tableId,由数据库产生**/
-	optional int64	 		tableId			=		1;
+	int64	 		tableId			=		1;
 
-	/**数据变更类型**/
-	optional EventType 		eventType		= 		2 [default = UPDATE];
 
-	/** 标识是否是ddl语句  **/
+	/**数据变更类型**/ //[default = UPDATE]
-	optional bool			isDdl			= 		10 [default = false];
+	oneof eventType_present {
+		EventType 		eventType		= 		2;
+	}
+
+
+	/** 标识是否是ddl语句  **/ // [default = false]
+	oneof isDdl_present {
+		bool			isDdl			= 		10;
+	}
+
 
 	/** ddl/query的sql语句  **/
-	optional string			sql 			= 		11;
+	string			sql 			= 		11;
 
 	/** 一次数据库变更可能存在多行  **/
 	repeated RowData		rowDatas		= 		12;
@@ -128,33 +149,33 @@ message RowChange {
 	repeated Pair			props			=		13;
 
 	/** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  **/
-	optional string			ddlSchemaName 	= 		14;
+	string			ddlSchemaName 	= 		14;
 }
 
 /**开始事务的一些信息**/
 message TransactionBegin{
 
 	/**已废弃，请使用header里的executeTime**/
-	optional int64			executeTime		=		1;
+	int64			executeTime		=		1;
 
 	/**已废弃，Begin里不提供事务id**/
-	optional string			transactionId	=		2;
+	string			transactionId	=		2;
 
 	/**预留扩展**/
 	repeated Pair			props			=		3;
 
 	/**执行的thread Id**/
-	optional int64			threadId		=		4;
+	int64			threadId		=		4;
 }
 
 /**结束事务的一些信息**/
 message TransactionEnd{
 
 	/**已废弃，请使用header里的executeTime**/
-	optional int64			executeTime		=		1;
+	int64			executeTime		=		1;
 
 	/**事务号**/
-	optional string			transactionId	=		2;
+	string			transactionId	=		2;
 
 	/**预留扩展**/
 	repeated Pair			props			=		3;
@@ -162,12 +183,13 @@ message TransactionEnd{
 
 /**预留扩展**/
 message Pair{
-	optional string 		key				= 			1;	
+	string 		key				= 			1;
-	optional string 		value			= 			2;	
+	string 		value			= 			2;
 }
 
 /**打散后的事件类型，主要用于标识事务的开始，变更数据，结束**/
 enum EntryType{
+	ENTRYTYPECOMPATIBLEPROTO2 = 0;
 	TRANSACTIONBEGIN 		=		1;
 	ROWDATA					=		2;
 	TRANSACTIONEND			=		3;
@@ -178,6 +200,7 @@ enum EntryType{
 
 /** 事件类型 **/
 enum EventType {
+	EVENTTYPECOMPATIBLEPROTO2 = 0;
     INSERT 		= 		1;
     UPDATE 		= 		2;
     DELETE 		= 		3;
@@ -200,6 +223,7 @@ enum EventType {
 
 /**数据库类型**/
 enum Type {
+	TYPECOMPATIBLEPROTO2 = 0;
     ORACLE		= 		1;
     MYSQL		= 		2;
     PGSQL		= 		3;

protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java

@@ -0,0 +1,304 @@
+package com.alibaba.otter.canal.protocol;
+
+import java.io.Serializable;
+import java.util.*;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * @author machengyuan 2018-9-13 下午10:31:14
+ * @version 1.0.0
+ */
+public class FlatMessage implements Serializable {
+
+    private static final long         serialVersionUID = -3386650678735860050L;
+
+    private long                      id;
+    private String                    database;
+    private String                    table;
+    private Boolean                   isDdl;
+    private String                    type;
+    private Long                      ts;
+    private String                    sql;
+    private Map<String, Integer>      sqlType;
+    private Map<String, String>       mysqlType;
+    private List<Map<String, String>> data;
+    private List<Map<String, String>> old;
+
+    public FlatMessage(){
+    }
+
+    public FlatMessage(long id){
+        this.id = id;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public Boolean getIsDdl() {
+        return isDdl;
+    }
+
+    public void setIsDdl(Boolean isDdl) {
+        this.isDdl = isDdl;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Long getTs() {
+        return ts;
+    }
+
+    public void setTs(Long ts) {
+        this.ts = ts;
+    }
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    public Map<String, Integer> getSqlType() {
+        return sqlType;
+    }
+
+    public void setSqlType(Map<String, Integer> sqlType) {
+        this.sqlType = sqlType;
+    }
+
+    public Map<String, String> getMysqlType() {
+        return mysqlType;
+    }
+
+    public void setMysqlType(Map<String, String> mysqlType) {
+        this.mysqlType = mysqlType;
+    }
+
+    public List<Map<String, String>> getData() {
+        return data;
+    }
+
+    public void setData(List<Map<String, String>> data) {
+        this.data = data;
+    }
+
+    public List<Map<String, String>> getOld() {
+        return old;
+    }
+
+    public void setOld(List<Map<String, String>> old) {
+        this.old = old;
+    }
+
+    /**
+     * 将Message转换为FlatMessage
+     * 
+     * @param message 原生message
+     * @return FlatMessage列表
+     */
+    public static List<FlatMessage> messageConverter(Message message) {
+        try {
+            if (message == null) {
+                return null;
+            }
+
+            List<FlatMessage> flatMessages = new ArrayList<>();
+
+            List<ByteString> rawEntries = message.getRawEntries();
+
+            for (ByteString byteString : rawEntries) {
+                CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
+                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                    continue;
+                }
+
+                CanalEntry.RowChange rowChange;
+                try {
+                    rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+                } catch (Exception e) {
+                    throw new RuntimeException(
+                        "ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
+                        e);
+                }
+
+                CanalEntry.EventType eventType = rowChange.getEventType();
+
+                FlatMessage flatMessage = new FlatMessage(message.getId());
+                flatMessages.add(flatMessage);
+                flatMessage.setDatabase(entry.getHeader().getSchemaName());
+                flatMessage.setTable(entry.getHeader().getTableName());
+                flatMessage.setIsDdl(rowChange.getIsDdl());
+                flatMessage.setType(eventType.toString());
+                flatMessage.setTs(System.currentTimeMillis());
+                flatMessage.setSql(rowChange.getSql());
+
+                if (!rowChange.getIsDdl()) {
+                    Map<String, Integer> sqlType = new LinkedHashMap<>();
+                    Map<String, String> mysqlType = new LinkedHashMap<>();
+                    List<Map<String, String>> data = new ArrayList<>();
+                    List<Map<String, String>> old = new ArrayList<>();
+
+                    Set<String> updateSet = new HashSet<>();
+                    for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                        if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                            && eventType != CanalEntry.EventType.DELETE) {
+                            continue;
+                        }
+
+                        Map<String, String> row = new LinkedHashMap<>();
+                        List<CanalEntry.Column> columns;
+
+                        if (eventType == CanalEntry.EventType.DELETE) {
+                            columns = rowData.getBeforeColumnsList();
+                        } else {
+                            columns = rowData.getAfterColumnsList();
+                        }
+
+                        for (CanalEntry.Column column : columns) {
+                            sqlType.put(column.getName(), column.getSqlType());
+                            mysqlType.put(column.getName(), column.getMysqlType());
+                            row.put(column.getName(), column.getValue());
+                            // 获取update为true的字段
+                            if (column.getUpdated()) {
+                                updateSet.add(column.getName());
+                            }
+                        }
+                        if (!row.isEmpty()) {
+                            data.add(row);
+                        }
+
+                        if (eventType == CanalEntry.EventType.UPDATE) {
+                            Map<String, String> rowOld = new LinkedHashMap<>();
+                            for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                                if (updateSet.contains(column.getName())) {
+                                    rowOld.put(column.getName(), column.getValue());
+                                }
+                            }
+                            // update操作将记录修改前的值
+                            if (!rowOld.isEmpty()) {
+                                old.add(rowOld);
+                            }
+                        }
+                    }
+                    if (!sqlType.isEmpty()) {
+                        flatMessage.setSqlType(sqlType);
+                    }
+                    if (!mysqlType.isEmpty()) {
+                        flatMessage.setMysqlType(mysqlType);
+                    }
+                    if (!data.isEmpty()) {
+                        flatMessage.setData(data);
+                    }
+                    if (!old.isEmpty()) {
+                        flatMessage.setOld(old);
+                    }
+                }
+            }
+            return flatMessages;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * 将FlatMessage按指定的字段值hash拆分
+     * 
+     * @param flatMessage flatMessage
+     * @param partitionsNum 分区数量
+     * @param pkHashConfig hash映射
+     * @return 拆分后的flatMessage数组
+     */
+    public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum,
+                                                 Map<String, String> pkHashConfig) {
+        if (partitionsNum == null) {
+            partitionsNum = 1;
+        }
+        FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
+
+        String pk = pkHashConfig.get(flatMessage.getDatabase() + "." + flatMessage.getTable());
+        if (pk == null || flatMessage.getIsDdl()) {
+            partitionMessages[0] = flatMessage;
+        } else {
+            if (flatMessage.getData() != null) {
+                int idx = 0;
+                for (Map<String, String> row : flatMessage.getData()) {
+                    String value = row.get(pk);
+                    if (value == null) {
+                        value = "";
+                    }
+                    int hash = value.hashCode();
+                    int pkHash = Math.abs(hash) % partitionsNum;
+                    // math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
+                    pkHash = Math.abs(pkHash);
+
+                    FlatMessage flatMessageTmp = partitionMessages[pkHash];
+                    if (flatMessageTmp == null) {
+                        flatMessageTmp = new FlatMessage(flatMessage.getId());
+                        partitionMessages[pkHash] = flatMessageTmp;
+                        flatMessageTmp.setDatabase(flatMessage.getDatabase());
+                        flatMessageTmp.setTable(flatMessage.getTable());
+                        flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
+                        flatMessageTmp.setType(flatMessage.getType());
+                        flatMessageTmp.setSql(flatMessage.getSql());
+                        flatMessageTmp.setSqlType(flatMessage.getSqlType());
+                        flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
+                    }
+                    List<Map<String, String>> data = flatMessageTmp.getData();
+                    if (data == null) {
+                        data = new ArrayList<>();
+                        flatMessageTmp.setData(data);
+                    }
+                    data.add(row);
+                    if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
+                        List<Map<String, String>> old = flatMessageTmp.getOld();
+                        if (old == null) {
+                            old = new ArrayList<>();
+                            flatMessageTmp.setOld(old);
+                        }
+                        old.add(flatMessage.getOld().get(idx));
+                    }
+                    idx++;
+                }
+            }
+        }
+        return partitionMessages;
+    }
+
+    @Override
+    public String toString() {
+        return "FlatMessage{" + "id=" + id + ", database='" + database + '\'' + ", table='" + table + '\'' + ", isDdl="
+               + isDdl + ", type='" + type + '\'' + ", ts=" + ts + ", sql='" + sql + '\'' + ", sqlType=" + sqlType
+               + ", mysqlType=" + mysqlType + ", data=" + data + ", old=" + old + '}';
+    }
+}

server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.common;
 
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
 import com.alibaba.otter.canal.protocol.Message;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.CodedOutputStream;
@@ -31,14 +32,14 @@ public class CanalMessageSerializer {
                         // packet size
                         int size = 0;
                         size += CodedOutputStream.computeEnumSize(3,
-                            CanalPacket.PacketType.MESSAGES.getNumber());
+                            PacketType.MESSAGES.getNumber());
                         size += CodedOutputStream.computeTagSize(5)
                             + CodedOutputStream.computeRawVarint32Size(messageSize)
                             + messageSize;
                         // build data
                         byte[] body = new byte[size];
                         CodedOutputStream output = CodedOutputStream.newInstance(body);
-                        output.writeEnum(3, CanalPacket.PacketType.MESSAGES.getNumber());
+                        output.writeEnum(3, PacketType.MESSAGES.getNumber());
 
                         output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
                         output.writeRawVarint32(messageSize);
@@ -56,7 +57,7 @@ public class CanalMessageSerializer {
                         }
 
                         CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
-                        packetBuilder.setType(CanalPacket.PacketType.MESSAGES);
+                        packetBuilder.setType(PacketType.MESSAGES);
                         packetBuilder.setBody(messageBuilder.build().toByteString());
                         return packetBuilder.build().toByteArray();
                     }

server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java

@@ -1,9 +1,8 @@
 package com.alibaba.otter.canal.common;
 
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
 
 /**
  * kafka 配置项
@@ -13,23 +12,26 @@ import java.util.Set;
  */
 public class MQProperties {
 
-    private String                 servers                = "localhost:6667";
+    private String servers = "localhost:6667";
-    private int                    retries                = 0;
+    private int retries = 0;
-    private int                    batchSize              = 16384;
+    private int batchSize = 16384;
-    private int                    lingerMs               = 1;
+    private int lingerMs = 1;
-    private long                   bufferMemory           = 33554432L;
+    private long bufferMemory = 33554432L;
-    private boolean                filterTransactionEntry = true;
+    private boolean filterTransactionEntry = true;
-    private int                    canalBatchSize         = 5;
+    private String producerGroup = "Canal-Producer";
-    private String                 producerGroup          = "Canal-Producer";
+    private int canalBatchSize = 50;
+    private Long canalGetTimeout;
+    private boolean flatMessage = true;
 
-    private List<CanalDestination> canalDestinations      = new ArrayList<CanalDestination>();
+    private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
 
     public static class CanalDestination {
 
-        private String     canalDestination;
+        private String canalDestination;
-        private String     topic;
+        private String topic;
-        private Integer    partition;
+        private Integer partition;
-        private Set<Topic> topics = new HashSet<Topic>();
+        private Integer partitionsNum;
+        private Map<String, String> partitionHash;
 
         public String getCanalDestination() {
             return canalDestination;
@@ -55,54 +57,23 @@ public class MQProperties {
             this.partition = partition;
         }
 
-        public Set<Topic> getTopics() {
+        public Integer getPartitionsNum() {
-            return topics;
+            return partitionsNum;
         }
 
-        public void setTopics(Set<Topic> topics) {
+        public void setPartitionsNum(Integer partitionsNum) {
-            this.topics = topics;
+            this.partitionsNum = partitionsNum;
+        }
+
+        public Map<String, String> getPartitionHash() {
+            return partitionHash;
+        }
+
+        public void setPartitionHash(Map<String, String> partitionHash) {
+            this.partitionHash = partitionHash;
         }
     }
 
-    public static class Topic {
-
-        private String  topic;
-        private Integer partition;
-
-        public String getTopic() {
-            return topic;
-        }
-
-        public void setTopic(String topic) {
-            this.topic = topic;
-        }
-
-        public Integer getPartition() {
-            return partition;
-        }
-
-        public void setPartition(Integer partition) {
-            this.partition = partition;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            Topic topic1 = (Topic) o;
-
-            if (topic != null ? !topic.equals(topic1.topic) : topic1.topic != null) return false;
-            return partition != null ? partition.equals(topic1.partition) : topic1.partition == null;
-        }
-
-        @Override
-        public int hashCode() {
-            int result = topic != null ? topic.hashCode() : 0;
-            result = 31 * result + (partition != null ? partition.hashCode() : 0);
-            return result;
-        }
-    }
 
     public String getServers() {
         return servers;
@@ -152,6 +123,22 @@ public class MQProperties {
         this.canalBatchSize = canalBatchSize;
     }
 
+    public Long getCanalGetTimeout() {
+        return canalGetTimeout;
+    }
+
+    public void setCanalGetTimeout(Long canalGetTimeout) {
+        this.canalGetTimeout = canalGetTimeout;
+    }
+
+    public boolean getFlatMessage() {
+        return flatMessage;
+    }
+
+    public void setFlatMessage(boolean flatMessage) {
+        this.flatMessage = flatMessage;
+    }
+
     public List<CanalDestination> getCanalDestinations() {
         return canalDestinations;
     }

server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java

@@ -1,10 +1,12 @@
 package com.alibaba.otter.canal.kafka;
 
+import com.alibaba.fastjson.JSON;
 import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.spi.CanalMQProducer;
-import java.io.IOException;
+import java.util.List;
 import java.util.Properties;
-
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
@@ -12,8 +14,6 @@ import org.apache.kafka.common.serialization.StringSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.protocol.Message;
-
 /**
  * kafka producer 主操作类
  *
@@ -26,8 +26,13 @@ public class CanalKafkaProducer implements CanalMQProducer {
 
     private Producer<String, Message> producer;
 
+    private Producer<String, String> producer2;                                                 // 用于扁平message的数据投递
+
+    private MQProperties kafkaProperties;
+
     @Override
     public void init(MQProperties kafkaProperties) {
+        this.kafkaProperties = kafkaProperties;
         Properties properties = new Properties();
         properties.put("bootstrap.servers", kafkaProperties.getServers());
         properties.put("acks", "all");
@@ -36,15 +41,27 @@ public class CanalKafkaProducer implements CanalMQProducer {
         properties.put("linger.ms", kafkaProperties.getLingerMs());
         properties.put("buffer.memory", kafkaProperties.getBufferMemory());
         properties.put("key.serializer", StringSerializer.class.getName());
-        properties.put("value.serializer", MessageSerializer.class.getName());
+        if (!kafkaProperties.getFlatMessage()) {
-        producer = new KafkaProducer<String, Message>(properties);
+            properties.put("value.serializer", MessageSerializer.class.getName());
+            producer = new KafkaProducer<String, Message>(properties);
+        } else {
+            properties.put("value.serializer", StringSerializer.class.getName());
+            producer2 = new KafkaProducer<String, String>(properties);
+        }
+
+        // producer.initTransactions();
     }
 
     @Override
     public void stop() {
         try {
             logger.info("## stop the kafka producer");
-            producer.close();
+            if (producer != null) {
+                producer.close();
+            }
+            if (producer2 != null) {
+                producer2.close();
+            }
         } catch (Throwable e) {
             logger.warn("##something goes wrong when stopping kafka producer:", e);
         } finally {
@@ -53,16 +70,72 @@ public class CanalKafkaProducer implements CanalMQProducer {
     }
 
     @Override
-    public void send(MQProperties.Topic topic, Message message) throws IOException {
+    public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
-        ProducerRecord<String, Message> record;
+        try {
-        if (topic.getPartition() != null) {
+            // producer.beginTransaction();
-            record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
+            if (!kafkaProperties.getFlatMessage()) {
-        } else {
+                ProducerRecord<String, Message> record;
-            record = new ProducerRecord<String, Message>(topic.getTopic(), message);
+                if (canalDestination.getPartition() != null) {
-        }
+                    record = new ProducerRecord<String, Message>(canalDestination.getTopic(),
-        producer.send(record);
+                        canalDestination.getPartition(),
-        if (logger.isDebugEnabled()) {
+                        null,
-            logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
+                        message);
+                } else {
+                    record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message);
+                }
+
+                producer.send(record);
+            } else {
+                // 发送扁平数据json
+                List<FlatMessage> flatMessages = FlatMessage.messageConverter(message);
+                if (flatMessages != null) {
+                    for (FlatMessage flatMessage : flatMessages) {
+                        if (canalDestination.getPartition() != null) {
+                            ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination
+                                .getTopic(), canalDestination.getPartition(), null, JSON.toJSONString(flatMessage));
+                            producer2.send(record);
+                        } else {
+                            if (canalDestination.getPartitionHash() != null
+                                && !canalDestination.getPartitionHash().isEmpty()) {
+                                FlatMessage[] partitionFlatMessage = FlatMessage.messagePartition(flatMessage,
+                                    canalDestination.getPartitionsNum(),
+                                    canalDestination.getPartitionHash());
+                                int length = partitionFlatMessage.length;
+                                for (int i = 0; i < length; i++) {
+                                    FlatMessage flatMessagePart = partitionFlatMessage[i];
+                                    if (flatMessagePart != null) {
+                                        ProducerRecord<String, String> record = new ProducerRecord<String, String>(
+                                            canalDestination.getTopic(),
+                                            i,
+                                            null,
+                                            JSON.toJSONString(flatMessagePart));
+                                        producer2.send(record);
+                                    }
+                                }
+                            } else {
+                                ProducerRecord<String, String> record = new ProducerRecord<String, String>(
+                                    canalDestination.getTopic(),
+                                    0,
+                                    null,
+                                    JSON.toJSONString(flatMessage));
+                                producer2.send(record);
+                            }
+                        }
+
+                    }
+                }
+            }
+
+            // producer.commitTransaction();
+            callback.commit();
+            if (logger.isDebugEnabled()) {
+                logger.debug("send message to kafka topic: {}", canalDestination.getTopic());
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+            // producer.abortTransaction();
+            callback.rollback();
         }
     }
+
 }

server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java

@@ -1,20 +1,9 @@
 package com.alibaba.otter.canal.kafka;
 
 import com.alibaba.otter.canal.common.CanalMessageSerializer;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kafka.common.errors.SerializationException;
-import org.apache.kafka.common.serialization.Serializer;
-import org.springframework.util.CollectionUtils;
-
-import com.alibaba.otter.canal.protocol.CanalEntry;
-import com.alibaba.otter.canal.protocol.CanalPacket;
-import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
 import com.alibaba.otter.canal.protocol.Message;
-import com.google.protobuf.ByteString;
+import java.util.Map;
-import com.google.protobuf.CodedOutputStream;
+import org.apache.kafka.common.serialization.Serializer;
-import com.google.protobuf.WireFormat;
 
 /**
  * Kafka Message类的序列化

server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java

@@ -35,17 +35,24 @@ public class CanalRocketMQProducer implements CanalMQProducer {
     }
 
     @Override
-    public void send(final MQProperties.Topic topic, com.alibaba.otter.canal.protocol.Message data) {
+    public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
+        Callback callback) {
         try {
-            Message message = new Message(topic.getTopic(), CanalMessageSerializer.serializer(data));
+            Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data));
             this.defaultMQProducer.send(message, new MessageQueueSelector() {
                 @Override
                 public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
-                    return mqs.get(topic.getPartition());
+                    int partition = 0;
+                    if (destination.getPartition() != null) {
+                        partition = destination.getPartition();
+                    }
+                    return mqs.get(partition);
                 }
             }, null);
+            callback.commit();
         } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
             logger.error("Send message error!", e);
+            callback.rollback();
         }
     }
 

server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.server;
 
 import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
 import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
@@ -9,17 +10,18 @@ import java.io.FileInputStream;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
 public class CanalMQStarter {
-    private static final Logger logger               = LoggerFactory.getLogger(CanalMQStarter.class);
+    private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
 
     private static final String CLASSPATH_URL_PREFIX = "classpath:";
 
-    private volatile boolean    running              = false;
+    private volatile boolean running = false;
 
     private ExecutorService executorService;
 
@@ -27,7 +29,7 @@ public class CanalMQStarter {
 
     private MQProperties properties;
 
-    public CanalMQStarter(CanalMQProducer canalMQProducer){
+    public CanalMQStarter(CanalMQProducer canalMQProducer) {
         this.canalMQProducer = canalMQProducer;
     }
 
@@ -51,6 +53,7 @@ public class CanalMQStarter {
             if (properties.isFilterTransactionEntry()) {
                 System.setProperty("canal.instance.filter.transaction.entry", "true");
             }
+
             // 对应每个instance启动一个worker线程
             List<MQProperties.CanalDestination> destinations = properties.getCanalDestinations();
 
@@ -95,8 +98,8 @@ public class CanalMQStarter {
         while (!running)
             ;
         logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
-        CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
+        final CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
-        ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
+        final ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
         while (running) {
             try {
                 if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
@@ -111,27 +114,41 @@ public class CanalMQStarter {
                 logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination());
 
                 while (running) {
-                    Message message = server.getWithoutAck(clientIdentity, properties.getCanalBatchSize()); // 获取指定数量的数据
+                    Message message;
-                    long batchId = message.getId();
+                    if (properties.getCanalGetTimeout() != null) {
+                        message = server.getWithoutAck(clientIdentity,
+                            properties.getCanalBatchSize(),
+                            properties.getCanalGetTimeout(),
+                            TimeUnit.MILLISECONDS);
+                    } else {
+                        message = server.getWithoutAck(clientIdentity, properties.getCanalBatchSize());
+                    }
+
+                    final long batchId = message.getId();
                     try {
                         int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
                         if (batchId != -1 && size != 0) {
-                            if (!StringUtils.isEmpty(destination.getTopic())) {
+                            canalMQProducer.send(destination, message, new CanalKafkaProducer.Callback() {
-                                MQProperties.Topic topic = new MQProperties.Topic();
+
-                                topic.setTopic(destination.getTopic());
+                                @Override
-                                topic.setPartition(destination.getPartition());
+                                public void commit() {
-                                destination.getTopics().add(topic);
+                                    server.ack(clientIdentity, batchId); // 提交确认
-                            }
+                                }
-                            for (MQProperties.Topic topic : destination.getTopics()) {
+
-                                canalMQProducer.send(topic, message); // 发送message到所有topic
+                                @Override
+                                public void rollback() {
+                                    server.rollback(clientIdentity, batchId);
+                                }
+                            }); // 发送message到topic
+                        } else {
+                            try {
+                                Thread.sleep(100);
+                            } catch (InterruptedException e) {
+                                // ignore
                             }
                         }
 
-                        if (batchId != -1) {
-                            server.ack(clientIdentity, batchId); // 提交确认
-                        }
                     } catch (Exception e) {
-                        server.rollback(clientIdentity);
                         logger.error(e.getMessage(), e);
                     }
                 }

server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java

@@ -311,9 +311,10 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
             }
 
             if (CollectionUtils.isEmpty(events.getEvents())) {
-                logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result is null",
+                // logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result
-                    clientIdentity.getClientId(),
+                // is null",
-                    batchSize);
+                // clientIdentity.getClientId(),
+                // batchSize);
                 return new Message(-1, true, new ArrayList()); // 返回空包，避免生成batchId，浪费性能
             } else {
                 // 记录到流式信息

server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java

@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.server.netty.handler;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.jboss.netty.buffer.ChannelBuffer;
@@ -34,6 +33,7 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
 import com.alibaba.otter.canal.server.netty.NettyUtils;
+import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.WireFormat;
@@ -50,6 +50,7 @@ public class SessionHandler extends SimpleChannelHandler {
         this.embeddedServer = embeddedServer;
     }
 
+    @SuppressWarnings("deprecation")
     public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
         logger.info("message receives in session handler...");
         long start = System.nanoTime();

server/src/main/java/com/alibaba/otter/canal/server/netty/listener/ChannelFutureAggregator.java

@@ -2,7 +2,7 @@ package com.alibaba.otter.canal.server.netty.listener;
 
 import com.alibaba.otter.canal.protocol.CanalPacket;
 import com.google.common.base.Preconditions;
-import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.GeneratedMessageV3;
 import org.jboss.netty.channel.ChannelFuture;
 import org.jboss.netty.channel.ChannelFutureListener;
 
@@ -16,19 +16,19 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
 
     private ClientRequestResult result;
 
-    public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, boolean empty) {
+    public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, boolean empty) {
         this(destination, request, type, amount, latency, empty, (short) 0);
     }
 
-    public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency) {
+    public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency) {
         this(destination, request, type, amount, latency, false, (short) 0);
     }
 
-    public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, short errorCode) {
+    public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, short errorCode) {
         this(destination, request, type, amount, latency, false, errorCode);
     }
 
-    private ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, boolean empty, short errorCode) {
+    private ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, boolean empty, short errorCode) {
         this.result = new ClientRequestResult.Builder()
                 .destination(destination)
                 .type(type)
@@ -56,7 +56,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
 
         private String                 destination;
         private CanalPacket.PacketType type;
-        private GeneratedMessage       request;
+        private GeneratedMessageV3       request;
         private int                    amount;
         private long                   latency;
         private short                  errorCode;
@@ -81,7 +81,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
 
             private String                 destination;
             private CanalPacket.PacketType type;
-            private GeneratedMessage       request;
+            private GeneratedMessageV3       request;
             private int                    amount;
             private long                   latency;
             private short                  errorCode;
@@ -98,7 +98,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
                 return this;
             }
 
-            Builder request(GeneratedMessage request) {
+            Builder request(GeneratedMessageV3 request) {
                 this.request = request;
                 return this;
             }
@@ -153,7 +153,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
             return type;
         }
 
-        public GeneratedMessage getRequest() {
+        public GeneratedMessageV3 getRequest() {
             return request;
         }
 

server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java

@@ -1,6 +1,7 @@
 package com.alibaba.otter.canal.spi;
 
 import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
 import com.alibaba.otter.canal.protocol.Message;
 import java.io.IOException;
 
@@ -15,14 +16,21 @@ public interface CanalMQProducer {
     /**
      * Send canal message to related topic
      *
-     * @param topic MQ topic
+     * @param canalDestination canal mq destination
      * @param message canal message
      * @throws IOException
      */
-    void send(MQProperties.Topic topic, Message message) throws IOException;
+    void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException;
 
     /**
      * Stop MQ producer service
      */
     void stop();
+
+    interface Callback {
+
+        void commit();
+
+        void rollback();
+    }
 }

server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java

@@ -66,6 +66,8 @@ public class CanalServerWithNettyTest {
 
     @Test
     public void testAuth() {
+
+
         try {
             SocketChannel channel = SocketChannel.open();
             channel.connect(new InetSocketAddress("127.0.0.1", 1088));
@@ -80,7 +82,7 @@ public class CanalServerWithNettyTest {
             }
             //
             Handshake handshake = Handshake.parseFrom(p.getBody());
-            System.out.println(handshake.getSupportedCompressionsList());
+            System.out.println(handshake.getSupportedCompressions());
             //
             ClientAuth ca = ClientAuth.newBuilder()
                 .setUsername("")

sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java

@@ -33,10 +33,13 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
     private static final Logger    logger                        = LoggerFactory.getLogger(EntryEventSink.class);
     private static final int       maxFullTimes                  = 10;
     private CanalEventStore<Event> eventStore;
-    protected boolean              filterTransactionEntry        = false;                                        // 是否需要过滤事务头/尾
+    protected boolean              filterTransactionEntry        = false;                                        // 是否需要尽可能过滤事务头/尾
     protected boolean              filterEmtryTransactionEntry   = true;                                         // 是否需要过滤空的事务头/尾
     protected long                 emptyTransactionInterval      = 5 * 1000;                                     // 空的事务输出的频率
-    protected long                 emptyTransctionThresold       = 8192;                                         // 超过1024个事务头，输出一个
+    protected long                 emptyTransctionThresold       = 8192;                                         // 超过8192个事务头，输出一个
+
+    protected volatile long        lastTransactionTimestamp      = 0L;
+    protected AtomicLong           lastTransactionCount          = new AtomicLong(0L);
     protected volatile long        lastEmptyTransactionTimestamp = 0L;
     protected AtomicLong           lastEmptyTransactionCount     = new AtomicLong(0L);
     private AtomicLong             eventsSinkBlockingTime        = new AtomicLong(0L);
@@ -74,17 +77,7 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
     public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
                                                                                                            throws CanalSinkException,
                                                                                                            InterruptedException {
-        List rowDatas = entrys;
+        return sinkData(entrys, remoteAddress);
-        if (filterTransactionEntry) {
-            rowDatas = new ArrayList<CanalEntry.Entry>();
-            for (CanalEntry.Entry entry : entrys) {
-                if (entry.getEntryType() == EntryType.ROWDATA) {
-                    rowDatas.add(entry);
-                }
-            }
-        }
-
-        return sinkData(rowDatas, remoteAddress);
     }
 
     private boolean sinkData(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress)
@@ -97,17 +90,26 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
                 continue;
             }
 
-            Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
+            if (filterTransactionEntry
-            events.add(event);
+                && (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND)) {
+                long currentTimestamp = entry.getHeader().getExecuteTime();
+                // 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常
+                if (Math.abs(currentTimestamp - lastTransactionTimestamp) > emptyTransactionInterval
+                    || lastTransactionCount.incrementAndGet() > emptyTransctionThresold) {
+                    lastTransactionCount.set(0L);
+                    lastTransactionTimestamp = currentTimestamp;
+                    continue;
+                }
+            }
+
             hasRowData |= (entry.getEntryType() == EntryType.ROWDATA);
             hasHeartBeat |= (entry.getEntryType() == EntryType.HEARTBEAT);
+            Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
+            events.add(event);
         }
 
-        if (hasRowData) {
+        if (hasRowData || hasHeartBeat) {
-            // 存在row记录
+            // 存在row记录 或者 存在heartbeat记录，直接跳给后续处理
-            return doSink(events);
-        } else if (hasHeartBeat) {
-            // 存在heartbeat记录，直接跳给后续处理
             return doSink(events);
         } else {
             // 需要过滤的数据

store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java

@@ -35,16 +35,16 @@ import com.alibaba.otter.canal.store.model.Events;
  */
 public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge implements CanalEventStore<Event>, CanalStoreScavenge {
 
-    private static final long INIT_SQEUENCE = -1;
+    private static final long INIT_SEQUENCE = -1;
     private int               bufferSize    = 16 * 1024;
     private int               bufferMemUnit = 1024;                         // memsize的单位，默认为1kb大小
     private int               indexMask;
     private Event[]           entries;
 
     // 记录下put/get/ack操作的三个下标
-    private AtomicLong        putSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前put操作最后一次写操作发生的位置
+    private AtomicLong        putSequence   = new AtomicLong(INIT_SEQUENCE); // 代表当前put操作最后一次写操作发生的位置
-    private AtomicLong        getSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前get操作读取的最后一条的位置
+    private AtomicLong        getSequence   = new AtomicLong(INIT_SEQUENCE); // 代表当前get操作读取的最后一条的位置
-    private AtomicLong        ackSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前ack操作的最后一条的位置
+    private AtomicLong        ackSequence   = new AtomicLong(INIT_SEQUENCE); // 代表当前ack操作的最后一条的位置
 
     // 记录下put/get/ack操作的三个memsize大小
     private AtomicLong        putMemSize    = new AtomicLong(0);
@@ -358,17 +358,17 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
         lock.lock();
         try {
             long firstSeqeuence = ackSequence.get();
-            if (firstSeqeuence == INIT_SQEUENCE && firstSeqeuence < putSequence.get()) {
+            if (firstSeqeuence == INIT_SEQUENCE && firstSeqeuence < putSequence.get()) {
                 // 没有ack过数据
                 Event event = entries[getIndex(firstSeqeuence + 1)]; // 最后一次ack为-1，需要移动到下一条,included
                                                                      // = false
                 return CanalEventUtils.createPosition(event, false);
-            } else if (firstSeqeuence > INIT_SQEUENCE && firstSeqeuence < putSequence.get()) {
+            } else if (firstSeqeuence > INIT_SEQUENCE && firstSeqeuence < putSequence.get()) {
                 // ack未追上put操作
                 Event event = entries[getIndex(firstSeqeuence + 1)]; // 最后一次ack的位置数据
                                                                      // + 1
                 return CanalEventUtils.createPosition(event, true);
-            } else if (firstSeqeuence > INIT_SQEUENCE && firstSeqeuence == putSequence.get()) {
+            } else if (firstSeqeuence > INIT_SEQUENCE && firstSeqeuence == putSequence.get()) {
                 // 已经追上，store中没有数据
                 Event event = entries[getIndex(firstSeqeuence)]; // 最后一次ack的位置数据，和last为同一条，included
                                                                  // = false
@@ -387,10 +387,10 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
         lock.lock();
         try {
             long latestSequence = putSequence.get();
-            if (latestSequence > INIT_SQEUENCE && latestSequence != ackSequence.get()) {
+            if (latestSequence > INIT_SEQUENCE && latestSequence != ackSequence.get()) {
                 Event event = entries[(int) putSequence.get() & indexMask]; // 最后一次写入的数据，最后一条未消费的数据
                 return CanalEventUtils.createPosition(event, true);
-            } else if (latestSequence > INIT_SQEUENCE && latestSequence == ackSequence.get()) {
+            } else if (latestSequence > INIT_SEQUENCE && latestSequence == ackSequence.get()) {
                 // ack已经追上了put操作
                 Event event = entries[(int) putSequence.get() & indexMask]; // 最后一次写入的数据，included
                                                                             // =
@@ -435,7 +435,7 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
                     if (batchMode.isMemSize()) {
                         ackMemSize.addAndGet(memsize);
                         // 尝试清空buffer中的内存，将ack之前的内存全部释放掉
-                        for (long index = sequence + 1; index <= next; index++) {
+                        for (long index = sequence + 1; index < next; index++) {
                             entries[getIndex(index)] = null;// 设置为null
                         }
                     }
@@ -473,9 +473,9 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
         final ReentrantLock lock = this.lock;
         lock.lock();
         try {
-            putSequence.set(INIT_SQEUENCE);
+            putSequence.set(INIT_SEQUENCE);
-            getSequence.set(INIT_SQEUENCE);
+            getSequence.set(INIT_SEQUENCE);
-            ackSequence.set(INIT_SQEUENCE);
+            ackSequence.set(INIT_SEQUENCE);
 
             putMemSize.set(0);
             getMemSize.set(0);